├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.md │ ├── feature-request.md │ └── general-question.md └── pull_request_template.md ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── OWNERS ├── OWNERS_ALIASES ├── README.md ├── arbiter ├── README.md ├── README_CN.md ├── arbiter.json ├── arbiter.png ├── arbiter.rules.yml ├── checkpoint.go ├── checkpoint_test.go ├── config.go ├── config_test.go ├── metrics.go ├── metrics_test.go ├── server.go └── server_test.go ├── binlogctl ├── config.go ├── config_test.go ├── encrypt.go ├── meta.go ├── meta_test.go ├── nodes.go └── nodes_test.go ├── cmd ├── OWNERS ├── arbiter │ ├── arbiter.toml │ └── main.go ├── binlogctl │ ├── README.md │ └── main.go ├── drainer │ ├── README.md │ ├── drainer.toml │ └── main.go ├── pump │ ├── README.md │ ├── main.go │ └── pump.toml └── reparo │ ├── main.go │ └── reparo.toml ├── docs ├── architecture.png └── binlog_http_api.md ├── drainer ├── bench_test.go ├── binlog_item.go ├── checkpoint │ ├── checkpoint.go │ ├── file.go │ ├── file_test.go │ ├── mysql.go │ ├── mysql_test.go │ ├── oracle.go │ ├── oracle_test.go │ ├── util.go │ └── util_test.go ├── collector.go ├── collector_test.go ├── config.go ├── config_test.go ├── loopbacksync │ ├── loopbacksync.go │ └── loopbacksync_test.go ├── merge.go ├── merge_test.go ├── metrics.go ├── pump.go ├── pump_test.go ├── relay.go ├── relay │ ├── reader.go │ ├── reader_test.go │ ├── relayer.go │ └── relayer_test.go ├── relay_test.go ├── safepoint.go ├── schema.go ├── schema_test.go ├── server.go ├── server_test.go ├── status.go ├── sync │ ├── bench_kafka_test.go │ ├── kafka.go │ ├── kafka_test.go │ ├── mysql.go │ ├── mysql_test.go │ ├── oracle.go │ ├── oracle_test.go │ ├── pb.go │ ├── syncer.go │ ├── syncer_test.go │ └── util.go ├── syncer.go ├── syncer_test.go ├── translator │ ├── kafka.go │ ├── kafka_test.go │ ├── mysql.go │ ├── mysql_test.go │ ├── oracle.go │ ├── oracle_test.go │ ├── pb.go │ ├── pb_test.go │ ├── sequence_iterator.go │ ├── sequence_iterator_test.go │ ├── table_info.go │ ├── testing.go │ ├── translator.go │ └── translator_test.go ├── util.go └── util_test.go ├── generate-binlog.sh ├── gitcookie.sh ├── go.mod ├── go.sum ├── hack └── clean_vendor.sh ├── metrics ├── alertmanager │ └── binlog.rules.yml └── grafana │ └── binlog.json ├── pkg ├── binlogfile │ ├── binlogger.go │ ├── binlogger_test.go │ ├── decoder.go │ ├── decoder_test.go │ ├── encoder.go │ ├── file.go │ ├── file_test.go │ └── metrics.go ├── dml │ ├── dml.go │ └── dml_test.go ├── encrypt │ ├── encrypt.go │ └── encrypt_test.go ├── etcd │ ├── etcd.go │ └── etcd_test.go ├── file │ ├── lock.go │ └── lock_test.go ├── filter │ ├── filter.go │ └── filter_test.go ├── flags │ ├── flag.go │ ├── flag_test.go │ ├── urls.go │ └── urls_test.go ├── loader │ ├── README.md │ ├── bench_test.go │ ├── causality.go │ ├── causality_test.go │ ├── example_loader_test.go │ ├── executor.go │ ├── executor_test.go │ ├── load.go │ ├── load_test.go │ ├── merge.go │ ├── merge_test.go │ ├── model.go │ ├── model_test.go │ ├── translate.go │ ├── translate_test.go │ ├── util.go │ └── util_test.go ├── node │ ├── node.go │ ├── node_test.go │ ├── registry.go │ └── registry_test.go ├── security │ ├── security.go │ └── security_test.go ├── sql │ ├── sql.go │ └── sql_test.go ├── types │ ├── urls.go │ └── urls_test.go ├── util │ ├── duration.go │ ├── duration_test.go │ ├── http.go │ ├── http_test.go │ ├── kafka.go │ ├── kafka_test.go │ ├── log.go │ ├── log_test.go │ ├── net.go │ ├── net_test.go │ ├── p8s.go │ ├── p8s_test.go │ ├── signal.go │ ├── signal_test.go │ ├── ts.go │ ├── ts_test.go │ ├── util.go │ └── util_test.go ├── version │ └── version.go └── zk │ ├── zk.go │ ├── zk_test.go │ └── zkmock_test.go ├── proto ├── binlog │ └── binlog.pb.go └── pb_binlog.proto ├── pump ├── config.go ├── config_test.go ├── metrics.go ├── node.go ├── node_test.go ├── server.go ├── server_test.go ├── status.go └── storage │ ├── bench_test.go │ ├── chaser.go │ ├── chaser_test.go │ ├── errors.go │ ├── helper.go │ ├── log.go │ ├── log_default.go │ ├── log_linux.go │ ├── log_test.go │ ├── metrics.go │ ├── sorter.go │ ├── sorter_test.go │ ├── storage.go │ ├── storage_test.go │ ├── util.go │ ├── util_test.go │ ├── vlog.go │ └── vlog_test.go ├── reparo ├── README.md ├── config.go ├── config_test.go ├── ddl.go ├── ddl_test.go ├── decode.go ├── decode_test.go ├── file.go ├── file_test.go ├── read.go ├── read_test.go ├── reparo.go ├── reparo_test.go └── syncer │ ├── memory.go │ ├── memory_test.go │ ├── mysql.go │ ├── mysql_test.go │ ├── print.go │ ├── print_test.go │ ├── syncer.go │ ├── syncer_test.go │ ├── translate.go │ ├── translate_test.go │ ├── util.go │ └── util_test.go ├── scripts └── groovy │ ├── binlog_ghpr_build.groovy │ ├── binlog_ghpr_check.groovy │ ├── binlog_ghpr_integration.groovy │ └── binlog_ghpr_unit_test.groovy ├── tests ├── README.md ├── _utils │ ├── check_contains │ ├── check_data │ ├── check_not_contains │ ├── check_status │ ├── down_run_sql │ ├── run_drainer │ ├── run_pump │ ├── run_reparo │ └── run_sql ├── attributes │ ├── drainer.toml │ └── run.sh ├── binlog │ ├── binlog.go │ ├── config.toml │ ├── drainer.toml │ └── run.sh ├── binlogfilter │ ├── drainer.toml │ └── run.sh ├── cache_table │ ├── drainer.toml │ └── run.sh ├── dailytest │ ├── case.go │ ├── dailytest.go │ ├── data.go │ ├── db.go │ ├── ddl.go │ ├── exector.go │ ├── job.go │ ├── parser.go │ └── rand.go ├── filter │ ├── drainer.toml │ └── run.sh ├── gencol │ ├── drainer.toml │ └── run.sh ├── kafka │ ├── drainer.toml │ ├── kafka.go │ └── run.sh ├── partition │ ├── drainer.toml │ └── run.sh ├── placement_rules │ ├── drainer.toml │ └── run.sh ├── reparo │ ├── binlog.go │ ├── config.toml │ ├── drainer.toml │ ├── reparo.toml │ ├── run.sh │ └── sync_diff_inspector.toml ├── resource_control │ ├── drainer.toml │ └── run.sh ├── restart │ ├── drainer.toml │ ├── insert_data │ ├── run.sh │ └── sync_diff_inspector.toml ├── run.sh ├── sequence │ ├── drainer.toml │ └── run.sh ├── status │ ├── drainer.toml │ └── run.sh └── util │ ├── config.go │ └── db.go └── tools └── check ├── check-tidy.sh └── revive.toml /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41B Bug Report" 3 | about: Something isn't working as expected 4 | labels: bug 5 | --- 6 | 7 | ## Bug Report 8 | 9 | Please answer these questions before submitting your issue. Thanks! 10 | 11 | 1. What did you do? 12 | If possible, provide a recipe for reproducing the error. 13 | 14 | 15 | 2. What did you expect to see? 16 | 17 | 18 | 19 | 3. What did you see instead? 20 | 21 | 22 | 23 | 4. Please provide the relate downstream type and version of drainer. 24 | (run `drainer -V` in terminal to get drainer's version) 25 | 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature Request" 3 | about: I have a suggestion 4 | labels: feature-request 5 | --- 6 | 7 | ## Feature Request 8 | 9 | **Is your feature request related to a problem? Please describe:** 10 | 11 | 12 | **Describe the feature you'd like:** 13 | 14 | 15 | **Describe alternatives you've considered:** 16 | 17 | 18 | **Teachability, Documentation, Adoption, Migration Strategy:** 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/general-question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F914 General Question" 3 | about: Usage question that isn't answered in docs or discussion 4 | labels: question 5 | --- 6 | 7 | ## General Question 8 | 9 | Before asking a question, make sure you have: 10 | 11 | - Searched existing Stack Overflow questions. 12 | - Googled your question. 13 | - Searched open and closed [GitHub issues](https://github.com/pingcap/tidb-binlog/issues?utf8=%E2%9C%93&q=is%3Aissue) 14 | - Read related documentation: 15 | - [TiDB Doc](https://pingcap.com/docs/) -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 4 | 5 | ### What problem does this PR solve? 6 | 14 | 15 | Issue Number: close #xxx 16 | 17 | ### What is changed and how it works? 18 | 19 | 20 | ### Check List 21 | 22 | Tests 23 | 24 | - Unit test 25 | - Integration test 26 | - Manual test (add detailed scripts or steps below) 27 | - No code 28 | 29 | Code changes 30 | 31 | - Has exported function/method change 32 | - Has exported variable/fields change 33 | - Has interface methods change 34 | - Has persistent data change 35 | 36 | Side effects 37 | 38 | - Possible performance regression 39 | - Increased code complexity 40 | - Breaking backward compatibility 41 | 42 | Related changes 43 | 44 | - Need to cherry-pick to the release branch 45 | - Need to update the documentation 46 | - Need to update the `tidb-ansible` repository 47 | - Need to be included in the release note 48 | 49 | ### Release note 50 | 51 | 52 | 53 | - No release note 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | *.log 26 | 27 | bin 28 | *.iml 29 | .idea 30 | .DS_Store 31 | 32 | cscope.* 33 | **/*.swp 34 | 35 | # Files generated when testing 36 | out 37 | vendor/ -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # TiDB-Binlog 2 | All notable changes to this project are documented in this file. 3 | 4 | ## [2.1.3] 5 | + Pump 6 | - Add a API to get binlog by ts [#449](https://github.com/pingcap/tidb-binlog/pull/449) 7 | 8 | + Drainer 9 | - Fix history job not sorted by schema version [#444](https://github.com/pingcap/tidb-binlog/pull/444) 10 | - Change default config value to more reasonable value [#439](https://github.com/pingcap/tidb-binlog/pull/439) [#442](https://github.com/pingcap/tidb-binlog/pull/442) 11 | - Fix not skip rollback state ddl job [#432](https://github.com/pingcap/tidb-binlog/pull/432) 12 | - Fix data may not consistent when no pk but has uk [#421](https://github.com/pingcap/tidb-binlog/pull/421) 13 | 14 | + Other 15 | - Add integration test for node status [#416](https://github.com/pingcap/tidb-binlog/pull/416) 16 | 17 | ## [2.1.4] 18 | - Nothing's changed 19 | 20 | ## [2.1.5] 21 | - Update the DDL binlog replication plan to guarantee the correctness of DDL [#466](https://github.com/pingcap/tidb-binlog/pull/466) 22 | - Switch juju/errors to pingcap/error [#464](https://github.com/pingcap/tidb-binlog/pull/464) 23 | - Open go mod and update despondencies [#475](https://github.com/pingcap/tidb-binlog/pull/475) 24 | - Add package pkg/loader [#471](https://github.com/pingcap/tidb-binlog/pull/471) 25 | - Add tool Arbiter sync from Kafka to Mysql [#441](https://github.com/pingcap/tidb-binlog/pull/441) 26 | 27 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | approvers: 3 | - 3AceShowHand 4 | - 3pointer 5 | - 5kbpers 6 | - amyangfei 7 | - asddongmen 8 | - Benjamin2037 9 | - buchuitoudegou 10 | - CharlesCheung96 11 | - csuzhangxc 12 | - D3Hunter 13 | - dsdashun 14 | - Ehco1996 15 | - glorv 16 | - GMHDBJD 17 | - gozssky 18 | - hicqu 19 | - holys 20 | - hongyunyan 21 | - IANTHEREAL 22 | - july2993 23 | - kennytm 24 | - lance6716 25 | - Leavrth 26 | - leoppro 27 | - lichunzhu 28 | - lidezhu 29 | - Little-Wallace 30 | - liuzix 31 | - lonng 32 | - maxshuang 33 | - niubell 34 | - okJiang 35 | - overvenus 36 | - Rustin170506 37 | - sdojjy 38 | - suzaku 39 | - Tammyxia 40 | - WangXiangUSTC 41 | - WizardXiao 42 | - wk989898 43 | - YuJuncen 44 | - zhangjinpeng87 45 | - zhaoxinyu 46 | - zwj-coder 47 | reviewers: 48 | - ben1009 49 | - charleszheng44 50 | - fengou1 51 | - iamxy 52 | - joccau 53 | - MoCuishle28 54 | - nongfushanquan 55 | - tiancaiamao 56 | -------------------------------------------------------------------------------- /OWNERS_ALIASES: -------------------------------------------------------------------------------- 1 | # Sort the member alphabetically. 2 | aliases: 3 | sig-critical-approvers-config: 4 | - Benjamin2037 5 | -------------------------------------------------------------------------------- /arbiter/README.md: -------------------------------------------------------------------------------- 1 | Arbiter 2 | ========== 3 | 4 | **Arbiter** is a tool used for syncing data from Kafka to TiDB incrementally. 5 | 6 | ![](./arbiter.png) 7 | 8 | The complete import process is as follows: 9 | 10 | 1. Read Binlog from Kafka in the format of [Protobuf](https://github.com/pingcap/tidb-tools/blob/master/tidb-binlog/proto/proto/binlog.proto). 11 | 2. While reaching a limit data size, construct the SQL according the Binlog and write to downstream concurrently(notice: Arbiter will split the upstream transaction). 12 | 3. Save the checkpoint. 13 | 14 | 15 | ## Checkpoint 16 | `arbiter` will write a record to the table `tidb_binlog.arbiter_checkpoint` at downstream TiDB. 17 | ``` 18 | mysql> select * from tidb_binlog.arbiter_checkpoint; 19 | +-------------+--------------------+--------+ 20 | | topic_name | ts | status | 21 | +-------------+--------------------+--------+ 22 | | test_kafka4 | 405809779094585347 | 1 | 23 | +-------------+--------------------+--------+ 24 | ``` 25 | - topic_name: the topic name of Kafka to consume. 26 | - ts: the timestamp checkpoint 27 | - status: 28 | * 0 29 | All Binlog data <= ts has synced to downstream. 30 | * 1 31 | means `Arbiter` is running or quit unexpectedly, Binlog with timestamp bigger than ts may partially synced to downstream. 32 | 33 | 34 | 35 | ## Monitor 36 | 37 | Arbiter supports metrics collection via [Prometheus](https://prometheus.io/). 38 | 39 | ###Metrics 40 | 41 | * **`binlog_arbiter_checkpoint_tso`** (Gauge) 42 | 43 | Corresponding to ts in table `tidb_binlog.arbiter_checkpoint` 44 | 45 | * **`binlog_arbiter_query_duration_time`** (Histogram) 46 | 47 | Bucketed histogram of the time needed to wirte to downstream. Labels: 48 | 49 | * **type**: `exec` `commit` time takes to execute and commit SQL. 50 | 51 | * **`binlog_arbiter_event`** (Counter) 52 | 53 | Event times counter. Labels: 54 | 55 | * **type**: e.g. `DDL` `Insert` `Update` `Delete` `Txn` 56 | 57 | * **`binlog_arbiter_queue_size`** (Gauge) 58 | 59 | Queue size. Labels: 60 | 61 | * **name**: e.g. `kafka_reader` `loader_input` 62 | 63 | * **`binlog_arbiter_txn_latency_seconds`** (Histogram) 64 | 65 | Bucketed histogram of the time duration between the time write to downstream and commit time of upstream transaction(phsical part of commitTS). 66 | 67 | 68 | -------------------------------------------------------------------------------- /arbiter/arbiter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingcap/tidb-binlog/552cffbb46230d7b4a41995379423ae9d2351a47/arbiter/arbiter.png -------------------------------------------------------------------------------- /arbiter/arbiter.rules.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: alert.rules 3 | rules: 4 | - alert: binlog_arbiter_checkpoint_high_delay 5 | expr: (time() - binlog_arbiter_checkpoint_tso / 1000) > 3600 6 | for: 1m 7 | labels: 8 | env: test-cluster 9 | level: warning 10 | expr: (time() - binlog_arbiter_checkpoint_tso / 1000) > 3600 11 | annotations: 12 | description: 'cluster: test-cluster, instance: {{ $labels.instance }}, values: {{ $value }}' 13 | value: '{{ $value }}' 14 | summary: arbiter arbiter checkpoint delay more than 1 hour 15 | 16 | - alert: binlog_arbiter_checkpoint_tso_no_change_for_1m 17 | expr: changes(binlog_arbiter_checkpoint_tso[1m]) < 1 18 | labels: 19 | env: test-cluster 20 | level: warning 21 | expr: changes(binlog_arbiter_checkpoint_tso[1m]) < 1 22 | annotations: 23 | description: 'cluster: test-cluster, instance: {{ $labels.instance }}, values: {{ $value }}' 24 | value: '{{ $value }}' 25 | summary: binlog arbiter checkpoint tso no change for 1m 26 | -------------------------------------------------------------------------------- /arbiter/checkpoint_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package arbiter 15 | 16 | import ( 17 | "fmt" 18 | "testing" 19 | 20 | gosql "database/sql" 21 | sqlmock "github.com/DATA-DOG/go-sqlmock" 22 | check "github.com/pingcap/check" 23 | "github.com/pingcap/errors" 24 | pkgsql "github.com/pingcap/tidb-binlog/pkg/sql" 25 | ) 26 | 27 | func Test(t *testing.T) { check.TestingT(t) } 28 | 29 | type CheckpointSuite struct { 30 | } 31 | 32 | var _ = check.Suite(&CheckpointSuite{}) 33 | 34 | func setNewExpect(mock sqlmock.Sqlmock) { 35 | mock.ExpectExec("CREATE DATABASE IF NOT EXISTS").WillReturnResult(sqlmock.NewResult(0, 1)) 36 | mock.ExpectExec("CREATE TABLE IF NOT EXISTS").WillReturnResult(sqlmock.NewResult(0, 1)) 37 | } 38 | 39 | func (cs *CheckpointSuite) TestNewCheckpoint(c *check.C) { 40 | db, mock, err := sqlmock.New() 41 | c.Assert(err, check.IsNil) 42 | 43 | setNewExpect(mock) 44 | 45 | _, err = createDbCheckpoint(db) 46 | c.Assert(err, check.IsNil) 47 | 48 | c.Assert(mock.ExpectationsWereMet(), check.IsNil) 49 | } 50 | 51 | func (cs *CheckpointSuite) TestSaveAndLoad(c *check.C) { 52 | db, mock, err := sqlmock.New() 53 | c.Assert(err, check.IsNil) 54 | 55 | setNewExpect(mock) 56 | cp, err := createDbCheckpoint(db) 57 | c.Assert(err, check.IsNil) 58 | sql := fmt.Sprintf("SELECT (.+) FROM %s WHERE topic_name = ?", 59 | pkgsql.QuoteSchema(cp.database, cp.table)) 60 | mock.ExpectQuery(sql).WithArgs(cp.topicName). 61 | WillReturnError(errors.NotFoundf("no checkpoint for: %s", cp.topicName)) 62 | 63 | _, _, err = cp.Load() 64 | c.Log(err) 65 | c.Assert(errors.IsNotFound(err), check.IsTrue) 66 | 67 | var saveTS int64 = 10 68 | saveStatus := 1 69 | mock.ExpectExec("REPLACE INTO"). 70 | WithArgs(cp.topicName, saveTS, saveStatus). 71 | WillReturnResult(sqlmock.NewResult(0, 1)) 72 | err = cp.Save(saveTS, saveStatus) 73 | c.Assert(err, check.IsNil) 74 | 75 | rows := sqlmock.NewRows([]string{"ts", "status"}). 76 | AddRow(saveTS, saveStatus) 77 | mock.ExpectQuery("SELECT ts, status FROM").WillReturnRows(rows) 78 | ts, status, err := cp.Load() 79 | c.Assert(err, check.IsNil) 80 | c.Assert(ts, check.Equals, saveTS) 81 | c.Assert(status, check.Equals, saveStatus) 82 | } 83 | 84 | func createDbCheckpoint(db *gosql.DB) (*dbCheckpoint, error) { 85 | cp, err := NewCheckpoint(db, "topic_name") 86 | return cp.(*dbCheckpoint), err 87 | } 88 | -------------------------------------------------------------------------------- /arbiter/metrics_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package arbiter 15 | 16 | import ( 17 | . "github.com/pingcap/check" 18 | "github.com/pingcap/errors" 19 | ) 20 | 21 | type instanceNameSuite struct{} 22 | 23 | var _ = Suite(&instanceNameSuite{}) 24 | 25 | func (s *instanceNameSuite) TestShouldRetUnknown(c *C) { 26 | orig := getHostname 27 | defer func() { 28 | getHostname = orig 29 | }() 30 | getHostname = func() (string, error) { 31 | return "", errors.New("host") 32 | } 33 | 34 | n := instanceName(9090) 35 | c.Assert(n, Equals, "unknown") 36 | } 37 | 38 | func (s *instanceNameSuite) TestShouldUseHostname(c *C) { 39 | orig := getHostname 40 | defer func() { 41 | getHostname = orig 42 | }() 43 | getHostname = func() (string, error) { 44 | return "kendoka", nil 45 | } 46 | 47 | n := instanceName(9090) 48 | c.Assert(n, Equals, "kendoka_9090") 49 | } 50 | -------------------------------------------------------------------------------- /binlogctl/config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package binlogctl 15 | 16 | import ( 17 | . "github.com/pingcap/check" 18 | ) 19 | 20 | type configSuite struct{} 21 | 22 | var _ = Suite(&configSuite{}) 23 | 24 | func (s *configSuite) TestConfig(c *C) { 25 | config := NewConfig() 26 | args := []string{"-pd-urls=127.0.0.1"} 27 | err := config.Parse(args) 28 | c.Assert(err, ErrorMatches, ".*parse EtcdURLs error.*") 29 | 30 | args = []string{"-cmd=pumps", "-node-id=nodeID", "-pd-urls=127.0.0.1:2379"} 31 | err = config.Parse(args) 32 | c.Assert(err, IsNil) 33 | c.Assert(config.Command, Equals, QueryPumps) 34 | c.Assert(config.NodeID, Equals, "nodeID") 35 | c.Assert(config.EtcdURLs, Equals, "127.0.0.1:2379") 36 | } 37 | -------------------------------------------------------------------------------- /binlogctl/encrypt.go: -------------------------------------------------------------------------------- 1 | package binlogctl 2 | 3 | import ( 4 | "github.com/pingcap/log" 5 | "github.com/pingcap/tidb-binlog/pkg/encrypt" 6 | "go.uber.org/zap" 7 | ) 8 | 9 | // EncryptHandler log the encrypted text if success or return error. 10 | func EncryptHandler(text string) error { 11 | enc, err := encrypt.Encrypt(text) 12 | if err != nil { 13 | return err 14 | } 15 | 16 | log.Info("encrypt text", zap.String("encrypted", string(enc))) 17 | return nil 18 | } 19 | -------------------------------------------------------------------------------- /binlogctl/meta_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package binlogctl 15 | 16 | import ( 17 | "context" 18 | "os" 19 | "path" 20 | "strings" 21 | 22 | . "github.com/pingcap/check" 23 | pd "github.com/tikv/pd/client" 24 | ) 25 | 26 | type metaSuite struct{} 27 | 28 | var _ = Suite(&metaSuite{}) 29 | 30 | func (s *metaSuite) TestMeta(c *C) { 31 | meta := &Meta{ 32 | CommitTS: 123, 33 | } 34 | metaStr := meta.String() 35 | c.Assert(metaStr, Equals, "commitTS: 123") 36 | } 37 | 38 | func (s *metaSuite) TestSaveMeta(c *C) { 39 | dir := c.MkDir() 40 | filename := path.Join(dir, "savepoint") 41 | err := saveMeta(filename, 123, "Local") 42 | c.Assert(err, IsNil) 43 | 44 | b, err := os.ReadFile(filename) 45 | c.Assert(err, IsNil) 46 | lines := strings.Split(strings.TrimSpace(string(b)), "\n") 47 | c.Assert(lines, HasLen, 3) 48 | c.Assert(lines[0], Equals, "commitTS = 123") 49 | c.Assert(lines[1], Equals, "1970-01-01 00:00:00 +0000 UTC") 50 | // the output depends on the local's timezone 51 | c.Assert(lines[2], Matches, "1970-01-0.*") 52 | } 53 | 54 | type dummyCli struct { 55 | pd.Client 56 | physical, logical int64 57 | err error 58 | } 59 | 60 | func (c dummyCli) GetTS(ctx context.Context) (int64, int64, error) { 61 | return c.physical, c.logical, c.err 62 | } 63 | 64 | func newFakePDClient([]string, pd.SecurityOption, ...pd.ClientOption) (pd.Client, error) { 65 | return &dummyCli{ 66 | physical: 123, 67 | logical: 456, 68 | }, nil 69 | } 70 | 71 | func (s *metaSuite) TestGenerateMetaInfo(c *C) { 72 | newPDClientFunc = newFakePDClient 73 | defer func() { 74 | newPDClientFunc = pd.NewClient 75 | }() 76 | 77 | dir := c.MkDir() 78 | cfg := &Config{ 79 | DataDir: dir, 80 | EtcdURLs: "127.0.0.1:2379", 81 | } 82 | 83 | err := GenerateMetaInfo(cfg) 84 | c.Assert(err, IsNil) 85 | 86 | b, err := os.ReadFile(path.Join(dir, "savepoint")) 87 | c.Assert(err, IsNil) 88 | lines := strings.Split(strings.TrimSpace(string(b)), "\n") 89 | c.Assert(lines, HasLen, 1) 90 | c.Assert(lines[0], Equals, "commitTS = 32244168") 91 | } 92 | -------------------------------------------------------------------------------- /cmd/OWNERS: -------------------------------------------------------------------------------- 1 | # See the OWNERS docs at https://go.k8s.io/owners 2 | options: 3 | no_parent_owners: true 4 | approvers: 5 | - sig-critical-approvers-config 6 | -------------------------------------------------------------------------------- /cmd/arbiter/arbiter.toml: -------------------------------------------------------------------------------- 1 | # Arbiter Configuration. 2 | 3 | # addr (i.e. 'host:port') to listen on for Arbiter connections 4 | # addr = "127.0.0.1:8251" 5 | 6 | [up] 7 | # if arbiter donesn't have checkpoint, use initial commitTS to initial checkpoint 8 | initial-commit-ts = 0 9 | kafka-addrs = "127.0.0.1:9092" 10 | kafka-version = "0.8.2.0" 11 | # topic name of kafka to consume binlog 12 | #topic = "" 13 | 14 | 15 | [down] 16 | host = "localhost" 17 | port = 3306 18 | user = "root" 19 | password = "" 20 | # max concurrent write to downstream 21 | # worker-count = 16 22 | # max DML operation in a transaction when write to downstream 23 | # batch-size = 64 24 | # safe-mode = false 25 | -------------------------------------------------------------------------------- /cmd/arbiter/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "io" 18 | stdlog "log" 19 | "net/http" 20 | _ "net/http/pprof" 21 | "os" 22 | 23 | "github.com/Shopify/sarama" 24 | _ "github.com/go-sql-driver/mysql" 25 | "github.com/pingcap/log" 26 | "github.com/pingcap/tidb-binlog/arbiter" 27 | "github.com/pingcap/tidb-binlog/pkg/util" 28 | "github.com/pingcap/tidb-binlog/pkg/version" 29 | "github.com/prometheus/client_golang/prometheus" 30 | "github.com/prometheus/client_golang/prometheus/promhttp" 31 | "go.uber.org/zap" 32 | ) 33 | 34 | func main() { 35 | cfg := arbiter.NewConfig() 36 | if err := cfg.Parse(os.Args[1:]); err != nil { 37 | log.Fatal("verifying flags failed. See 'arbiter --help'.", zap.Error(err)) 38 | } 39 | 40 | if err := util.InitLogger(cfg.LogLevel, cfg.LogFile); err != nil { 41 | log.Fatal("Failed to initialize log", zap.Error(err)) 42 | } 43 | 44 | // We have set sarama.Logger in util.InitLogger. 45 | if !cfg.OpenSaramaLog { 46 | // may too many noise, discard sarama log now 47 | sarama.Logger = stdlog.New(io.Discard, "[Sarama] ", stdlog.LstdFlags) 48 | } 49 | 50 | log.Info("start arbiter...", zap.Reflect("config", cfg)) 51 | version.PrintVersionInfo("Arbiter") 52 | 53 | go startHTTPServer(cfg.ListenAddr) 54 | 55 | srv, err := arbiter.NewServer(cfg) 56 | if err != nil { 57 | log.Error("new server failed", zap.Error(err)) 58 | return 59 | } 60 | 61 | util.SetupSignalHandler(func(_ os.Signal) { 62 | srv.Close() 63 | }) 64 | 65 | log.Info("start run server...") 66 | err = srv.Run() 67 | if err != nil { 68 | log.Error("run server failed", zap.Error(err)) 69 | } 70 | 71 | log.Info("server exit") 72 | } 73 | 74 | func startHTTPServer(addr string) { 75 | prometheus.DefaultGatherer = arbiter.Registry 76 | http.Handle("/metrics", promhttp.Handler()) 77 | 78 | err := http.ListenAndServe(addr, nil) 79 | if err != nil { 80 | log.Fatal("listen and server http failed", zap.String("addr", addr), zap.Error(err)) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /cmd/binlogctl/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "flag" 18 | "os" 19 | 20 | "github.com/pingcap/errors" 21 | "github.com/pingcap/log" 22 | ctl "github.com/pingcap/tidb-binlog/binlogctl" 23 | "github.com/pingcap/tidb-binlog/pkg/node" 24 | "go.uber.org/zap" 25 | ) 26 | 27 | const ( 28 | pause = "pause" 29 | close = "close" 30 | ) 31 | 32 | func main() { 33 | cfg := ctl.NewConfig() 34 | err := cfg.Parse(os.Args[1:]) 35 | switch err { 36 | case nil: 37 | case flag.ErrHelp: 38 | os.Exit(0) 39 | default: 40 | log.Error("parse cmd flags", zap.Error(err)) 41 | os.Exit(2) 42 | } 43 | 44 | switch cfg.Command { 45 | case ctl.GenerateMeta: 46 | err = ctl.GenerateMetaInfo(cfg) 47 | case ctl.QueryPumps: 48 | err = ctl.QueryNodesByKind(cfg.EtcdURLs, node.PumpNode, cfg.ShowOfflineNodes, cfg.TLS) 49 | case ctl.QueryDrainers: 50 | err = ctl.QueryNodesByKind(cfg.EtcdURLs, node.DrainerNode, cfg.ShowOfflineNodes, cfg.TLS) 51 | case ctl.UpdatePump: 52 | err = ctl.UpdateNodeState(cfg.EtcdURLs, node.PumpNode, cfg.NodeID, cfg.State, cfg.TLS) 53 | case ctl.UpdateDrainer: 54 | err = ctl.UpdateNodeState(cfg.EtcdURLs, node.DrainerNode, cfg.NodeID, cfg.State, cfg.TLS) 55 | case ctl.PausePump: 56 | err = ctl.ApplyAction(cfg.EtcdURLs, node.PumpNode, cfg.NodeID, pause, cfg.TLS) 57 | case ctl.PauseDrainer: 58 | err = ctl.ApplyAction(cfg.EtcdURLs, node.DrainerNode, cfg.NodeID, pause, cfg.TLS) 59 | case ctl.OfflinePump: 60 | err = ctl.ApplyAction(cfg.EtcdURLs, node.PumpNode, cfg.NodeID, close, cfg.TLS) 61 | case ctl.OfflineDrainer: 62 | err = ctl.ApplyAction(cfg.EtcdURLs, node.DrainerNode, cfg.NodeID, close, cfg.TLS) 63 | case ctl.Encrypt: 64 | if len(cfg.Text) == 0 { 65 | err = errors.New("need to specify the text to be encrypt") 66 | } else { 67 | err = ctl.EncryptHandler(cfg.Text) 68 | } 69 | default: 70 | err = errors.NotSupportedf("cmd %s", cfg.Command) 71 | } 72 | 73 | if err != nil { 74 | log.Fatal("fail to execute command", zap.String("command", cfg.Command), zap.Error(err)) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /cmd/drainer/README.md: -------------------------------------------------------------------------------- 1 | ## drainer 2 | 3 | drainer collects binlog from each pump in cluster, transforms binlog to various dialects of SQL, and applies to downstream database or filesystem. 4 | 5 | ## How to use 6 | 7 | ``` 8 | Usage of drainer: 9 | -L string 10 | log level: debug, info, warn, error, fatal (default "info") 11 | -V print version info 12 | -addr string 13 | addr (i.e. 'host:port') to listen on for drainer connections (default "127.0.0.1:8249") 14 | -c int 15 | parallel worker count (default 1) 16 | -cache-binlog-count int 17 | blurry count of binlogs in cache, limit cache size (default 65536) 18 | -config string 19 | path to the configuration file 20 | -data-dir string 21 | drainer data directory path (default data.drainer) (default "data.drainer") 22 | -dest-db-type string 23 | target db type: mysql or tidb or file or kafka; see syncer section in conf/drainer.toml (default "mysql") 24 | -detect-interval int 25 | the interval time (in seconds) of detect pumps' status (default 10) 26 | -disable-detect 27 | disbale detect causality 28 | -disable-dispatch 29 | disable dispatching sqls that in one same binlog; if set true, work-count and txn-batch would be useless 30 | -ignore-schemas string 31 | disable sync those schemas (default "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql") 32 | -initial-commit-ts int 33 | if drainer donesn't have checkpoint, use initial commitTS to initial checkpoint 34 | -kafka-addrs string 35 | a comma separated list of the kafka broker endpoints (default "127.0.0.1:9092") 36 | -log-file string 37 | log file path 38 | -log-rotate string 39 | log file rotate type, hour/day 40 | -metrics-addr string 41 | prometheus pushgateway address, leaves it empty will disable prometheus push 42 | -metrics-interval int 43 | prometheus client push interval in second, set "0" to disable prometheus push (default 15) 44 | -node-id string 45 | the ID of drainer node; if not specified, we will generate one from hostname and the listening port 46 | -pd-urls string 47 | a comma separated list of PD endpoints (default "http://127.0.0.1:2379") 48 | -safe-mode 49 | enable safe mode to make syncer reentrant 50 | -txn-batch int 51 | number of binlog events in a transaction batch (default 1) 52 | -zookeeper-addrs string 53 | a comma separated list of the zookeeper endpoints 54 | ``` 55 | 56 | 57 | ## Example 58 | 59 | ``` 60 | ./bin/drainer -pd-urls http://127.0.0.1:2379 \ 61 | -data-dir ./data.drainer 62 | ``` 63 | or use configuration file 64 | 65 | ``` 66 | ./bin/drainer -config ./conf/drainer.toml 67 | ``` 68 | -------------------------------------------------------------------------------- /cmd/drainer/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "math/rand" 18 | _ "net/http/pprof" 19 | "os" 20 | "os/signal" 21 | "runtime" 22 | "syscall" 23 | "time" 24 | 25 | "github.com/pingcap/log" 26 | "github.com/pingcap/tidb-binlog/drainer" 27 | "github.com/pingcap/tidb-binlog/pkg/util" 28 | "github.com/pingcap/tidb-binlog/pkg/version" 29 | "go.uber.org/zap" 30 | _ "google.golang.org/grpc/encoding/gzip" 31 | ) 32 | 33 | func main() { 34 | runtime.GOMAXPROCS(runtime.NumCPU()) 35 | rand.Seed(time.Now().UTC().UnixNano()) 36 | 37 | cfg := drainer.NewConfig() 38 | if err := cfg.Parse(os.Args[1:]); err != nil { 39 | log.Fatal("verifying flags failed, See 'drainer --help'.", zap.Error(err)) 40 | } 41 | 42 | if err := util.InitLogger(cfg.LogLevel, cfg.LogFile); err != nil { 43 | log.Fatal("Failed to initialize log", zap.Error(err)) 44 | } 45 | version.PrintVersionInfo("Drainer") 46 | log.Info("start drainer...", zap.Reflect("config", cfg)) 47 | 48 | bs, err := drainer.NewServer(cfg) 49 | if err != nil { 50 | log.Fatal("create drainer server failed", zap.Error(err)) 51 | } 52 | 53 | sc := make(chan os.Signal, 1) 54 | 55 | signal.Notify(sc, 56 | syscall.SIGHUP, 57 | syscall.SIGINT, 58 | syscall.SIGTERM, 59 | syscall.SIGQUIT) 60 | 61 | go func() { 62 | sig := <-sc 63 | log.Info("got signal to exit.", zap.Stringer("signal", sig)) 64 | bs.Close() 65 | os.Exit(0) 66 | }() 67 | 68 | if err := bs.Start(); err != nil { 69 | log.Error("start drainer server failed", zap.Error(err)) 70 | os.Exit(2) 71 | } 72 | 73 | log.Info("drainer exit") 74 | } 75 | -------------------------------------------------------------------------------- /cmd/pump/README.md: -------------------------------------------------------------------------------- 1 | ## pump 2 | 3 | pump is a daemon that receives realtime binlog from tidb-server and writes in sequential disk files synchronously. 4 | 5 | ## How to use 6 | 7 | ``` 8 | Usage of pump: 9 | -L string 10 | log level: debug, info, warn, error, fatal (default "info") 11 | -V print pump version info 12 | -addr string 13 | addr(i.e. 'host:port') to listen on for client traffic (default "127.0.0.1:8250") 14 | -advertise-addr string 15 | addr(i.e. 'host:port') to advertise to the public 16 | -config string 17 | path to the pump configuration file 18 | -data-dir string 19 | the path to store binlog data 20 | -gc int 21 | recycle binlog files older than gc days, zero means never recycle (default 7) 22 | -heartbeat-interval int 23 | number of seconds between heartbeat ticks (default 2) 24 | -kafka-addrs string 25 | a comma separated list of the kafka broker endpoints (default "127.0.0.1:9092") 26 | -log-file string 27 | log file path 28 | -log-rotate string 29 | log file rotate type, hour/day 30 | -max-message-size int 31 | max msg size producer produce into kafka (default 1073741824) 32 | -metrics-addr string 33 | prometheus pushgateway address, leaves it empty will disable prometheus push 34 | -metrics-interval int 35 | prometheus client push interval in second, set "0" to disable prometheus push (default 15) 36 | -node-id string 37 | the ID of pump node; if not specified, we will generate one from hostname and the listening port 38 | -pd-urls string 39 | a comma separated list of the PD endpoints (default "http://127.0.0.1:2379") 40 | -zookeeper-addrs string 41 | a comma separated list of the zookeeper broker endpoints 42 | ``` 43 | 44 | 45 | ## Example 46 | 47 | ``` 48 | ./bin/pump -socket unix:///tmp/pump.sock \ 49 | -pd-urls http://127.0.0.1:2379 \ 50 | -data-dir ./data.pump 51 | ``` 52 | or use configuration file 53 | 54 | ``` 55 | ./bin/pump -config ./conf/pump.toml 56 | ``` 57 | 58 | ## Deployment 59 | You should deployment pump server for each TiDB node in the cluster, pump can serve tidb-servers across different clusters. 60 | -------------------------------------------------------------------------------- /cmd/pump/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "math/rand" 18 | "os" 19 | "os/signal" 20 | "runtime" 21 | "sync" 22 | "syscall" 23 | "time" 24 | 25 | _ "net/http/pprof" 26 | 27 | "github.com/pingcap/log" 28 | "github.com/pingcap/tidb-binlog/pkg/util" 29 | "github.com/pingcap/tidb-binlog/pkg/version" 30 | "github.com/pingcap/tidb-binlog/pump" 31 | "go.uber.org/zap" 32 | _ "google.golang.org/grpc/encoding/gzip" 33 | ) 34 | 35 | func main() { 36 | runtime.GOMAXPROCS(runtime.NumCPU()) 37 | rand.Seed(time.Now().UTC().UnixNano()) 38 | 39 | cfg := pump.NewConfig() 40 | if err := cfg.Parse(os.Args[1:]); err != nil { 41 | log.Fatal("verifying flags failed. See 'pump --help'.", zap.Error(err)) 42 | } 43 | 44 | if err := util.InitLogger(cfg.LogLevel, cfg.LogFile); err != nil { 45 | log.Fatal("Failed to initialize log", zap.Error(err)) 46 | } 47 | version.PrintVersionInfo("Pump") 48 | log.Info("start pump...", zap.Reflect("config", cfg)) 49 | 50 | p, err := pump.NewServer(cfg) 51 | if err != nil { 52 | log.Fatal("creating pump server failed", zap.Error(err)) 53 | } 54 | 55 | sc := make(chan os.Signal, 1) 56 | signal.Notify(sc, 57 | syscall.SIGHUP, 58 | syscall.SIGINT, 59 | syscall.SIGTERM, 60 | syscall.SIGQUIT) 61 | 62 | var wg sync.WaitGroup 63 | 64 | go func() { 65 | sig := <-sc 66 | log.Info("got signal to exit.", zap.Stringer("signal", sig)) 67 | wg.Add(1) 68 | p.Close() 69 | log.Info("pump is closed") 70 | wg.Done() 71 | }() 72 | 73 | // Start will block until the server is closed 74 | if err := p.Start(); err != nil { 75 | log.Error("start pump server failed", zap.Error(err)) 76 | // exit when start fail 77 | os.Exit(2) 78 | } 79 | 80 | wg.Wait() 81 | log.Info("pump exit") 82 | } 83 | -------------------------------------------------------------------------------- /cmd/pump/pump.toml: -------------------------------------------------------------------------------- 1 | # pump Configuration. 2 | 3 | # addr(i.e. 'host:port') to listen on for client traffic 4 | addr = "127.0.0.1:8250" 5 | 6 | # addr(i.e. 'host:port') to advertise to the public 7 | advertise-addr = "" 8 | 9 | # an integer value to control expiry date of the binlog data, indicates for how long (in days) the binlog data would be stored. 10 | # must bigger than 0 11 | gc = 7 12 | 13 | # path to the data directory of pump's data 14 | data-dir = "data.pump" 15 | 16 | # number of seconds between heartbeat ticks (in 2 seconds) 17 | heartbeat-interval = 2 18 | 19 | # a comma separated list of PD endpoints 20 | pd-urls = "http://127.0.0.1:2379" 21 | 22 | #[security] 23 | # Path of file that contains list of trusted SSL CAs for connection with cluster components. 24 | # ssl-ca = "/path/to/ca.pem" 25 | # Path of file that contains X509 certificate in PEM format for connection with cluster components. 26 | # ssl-cert = "/path/to/drainer.pem" 27 | # Path of file that contains X509 key in PEM format for connection with cluster components. 28 | # ssl-key = "/path/to/drainer-key.pem" 29 | # The common name which is allowed to connection with cluster components. 30 | # cert-allowed-cn = ["binlog"] 31 | # 32 | # [storage] 33 | # Set to `true` (default) for best reliability, which prevents data loss when there is a power failure. 34 | # sync-log = true 35 | 36 | # stop write when disk available space less then the configured size 37 | # 42 MB -> 42000000, 42 mib -> 44040192 38 | # default: 10 gib 39 | # stop-write-at-available-space = "10 gib" 40 | 41 | # 42 | # we suggest using the default config of the embedded LSM DB now, do not change it useless you know what you are doing 43 | # [storage.kv] 44 | # block-cache-capacity = 8388608 45 | # block-restart-interval = 16 46 | # block-size = 4096 47 | # compaction-L0-trigger = 8 48 | # compaction-table-size = 67108864 49 | # compaction-total-size = 536870912 50 | # compaction-total-size-multiplier = 8.0 51 | # write-buffer = 67108864 52 | # write-L0-pause-trigger = 24 53 | # write-L0-slowdown-trigger = 17 54 | -------------------------------------------------------------------------------- /cmd/reparo/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "math/rand" 18 | "os" 19 | "os/signal" 20 | "runtime" 21 | "syscall" 22 | "time" 23 | 24 | _ "net/http/pprof" 25 | 26 | "github.com/pingcap/log" 27 | "github.com/pingcap/tidb-binlog/pkg/util" 28 | "github.com/pingcap/tidb-binlog/pkg/version" 29 | reparo "github.com/pingcap/tidb-binlog/reparo" 30 | "go.uber.org/zap" 31 | ) 32 | 33 | func main() { 34 | runtime.GOMAXPROCS(runtime.NumCPU()) 35 | rand.Seed(time.Now().UTC().UnixNano()) 36 | 37 | cfg := reparo.NewConfig() 38 | if err := cfg.Parse(os.Args[1:]); err != nil { 39 | log.Fatal("verifying flags failed. See 'reparo --help'.", zap.Error(err)) 40 | } 41 | 42 | if err := util.InitLogger(cfg.LogLevel, cfg.LogFile); err != nil { 43 | log.Fatal("Failed to initialize log", zap.Error(err)) 44 | } 45 | version.PrintVersionInfo("Reparo") 46 | 47 | sc := make(chan os.Signal, 1) 48 | signal.Notify(sc, 49 | syscall.SIGHUP, 50 | syscall.SIGINT, 51 | syscall.SIGTERM, 52 | syscall.SIGQUIT) 53 | 54 | r, err := reparo.New(cfg) 55 | if err != nil { 56 | log.Fatal("create reparo failed", zap.Error(err)) 57 | } 58 | 59 | go func() { 60 | sig := <-sc 61 | log.Info("got signal to exit.", zap.Stringer("signale", sig)) 62 | r.Close() 63 | os.Exit(0) 64 | }() 65 | 66 | if err := r.Process(); err != nil { 67 | log.Error("reparo processing failed", zap.Error(err)) 68 | } 69 | if err := r.Close(); err != nil { 70 | log.Fatal("close reparo failed", zap.Error(err)) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /cmd/reparo/reparo.toml: -------------------------------------------------------------------------------- 1 | # data-dir contains protobuf files. It's suggested to use fullpath. 2 | data-dir = "./data.drainer" 3 | 4 | # log-file = "" 5 | # log-rotate = "hour" 6 | log-level = "info" 7 | 8 | # start-datetime and stop-datetime enable you to pick a range of binlog to recovery. 9 | # The datetime format is like '2018-02-28 12:12:12'. 10 | # start-datetime = "" 11 | # stop-datetime = "" 12 | 13 | # Start-tso is similar to start-datetime, but in pd-server tso format. e.g. 395181938313123110 14 | # Stop-tso is similar to stop-datetime, but in pd-server tso format. e.g. 395181938313123110 15 | # start-tso = 0 16 | # stop-tso = 0 17 | 18 | # dest-type choose a destination, which value can be "mysql", "print". 19 | # for print, it just prints decoded value. 20 | dest-type = "mysql" 21 | 22 | # number of binlog events in a transaction batch 23 | txn-batch = 20 24 | 25 | # work count to execute binlogs 26 | # if the latency between reparo and downstream(mysql or tidb) are too high, you might want to increase this 27 | # to get higher throughput by higher concurrent write to the downstream 28 | worker-count = 16 29 | 30 | # Enable safe mode to make reparo reentrant, which value can be "true", "false". If the value is "true", reparo will change the "update" command into "delete+replace". 31 | # The default value of safe-mode is false. 32 | # safe-mode = false 33 | 34 | ##replicate-do-db priority over replicate-do-table if have same db name 35 | ##and we support regular expression , start with '~' declare use regular expression. 36 | # 37 | #replicate-do-db = ["~^b.*","s1"] 38 | #[[replicate-do-table]] 39 | #db-name = "test" 40 | #tbl-name = "log" 41 | 42 | #[[replicate-do-table]] 43 | #db-name = "test" 44 | #tbl-name = "~^a.*" 45 | 46 | #replicate-ignore-db = ["~^c.*","s2"] 47 | #[[replicate-ignore-table]] 48 | #db-name = "test" 49 | #tbl-name = "~^a.*" 50 | 51 | [dest-db] 52 | host = "127.0.0.1" 53 | port = 3309 54 | user = "root" 55 | password = "" 56 | -------------------------------------------------------------------------------- /docs/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingcap/tidb-binlog/552cffbb46230d7b4a41995379423ae9d2351a47/docs/architecture.png -------------------------------------------------------------------------------- /drainer/bench_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package drainer 15 | 16 | import ( 17 | "strconv" 18 | "testing" 19 | 20 | pb "github.com/pingcap/tipb/go-binlog" 21 | ) 22 | 23 | const ( 24 | maxSourceSize = 100 25 | ) 26 | 27 | func BenchmarkMergeNormal5Source(b *testing.B) { 28 | merger := CreateMerger(5, b.N, normalStrategy) 29 | b.ResetTimer() 30 | ReadItem(merger.Output(), b.N) 31 | } 32 | 33 | func BenchmarkMergeHeap5Source(b *testing.B) { 34 | merger := CreateMerger(5, b.N, heapStrategy) 35 | b.ResetTimer() 36 | ReadItem(merger.Output(), b.N) 37 | } 38 | 39 | func BenchmarkMergeNormal10Source(b *testing.B) { 40 | merger := CreateMerger(10, b.N, normalStrategy) 41 | b.ResetTimer() 42 | ReadItem(merger.Output(), b.N) 43 | } 44 | 45 | func BenchmarkMergeHeap10Source(b *testing.B) { 46 | merger := CreateMerger(10, b.N, heapStrategy) 47 | b.ResetTimer() 48 | ReadItem(merger.Output(), b.N) 49 | } 50 | 51 | func BenchmarkMergeNormal50Source(b *testing.B) { 52 | merger := CreateMerger(50, b.N, normalStrategy) 53 | b.ResetTimer() 54 | ReadItem(merger.Output(), b.N) 55 | } 56 | 57 | func BenchmarkMergeHeap50Source(b *testing.B) { 58 | merger := CreateMerger(50, b.N, heapStrategy) 59 | b.ResetTimer() 60 | ReadItem(merger.Output(), b.N) 61 | } 62 | 63 | func ReadItem(itemCh chan MergeItem, total int) { 64 | num := 0 65 | for range itemCh { 66 | num++ 67 | if num > total { 68 | break 69 | } 70 | } 71 | } 72 | 73 | func CreateMerger(sourceNum int, binlogNum int, strategy string) *Merger { 74 | sources := make([]MergeSource, sourceNum) 75 | for i := 0; i < sourceNum; i++ { 76 | source := MergeSource{ 77 | ID: strconv.Itoa(i), 78 | Source: make(chan MergeItem, binlogNum/sourceNum+sourceNum), 79 | } 80 | sources[i] = source 81 | } 82 | merger := NewMerger(0, strategy, sources...) 83 | 84 | for id := range sources { 85 | for j := 1; j <= binlogNum/sourceNum+sourceNum; j++ { 86 | binlog := new(pb.Binlog) 87 | binlog.CommitTs = int64(j*maxSourceSize + id) 88 | binlogItem := newBinlogItem(binlog, strconv.Itoa(id)) 89 | sources[id].Source <- binlogItem 90 | } 91 | } 92 | 93 | return merger 94 | } 95 | -------------------------------------------------------------------------------- /drainer/binlog_item.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package drainer 15 | 16 | import ( 17 | "fmt" 18 | 19 | "github.com/pingcap/tidb/parser/model" 20 | pb "github.com/pingcap/tipb/go-binlog" 21 | ) 22 | 23 | type binlogItem struct { 24 | binlog *pb.Binlog 25 | nodeID string 26 | job *model.Job 27 | } 28 | 29 | // GetCommitTs implements Item interface in merger.go 30 | func (b *binlogItem) GetCommitTs() int64 { 31 | return b.binlog.CommitTs 32 | } 33 | 34 | // GetSourceID implements Item interface in merger.go 35 | func (b *binlogItem) GetSourceID() string { 36 | return b.nodeID 37 | } 38 | 39 | // String returns the string of this binlogItem 40 | func (b *binlogItem) String() string { 41 | return fmt.Sprintf("{startTS: %d, commitTS: %d, node: %s}", b.binlog.StartTs, b.binlog.CommitTs, b.nodeID) 42 | } 43 | 44 | func newBinlogItem(b *pb.Binlog, nodeID string) *binlogItem { 45 | itemp := &binlogItem{ 46 | binlog: b, 47 | nodeID: nodeID, 48 | } 49 | 50 | return itemp 51 | } 52 | 53 | func (b *binlogItem) SetJob(job *model.Job) { 54 | b.job = job 55 | } 56 | -------------------------------------------------------------------------------- /drainer/checkpoint/checkpoint.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package checkpoint 15 | 16 | import ( 17 | "github.com/pingcap/errors" 18 | "github.com/pingcap/log" 19 | "go.uber.org/zap" 20 | ) 21 | 22 | var ( 23 | // ErrCheckPointClosed indicates the CheckPoint already closed. 24 | ErrCheckPointClosed = errors.New("CheckPoint already closed") 25 | ) 26 | 27 | // CheckPoint is the binlog sync pos meta. 28 | // When syncer restarts, we should reload meta info to guarantee continuous transmission. 29 | type CheckPoint interface { 30 | // Load loads checkpoint information. 31 | Load() error 32 | 33 | // Save saves checkpoint information. 34 | Save(commitTS int64, secondaryTS int64, consistent bool, version int64) error 35 | 36 | // TS gets checkpoint commit timestamp. 37 | TS() int64 38 | 39 | // SchemaVersion gets checkpoint current schemaversion. 40 | SchemaVersion() int64 41 | 42 | // IsConsistent return the Consistent status saved. 43 | IsConsistent() bool 44 | 45 | // Close closes the CheckPoint and release resources, after closed other methods should not be called again. 46 | Close() error 47 | } 48 | 49 | // NewCheckPoint returns a CheckPoint instance by giving name 50 | func NewCheckPoint(cfg *Config) (CheckPoint, error) { 51 | var ( 52 | cp CheckPoint 53 | err error 54 | ) 55 | switch cfg.CheckpointType { 56 | case "mysql", "tidb": 57 | cp, err = newMysql(cfg) 58 | case "oracle": 59 | cp, err = newOracle(cfg) 60 | case "file": 61 | cp, err = NewFile(cfg.InitialCommitTS, cfg.CheckPointFile) 62 | default: 63 | err = errors.Errorf("unsupported checkpoint type %s", cfg.CheckpointType) 64 | } 65 | if err != nil { 66 | return nil, errors.Annotatef(err, "initialize %s type checkpoint with config %+v", cfg.CheckpointType, cfg) 67 | } 68 | 69 | log.Info("initialize checkpoint", zap.String("type", cfg.CheckpointType), zap.Int64("checkpoint", cp.TS()), zap.Int64("version", cp.SchemaVersion()), zap.Reflect("cfg", cfg)) 70 | 71 | return cp, nil 72 | } 73 | -------------------------------------------------------------------------------- /drainer/checkpoint/file_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package checkpoint 15 | 16 | import ( 17 | "os" 18 | 19 | . "github.com/pingcap/check" 20 | "github.com/pingcap/errors" 21 | ) 22 | 23 | func (t *testCheckPointSuite) TestFile(c *C) { 24 | fileName := "/tmp/test" 25 | notExistFileName := "test_not_exist" 26 | meta, err := NewFile(0, fileName) 27 | c.Assert(err, IsNil) 28 | defer os.RemoveAll(fileName) 29 | 30 | // zero (initial) CommitTs 31 | c.Assert(meta.TS(), Equals, int64(0)) 32 | c.Assert(meta.IsConsistent(), Equals, false) 33 | 34 | testTs := int64(1) 35 | // save ts 36 | err = meta.Save(testTs, 0, false, 0) 37 | c.Assert(err, IsNil) 38 | // check ts 39 | ts := meta.TS() 40 | c.Assert(ts, Equals, testTs) 41 | c.Assert(meta.IsConsistent(), Equals, false) 42 | 43 | // check consistent true case. 44 | err = meta.Save(testTs, 0, true, 0) 45 | c.Assert(err, IsNil) 46 | ts = meta.TS() 47 | c.Assert(ts, Equals, testTs) 48 | c.Assert(meta.IsConsistent(), Equals, true) 49 | 50 | // check load ts 51 | err = meta.Load() 52 | c.Assert(err, IsNil) 53 | ts = meta.TS() 54 | c.Assert(ts, Equals, testTs) 55 | 56 | // check not exist meta file 57 | meta, err = NewFile(0, notExistFileName) 58 | c.Assert(err, IsNil) 59 | err = meta.Load() 60 | c.Assert(err, IsNil) 61 | c.Assert(meta.TS(), Equals, int64(0)) 62 | 63 | // check not exist meta file, but with initialCommitTs 64 | var initialCommitTS int64 = 123 65 | meta, err = NewFile(initialCommitTS, notExistFileName) 66 | c.Assert(err, IsNil) 67 | c.Assert(meta.TS(), Equals, initialCommitTS) 68 | 69 | // close the checkpoint 70 | err = meta.Close() 71 | c.Assert(err, IsNil) 72 | c.Assert(errors.Cause(meta.Load()), Equals, ErrCheckPointClosed) 73 | c.Assert(errors.Cause(meta.Save(0, 0, true, 0)), Equals, ErrCheckPointClosed) 74 | c.Assert(errors.Cause(meta.Close()), Equals, ErrCheckPointClosed) 75 | } 76 | -------------------------------------------------------------------------------- /drainer/checkpoint/util_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package checkpoint 15 | 16 | import ( 17 | "github.com/DATA-DOG/go-sqlmock" 18 | . "github.com/pingcap/check" 19 | "github.com/pingcap/errors" 20 | ) 21 | 22 | var _ = Suite(&testUtil{}) 23 | 24 | type testUtil struct{} 25 | 26 | func (t *testUtil) TestG(c *C) { 27 | tests := []struct { 28 | name string 29 | rows []uint64 30 | id uint64 31 | err bool 32 | checkSpecifiedErr error 33 | }{ 34 | {"no row", nil, 0, true, ErrNoCheckpointItem}, 35 | {"on row", []uint64{1}, 1, false, nil}, 36 | {"multi row", []uint64{1, 2}, 0, true, nil}, 37 | } 38 | 39 | for _, test := range tests { 40 | db, mock, err := sqlmock.New() 41 | c.Assert(err, IsNil) 42 | 43 | rows := sqlmock.NewRows([]string{"clusterID"}) 44 | for _, row := range test.rows { 45 | rows.AddRow(row) 46 | } 47 | 48 | mock.ExpectQuery("select clusterID from .*").WillReturnRows(rows) 49 | 50 | c.Log("test: ", test.name) 51 | id, err := getClusterID(db, "schema", "table") 52 | if test.err { 53 | c.Assert(err, NotNil) 54 | c.Assert(id, Equals, test.id) 55 | if test.checkSpecifiedErr != nil { 56 | c.Assert(errors.Cause(err), Equals, test.checkSpecifiedErr) 57 | } 58 | } else { 59 | c.Assert(err, IsNil) 60 | c.Assert(id, Equals, test.id) 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /drainer/safepoint.go: -------------------------------------------------------------------------------- 1 | package drainer 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/pingcap/log" 9 | "github.com/pingcap/tidb-binlog/drainer/checkpoint" 10 | pd "github.com/tikv/pd/client" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | const ( 15 | drainerServiceSafePointPrefix = "drainer" 16 | defaultDrainerGCSafePointTTL = 5 * 60 17 | ) 18 | 19 | func updateServiceSafePoint(ctx context.Context, pdClient pd.Client, cpt checkpoint.CheckPoint, ttl int64) { 20 | updateInterval := time.Duration(ttl/2) * time.Second 21 | tick := time.NewTicker(updateInterval) 22 | defer tick.Stop() 23 | dumplingServiceSafePointID := fmt.Sprintf("%s_%d", drainerServiceSafePointPrefix, time.Now().UnixNano()) 24 | log.Info("generate drainer gc safePoint id", zap.String("id", dumplingServiceSafePointID)) 25 | 26 | for { 27 | snapshotTS := uint64(cpt.TS()) 28 | log.Debug("update PD safePoint limit with ttl", 29 | zap.Uint64("safePoint", snapshotTS), 30 | zap.Int64("ttl", ttl)) 31 | for retryCnt := 0; retryCnt <= 10; retryCnt++ { 32 | _, err := pdClient.UpdateServiceGCSafePoint(ctx, dumplingServiceSafePointID, ttl, snapshotTS) 33 | if err == nil { 34 | break 35 | } 36 | log.Debug("update PD safePoint failed", zap.Error(err), zap.Int("retryTime", retryCnt)) 37 | select { 38 | case <-ctx.Done(): 39 | return 40 | case <-time.After(time.Second): 41 | } 42 | } 43 | select { 44 | case <-ctx.Done(): 45 | return 46 | case <-tick.C: 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /drainer/status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package drainer 15 | 16 | import ( 17 | "encoding/json" 18 | "net/http" 19 | 20 | "github.com/pingcap/log" 21 | "go.uber.org/zap" 22 | ) 23 | 24 | // HTTPStatus exposes current status of the collector via HTTP 25 | type HTTPStatus struct { 26 | PumpPos map[string]int64 `json:"PumpPos"` 27 | Synced bool `json:"Synced"` 28 | LastTS int64 `json:"LastTS"` 29 | TsMap string `json:"TsMap"` 30 | } 31 | 32 | // Status implements http.ServeHTTP interface 33 | func (s *HTTPStatus) Status(w http.ResponseWriter, r *http.Request) { 34 | if err := json.NewEncoder(w).Encode(s); err != nil { 35 | log.Error("Failed to encode status", zap.Error(err), zap.Any("status", *s)) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /drainer/sync/bench_kafka_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package sync 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/gogo/protobuf/proto" 20 | obinlog "github.com/pingcap/tidb/tidb-binlog/proto/go-binlog" 21 | ti "github.com/pingcap/tipb/go-binlog" 22 | ) 23 | 24 | var bytes = make([]byte, 5*(1<<10)) 25 | var table = &obinlog.Table{ 26 | SchemaName: proto.String("test"), 27 | TableName: proto.String("test"), 28 | ColumnInfo: []*obinlog.ColumnInfo{ 29 | {Name: "id", MysqlType: "int"}, 30 | {Name: "a1", MysqlType: "blob"}, 31 | }, 32 | Mutations: []*obinlog.TableMutation{ 33 | { 34 | Type: obinlog.MutationType_Insert.Enum(), 35 | Row: &obinlog.Row{ 36 | Columns: []*obinlog.Column{ 37 | { 38 | Int64Value: proto.Int64(1), 39 | }, 40 | { 41 | BytesValue: bytes, 42 | }, 43 | }, 44 | }, 45 | }, 46 | }, 47 | } 48 | 49 | // with bytes = 5KB 50 | // BenchmarkBinlogMarshal-4 100000 573941 ns/op 51 | // means only 1742 op/second 52 | func BenchmarkBinlogMarshal(b *testing.B) { 53 | binlog := &obinlog.Binlog{ 54 | DmlData: &obinlog.DMLData{ 55 | Tables: []*obinlog.Table{table}, 56 | }, 57 | } 58 | var s string 59 | for i := 0; i < b.N; i++ { 60 | s = binlog.String() 61 | } 62 | if len(s) == 0 { 63 | b.Fail() 64 | } 65 | } 66 | 67 | // with bytes = 5KB 68 | // BenchmarkKafka-4 1000000 42384 ns/op 69 | // means 23593 op/second 70 | func BenchmarkKafka(b *testing.B) { 71 | cfg := &DBConfig{ 72 | KafkaAddrs: "127.0.0.1:9092", 73 | KafkaVersion: "0.8.2.0", 74 | ClusterID: 99900, 75 | } 76 | 77 | binlog := &obinlog.Binlog{ 78 | DmlData: &obinlog.DMLData{ 79 | Tables: []*obinlog.Table{table}, 80 | }, 81 | } 82 | 83 | item := &Item{Binlog: &ti.Binlog{}} 84 | 85 | syncer, err := NewKafka(cfg, nil) 86 | if err != nil { 87 | b.Fatal(err) 88 | } 89 | 90 | b.ResetTimer() 91 | 92 | // Just drain is, or may be block if the buffer is full 93 | go func() { 94 | for range syncer.Successes() { 95 | } 96 | }() 97 | 98 | for i := 0; i < b.N; i++ { 99 | err = syncer.saveBinlog(binlog, item) 100 | if err != nil { 101 | b.Fatal(err) 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /drainer/sync/kafka_test.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "github.com/pingcap/check" 5 | pb "github.com/pingcap/tipb/go-binlog" 6 | ) 7 | 8 | var _ = check.Suite(&kafkaSuite{}) 9 | 10 | type kafkaSuite struct { 11 | } 12 | 13 | func newMockItem(ts int64) *Item { 14 | return &Item{Binlog: &pb.Binlog{CommitTs: ts}} 15 | } 16 | 17 | func (s kafkaSuite) TestAckWindow(c *check.C) { 18 | win := newAckWindow() 19 | _, ok := win.getReadyItem() 20 | c.Assert(ok, check.IsFalse) 21 | 22 | win.appendTS(1, 101) 23 | win.appendTS(3, 102) 24 | win.appendTS(7, 103) 25 | _, ok = win.getReadyItem() 26 | c.Assert(ok, check.IsFalse) 27 | c.Assert(win.unackedCount, check.Equals, 3) 28 | c.Assert(win.unackedSize, check.Equals, 306) 29 | 30 | win.handleSuccess(newMockItem(3)) 31 | _, ok = win.getReadyItem() 32 | c.Assert(ok, check.IsFalse) 33 | c.Assert(win.unackedCount, check.Equals, 2) 34 | c.Assert(win.unackedSize, check.Equals, 204) 35 | 36 | win.handleSuccess(newMockItem(7)) 37 | _, ok = win.getReadyItem() 38 | c.Assert(ok, check.IsFalse) 39 | c.Assert(win.unackedCount, check.Equals, 1) 40 | c.Assert(win.unackedSize, check.Equals, 101) 41 | 42 | win.handleSuccess(newMockItem(1)) 43 | c.Assert(win.unackedCount, check.Equals, 0) 44 | c.Assert(win.unackedSize, check.Equals, 0) 45 | item, ok := win.getReadyItem() 46 | c.Assert(ok, check.IsTrue) 47 | c.Assert(item.Binlog.GetCommitTs(), check.Equals, int64(1)) 48 | item, ok = win.getReadyItem() 49 | c.Assert(ok, check.IsTrue) 50 | c.Assert(item.Binlog.GetCommitTs(), check.Equals, int64(3)) 51 | item, ok = win.getReadyItem() 52 | c.Assert(ok, check.IsTrue) 53 | c.Assert(item.Binlog.GetCommitTs(), check.Equals, int64(7)) 54 | } 55 | -------------------------------------------------------------------------------- /drainer/translator/sequence_iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package translator 15 | 16 | import ( 17 | "io" 18 | 19 | "github.com/pingcap/errors" 20 | "github.com/pingcap/tipb/go-binlog" 21 | ) 22 | 23 | // sequenceIterator is a helper to iterate row event by sequence 24 | type sequenceIterator struct { 25 | mutation *binlog.TableMutation 26 | idx int 27 | insertIdx int 28 | deleteIdx int 29 | updateIdx int 30 | } 31 | 32 | func newSequenceIterator(mutation *binlog.TableMutation) *sequenceIterator { 33 | return &sequenceIterator{mutation: mutation} 34 | } 35 | 36 | func (si *sequenceIterator) next() (tp binlog.MutationType, row []byte, err error) { 37 | if si.idx >= len(si.mutation.Sequence) { 38 | err = io.EOF 39 | return 40 | } 41 | 42 | tp = si.mutation.Sequence[si.idx] 43 | si.idx++ 44 | 45 | switch tp { 46 | case binlog.MutationType_Insert: 47 | row = si.mutation.InsertedRows[si.insertIdx] 48 | si.insertIdx++ 49 | case binlog.MutationType_Update: 50 | row = si.mutation.UpdatedRows[si.updateIdx] 51 | si.updateIdx++ 52 | case binlog.MutationType_DeleteRow: 53 | row = si.mutation.DeletedRows[si.deleteIdx] 54 | si.deleteIdx++ 55 | default: 56 | err = errors.Errorf("unknown mutation type: %v", tp) 57 | return 58 | } 59 | 60 | return 61 | } 62 | -------------------------------------------------------------------------------- /drainer/translator/sequence_iterator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package translator 15 | 16 | import ( 17 | "io" 18 | 19 | . "github.com/pingcap/check" 20 | ti "github.com/pingcap/tipb/go-binlog" 21 | ) 22 | 23 | type testSequenceIteratorSuite struct{} 24 | 25 | var _ = Suite(&testSequenceIteratorSuite{}) 26 | 27 | func (t *testSequenceIteratorSuite) TestIterator(c *C) { 28 | mut := new(ti.TableMutation) 29 | var tps []ti.MutationType 30 | var rows [][]byte 31 | 32 | // generate test data 33 | for i := 0; i < 10; i++ { 34 | row := []byte{byte(i)} 35 | rows = append(rows, row) 36 | switch i % 3 { 37 | case 0: 38 | mut.Sequence = append(mut.Sequence, ti.MutationType_Insert) 39 | mut.InsertedRows = append(mut.InsertedRows, row) 40 | tps = append(tps, ti.MutationType_Insert) 41 | case 1: 42 | mut.Sequence = append(mut.Sequence, ti.MutationType_Update) 43 | mut.UpdatedRows = append(mut.UpdatedRows, row) 44 | tps = append(tps, ti.MutationType_Update) 45 | case 2: 46 | mut.Sequence = append(mut.Sequence, ti.MutationType_DeleteRow) 47 | mut.DeletedRows = append(mut.DeletedRows, row) 48 | tps = append(tps, ti.MutationType_DeleteRow) 49 | } 50 | } 51 | 52 | // get back by iterator 53 | iter := newSequenceIterator(mut) 54 | var getTps []ti.MutationType 55 | var getRows [][]byte 56 | 57 | for { 58 | tp, row, err := iter.next() 59 | if err == io.EOF { 60 | break 61 | } 62 | 63 | c.Assert(err, IsNil) 64 | 65 | getTps = append(getTps, tp) 66 | getRows = append(getRows, row) 67 | } 68 | 69 | c.Assert(getTps, DeepEquals, tps) 70 | c.Assert(getRows, DeepEquals, rows) 71 | } 72 | -------------------------------------------------------------------------------- /drainer/translator/table_info.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package translator 15 | 16 | import "github.com/pingcap/tidb/parser/model" 17 | 18 | // TableInfoGetter is used to get table info by table id of TiDB 19 | type TableInfoGetter interface { 20 | TableByID(id int64) (info *model.TableInfo, ok bool) 21 | SchemaAndTableName(id int64) (string, string, bool) 22 | CanAppendDefaultValue(id int64, schemaVersion int64) bool 23 | // IsDroppingColumn(id int64) bool 24 | TableBySchemaVersion(id int64, schemaVersion int64) (info *model.TableInfo, ok bool) 25 | } 26 | -------------------------------------------------------------------------------- /drainer/translator/translator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package translator 15 | 16 | import ( 17 | "testing" 18 | 19 | . "github.com/pingcap/check" 20 | ) 21 | 22 | func TestClient(t *testing.T) { 23 | TestingT(t) 24 | } 25 | 26 | var _ = Suite(&testTranslatorSuite{}) 27 | 28 | type testTranslatorSuite struct{} 29 | -------------------------------------------------------------------------------- /generate-binlog.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd proto/ 3 | 4 | echo "generate binlog code..." 5 | GOGO_ROOT=${GOPATH}/src/github.com/gogo/protobuf 6 | protoc -I.:${GOGO_ROOT}:${GOGO_ROOT}/protobuf --gofast_out=./binlog binlog.proto 7 | cd ./binlog 8 | sed -i.bak -E 's/import _ \"gogoproto\"//g' *.pb.go 9 | sed -i.bak -E 's/import fmt \"fmt\"//g' *.pb.go 10 | rm -f *.bak 11 | goimports -w *.pb.go 12 | -------------------------------------------------------------------------------- /gitcookie.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | touch ~/.gitcookies 3 | chmod 0600 ~/.gitcookies 4 | 5 | git config --global http.cookiefile ~/.gitcookies 6 | 7 | tr , \\t <<\__END__ >>~/.gitcookies 8 | go.googlesource.com,FALSE,/,TRUE,2147483647,o,git-shenli.pingcap.com=1/rGvVlvFq_x9rxOmXqQe_rfcrjbOk6NSOHIQKhhsfidM 9 | go-review.googlesource.com,FALSE,/,TRUE,2147483647,o,git-shenli.pingcap.com=1/rGvVlvFq_x9rxOmXqQe_rfcrjbOk6NSOHIQKhhsfidM 10 | __END__ -------------------------------------------------------------------------------- /hack/clean_vendor.sh: -------------------------------------------------------------------------------- 1 | # delete internal vendor folders 2 | find vendor -type d -name "_vendor" | xargs -I {} rm -r {} 3 | # delete all files that are not go, c, h, or legal 4 | find vendor -type f -not -name "*.go" -not -name "NOTICE*" -not -name "COPYING*" -not -name "LICENSE*" -not -name "*.s" -not -name "PATENTS*" -not -name "*.h" -not -name "*.c" | xargs -I {} rm {} 5 | # delete all generated files 6 | find vendor -type f -name "*_generated.go" | xargs -I {} rm {} 7 | # delete all test files 8 | find vendor -type f -name "*_test.go" | xargs -I {} rm {} 9 | find vendor -type d -name "fixtures" | xargs -I {} rm -r {} 10 | # Delete documentation files. Keep doc.go. 11 | find vendor -type d -name "Documentation" | xargs -I {} rm -r {} 12 | find vendor -type d -name "tutorial" | xargs -I {} rm -r {} 13 | find vendor -name "*.md" | xargs -I {} rm {} 14 | # Delete unused languages 15 | find vendor -type d -name "ruby" | xargs -I {} rm -r {} 16 | -------------------------------------------------------------------------------- /pkg/binlogfile/decoder_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package binlogfile 15 | 16 | import ( 17 | "bytes" 18 | "io" 19 | "testing" 20 | 21 | "github.com/pingcap/check" 22 | ) 23 | 24 | func Test(t *testing.T) { check.TestingT(t) } 25 | 26 | type decoderSuite struct{} 27 | 28 | var _ = check.Suite(&decoderSuite{}) 29 | 30 | func (s *decoderSuite) TestDecode(c *check.C) { 31 | buf := new(bytes.Buffer) 32 | 33 | // write one record 34 | _, err := buf.Write(Encode([]byte("payload"))) 35 | c.Assert(err, check.IsNil) 36 | 37 | decoder := NewDecoder(buf, 0) 38 | 39 | // read the record back and check 40 | payload, _, err := decoder.Decode() 41 | c.Assert(err, check.IsNil) 42 | c.Assert(payload, check.BytesEquals, []byte("payload")) 43 | 44 | // only one byte will reach io.ErrUnexpectedEOF error 45 | err = buf.WriteByte(1) 46 | c.Assert(err, check.IsNil) 47 | _, _, err = decoder.Decode() 48 | c.Assert(err, check.Equals, io.ErrUnexpectedEOF) 49 | } 50 | -------------------------------------------------------------------------------- /pkg/binlogfile/encoder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package binlogfile 15 | 16 | import ( 17 | "encoding/binary" 18 | "hash/crc32" 19 | "io" 20 | 21 | "github.com/pingcap/errors" 22 | ) 23 | 24 | var magic uint32 = 471532804 25 | 26 | // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 27 | // | magic word (4 byte)| Size (8 byte, len(payload)) | payload | crc | 28 | // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 29 | 30 | // Encoder is an interface wraps basic Encode method which encodes payload and write it, and returns offset. 31 | type Encoder interface { 32 | Encode(payload []byte) (int64, error) 33 | } 34 | 35 | type encoder struct { 36 | bw io.Writer 37 | offset int64 38 | } 39 | 40 | // NewEncoder creates a Encoder instance 41 | func NewEncoder(w io.Writer, initOffset int64) Encoder { 42 | return &encoder{ 43 | bw: w, 44 | offset: initOffset, 45 | } 46 | } 47 | 48 | // Encode implements interface of Encoder 49 | func (e *encoder) Encode(payload []byte) (int64, error) { 50 | data := Encode(payload) 51 | _, err := e.bw.Write(data) 52 | if err != nil { 53 | return 0, errors.Trace(err) 54 | } 55 | 56 | e.offset += int64(len(data)) 57 | 58 | return e.offset, nil 59 | } 60 | 61 | // Encode encodes the payload 62 | func Encode(payload []byte) []byte { 63 | crc := crc32.Checksum(payload, crcTable) 64 | 65 | // length count payload 66 | length := len(payload) 67 | 68 | // size is length of magic + size + crc + payload 69 | size := length + 16 70 | data := make([]byte, size) 71 | 72 | binary.LittleEndian.PutUint32(data[:4], magic) 73 | binary.LittleEndian.PutUint64(data[4:12], uint64(length)) 74 | copy(data[12:size-4], payload) 75 | binary.LittleEndian.PutUint32(data[size-4:], crc) 76 | return data 77 | } 78 | -------------------------------------------------------------------------------- /pkg/binlogfile/metrics.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package binlogfile 15 | 16 | import ( 17 | "github.com/prometheus/client_golang/prometheus" 18 | ) 19 | 20 | var ( 21 | writeBinlogSizeHistogram = prometheus.NewHistogramVec( 22 | prometheus.HistogramOpts{ 23 | Namespace: "binlog", 24 | Subsystem: "binlogfile", 25 | Name: "write_binlog_size", 26 | Help: "write binlog size", 27 | Buckets: prometheus.ExponentialBuckets(16, 2, 20), 28 | }, []string{"label"}) 29 | 30 | writeBinlogHistogram = prometheus.NewHistogramVec( 31 | prometheus.HistogramOpts{ 32 | Namespace: "binlog", 33 | Subsystem: "binlogfile", 34 | Name: "write_binlog_duration_time", 35 | Help: "Bucketed histogram of write time (s) of a binlog.", 36 | Buckets: prometheus.ExponentialBuckets(0.00005, 2, 18), 37 | }, []string{"label"}) 38 | 39 | readBinlogHistogram = prometheus.NewHistogramVec( 40 | prometheus.HistogramOpts{ 41 | Namespace: "binlog", 42 | Subsystem: "binlogfile", 43 | Name: "read_binlog_duration_time", 44 | Help: "Bucketed histogram of read time (s) of a binlog.", 45 | Buckets: prometheus.ExponentialBuckets(0.00005, 2, 18), 46 | }, []string{"label"}) 47 | 48 | corruptionBinlogCounter = prometheus.NewCounter( 49 | prometheus.CounterOpts{ 50 | Namespace: "binlog", 51 | Subsystem: "binlogfile", 52 | Name: "corruption_binlog_count", 53 | Help: "corruption binlog count", 54 | }) 55 | ) 56 | 57 | // InitMetircs register the metrics to registry 58 | func InitMetircs(registry *prometheus.Registry) { 59 | registry.MustRegister(writeBinlogSizeHistogram) 60 | registry.MustRegister(readBinlogHistogram) 61 | registry.MustRegister(writeBinlogHistogram) 62 | registry.MustRegister(corruptionBinlogCounter) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/dml/dml.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package dml 15 | 16 | import "strings" 17 | 18 | // GenColumnPlaceholders generates placeholders in question mark format,like "?,?,?". 19 | func GenColumnPlaceholders(length int) string { 20 | var b strings.Builder 21 | b.Grow(2 * length) 22 | for i := 0; i < length; i++ { 23 | if i > 0 { 24 | b.WriteByte(',') 25 | } 26 | b.WriteByte('?') 27 | } 28 | return b.String() 29 | } 30 | -------------------------------------------------------------------------------- /pkg/dml/dml_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package dml 15 | 16 | import ( 17 | "testing" 18 | 19 | . "github.com/pingcap/check" 20 | ) 21 | 22 | func TestClient(t *testing.T) { 23 | TestingT(t) 24 | } 25 | 26 | var _ = Suite(&testDMLSuite{}) 27 | 28 | type testDMLSuite struct{} 29 | 30 | func (s *testDMLSuite) TestGenColumnPlaceholders(c *C) { 31 | got := GenColumnPlaceholders(3) 32 | c.Assert(got, Equals, "?,?,?") 33 | } 34 | -------------------------------------------------------------------------------- /pkg/encrypt/encrypt_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package encrypt 15 | 16 | import ( 17 | "crypto/aes" 18 | "crypto/rand" 19 | "testing" 20 | 21 | . "github.com/pingcap/check" 22 | ) 23 | 24 | var _ = Suite(&testEncryptSuite{}) 25 | 26 | func TestSuite(t *testing.T) { 27 | TestingT(t) 28 | } 29 | 30 | type testEncryptSuite struct { 31 | } 32 | 33 | func (t *testEncryptSuite) TestSetSecretKey(c *C) { 34 | // 16 bit 35 | b16 := make([]byte, 16) 36 | _, err := rand.Read(b16) 37 | c.Assert(err, IsNil) 38 | 39 | err = SetSecretKey(b16) 40 | c.Assert(err, IsNil) 41 | 42 | // 20 bit 43 | b20 := make([]byte, 20) 44 | _, err = rand.Read(b20) 45 | c.Assert(err, IsNil) 46 | 47 | err = SetSecretKey(b20) 48 | c.Assert(err, NotNil) 49 | } 50 | 51 | func removeChar(input []byte, c byte) []byte { 52 | i := 0 53 | for _, x := range input { 54 | if x != c { 55 | input[i] = x 56 | i++ 57 | } 58 | } 59 | return input[:i] 60 | } 61 | 62 | func (t *testEncryptSuite) TestEncrypt(c *C) { 63 | plaintext := []byte("a plain text") 64 | 65 | // encrypt 66 | ciphertext, err := encrypt(plaintext) 67 | c.Assert(err, IsNil) 68 | 69 | // decrypt 70 | plaintext2, err := decrypt(ciphertext) 71 | c.Assert(err, IsNil) 72 | c.Assert(plaintext2, DeepEquals, plaintext) 73 | 74 | // invalid length 75 | _, err = decrypt(ciphertext[:len(ciphertext)-len(plaintext)-1]) 76 | c.Assert(err, NotNil) 77 | 78 | // invalid content 79 | _, err = decrypt(removeChar(ciphertext, ivSep[0])) 80 | c.Assert(err, NotNil) 81 | 82 | // a special case, we construct a ciphertext that can be decrypted but the 83 | // plaintext is not what we want. This is because currently encrypt mechanism 84 | // doesn't keep enough information to decide whether the new ciphertext is valid 85 | block, err := aes.NewCipher(secretKey) 86 | c.Assert(err, IsNil) 87 | blockSize := block.BlockSize() 88 | c.Assert(len(ciphertext), Greater, blockSize+2) 89 | plaintext3, err := decrypt(append(ciphertext[1:blockSize+1], append([]byte{ivSep[0]}, ciphertext[blockSize+2:]...)...)) 90 | c.Assert(err, IsNil) 91 | c.Assert(plaintext3, Not(DeepEquals), plaintext) 92 | } 93 | -------------------------------------------------------------------------------- /pkg/file/lock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package file 15 | 16 | import ( 17 | "errors" 18 | "os" 19 | "syscall" 20 | ) 21 | 22 | var ( 23 | // ErrLocked means that fail to get file lock 24 | ErrLocked = errors.New("pkg/file: file already locked") 25 | ) 26 | 27 | const ( 28 | // PrivateFileMode is the permission for service file 29 | PrivateFileMode = 0600 30 | // PrivateDirMode is the permission for service dir 31 | PrivateDirMode = 0700 32 | ) 33 | 34 | // LockedFile wraps the file into a LockedFile concept simply 35 | type LockedFile struct{ *os.File } 36 | 37 | // TryLockFile tries to open the file with the file lock, it's unblock 38 | func TryLockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { 39 | f, err := os.OpenFile(path, flag, perm) 40 | if err != nil { 41 | return nil, err 42 | } 43 | if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { 44 | f.Close() 45 | if err == syscall.EWOULDBLOCK { 46 | err = ErrLocked 47 | } 48 | return nil, err 49 | } 50 | return &LockedFile{f}, nil 51 | } 52 | 53 | // LockFile opens file with the file lock, it's blocked 54 | func LockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { 55 | f, err := os.OpenFile(path, flag, perm) 56 | if err != nil { 57 | return nil, err 58 | } 59 | if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { 60 | f.Close() 61 | return nil, err 62 | } 63 | return &LockedFile{f}, err 64 | } 65 | -------------------------------------------------------------------------------- /pkg/file/lock_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package file 15 | 16 | import ( 17 | "os" 18 | "testing" 19 | "time" 20 | 21 | . "github.com/pingcap/check" 22 | ) 23 | 24 | var _ = Suite(&testLockSuite{}) 25 | 26 | func Test(t *testing.T) { TestingT(t) } 27 | 28 | type testLockSuite struct{} 29 | 30 | func (t *testLockSuite) TestLockAndUnlock(c *C) { 31 | // lock the nonexist file that would return error 32 | _, err := LockFile("testNoExistFile", os.O_WRONLY, PrivateFileMode) 33 | c.Assert(err, NotNil) 34 | 35 | // lock the nonexist file that would return error 36 | _, err = TryLockFile("testNoExistFile", os.O_WRONLY, PrivateFileMode) 37 | c.Assert(err, NotNil) 38 | 39 | // create test file 40 | f, err := os.CreateTemp("", "lock") 41 | c.Assert(err, IsNil) 42 | f.Close() 43 | defer func() { 44 | err = os.Remove(f.Name()) 45 | c.Assert(err, IsNil) 46 | }() 47 | 48 | // lock the file 49 | l, err := LockFile(f.Name(), os.O_WRONLY, PrivateFileMode) 50 | c.Assert(err, IsNil) 51 | 52 | // try lock a locked file 53 | if _, err = TryLockFile(f.Name(), os.O_WRONLY, PrivateFileMode); err != ErrLocked { 54 | c.Fatal(err) 55 | } 56 | 57 | // unlock the file 58 | err = l.Close() 59 | c.Assert(err, IsNil) 60 | 61 | // try lock the unlocked file 62 | dupl, err := TryLockFile(f.Name(), os.O_WRONLY, PrivateFileMode) 63 | c.Assert(err, IsNil) 64 | 65 | // blocking on locked file 66 | locked := make(chan struct{}, 1) 67 | go func() { 68 | bl, blerr := LockFile(f.Name(), os.O_WRONLY, PrivateFileMode) 69 | c.Assert(blerr, IsNil) 70 | 71 | locked <- struct{}{} 72 | blerr = bl.Close() 73 | c.Assert(blerr, IsNil) 74 | }() 75 | 76 | select { 77 | case <-locked: 78 | c.Error("unexpected unblocking") 79 | case <-time.After(100 * time.Millisecond): 80 | } 81 | 82 | // unlock 83 | err = dupl.Close() 84 | c.Assert(err, IsNil) 85 | 86 | // the previously blocked routine should be unblocked 87 | select { 88 | case <-locked: 89 | case <-time.After(1 * time.Second): 90 | c.Error("unexpected blocking") 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /pkg/filter/filter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package filter 15 | 16 | import ( 17 | "testing" 18 | 19 | . "github.com/pingcap/check" 20 | ) 21 | 22 | func Test(t *testing.T) { TestingT(t) } 23 | 24 | type testFilterSuite struct { 25 | } 26 | 27 | var _ = Suite(&testFilterSuite{}) 28 | 29 | func (t *testFilterSuite) TestFilter(c *C) { 30 | DoDBs := []string{"fulldb", "~fulldb_re.*"} 31 | DoTables := []TableName{{"db", "table"}, {"db2", "~table"}} 32 | 33 | filter := NewFilter(nil, nil, DoDBs, DoTables) 34 | 35 | c.Assert(filter.SkipSchemaAndTable("Fulldb", "t1"), IsFalse) 36 | c.Assert(filter.SkipSchemaAndTable("fulldb_re_x", ""), IsFalse) 37 | c.Assert(filter.SkipSchemaAndTable("db", "table_skip"), IsTrue) 38 | c.Assert(filter.SkipSchemaAndTable("db2", "table"), IsFalse) 39 | 40 | // with ignore db 41 | filter = NewFilter([]string{"db2"}, nil, DoDBs, DoTables) 42 | c.Assert(filter.SkipSchemaAndTable("Fulldb", "t1"), IsFalse) 43 | c.Assert(filter.SkipSchemaAndTable("fulldb_re_x", ""), IsFalse) 44 | c.Assert(filter.SkipSchemaAndTable("db", "table_skip"), IsTrue) 45 | c.Assert(filter.SkipSchemaAndTable("db2", "table"), IsTrue) 46 | 47 | // with ignore table 48 | ignoreTables := []TableName{{"ignore", "ignore"}} 49 | filter = NewFilter(nil, ignoreTables, nil, nil) 50 | c.Assert(filter.SkipSchemaAndTable("ignore", "ignore"), IsTrue) 51 | c.Assert(filter.SkipSchemaAndTable("not_ignore", "not_ignore"), IsFalse) 52 | 53 | // with empty string 54 | filter = NewFilter(nil, nil, []string{""} /*doDBs*/, nil) 55 | c.Assert(filter.SkipSchemaAndTable("", "any"), IsFalse) 56 | c.Assert(filter.SkipSchemaAndTable("any", ""), IsTrue) 57 | } 58 | -------------------------------------------------------------------------------- /pkg/flags/flag.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package flags 15 | 16 | import ( 17 | "flag" 18 | "net/url" 19 | "os" 20 | "strings" 21 | 22 | "github.com/pingcap/errors" 23 | "github.com/pingcap/log" 24 | "go.uber.org/zap" 25 | ) 26 | 27 | func flagToEnv(prefix, name string) string { 28 | return prefix + "_" + strings.ToUpper(strings.Replace(name, "-", "_", -1)) 29 | } 30 | 31 | // SetFlagsFromEnv parses all registered flags in the given flagset, 32 | // and if they are not already set it attempts to set their values from 33 | // environment variables. Environment variables take the name of the flag but 34 | // are UPPERCASE, have the given prefix and any dashes are replaced by 35 | // underscores - for example: some-flag => PUMP_SOME_FLAG 36 | func SetFlagsFromEnv(prefix string, fs *flag.FlagSet) error { 37 | var err error 38 | alreadySet := make(map[string]bool) 39 | fs.Visit(func(f *flag.Flag) { 40 | alreadySet[flagToEnv(prefix, f.Name)] = true 41 | }) 42 | usedEnvKey := make(map[string]bool) 43 | fs.VisitAll(func(f *flag.Flag) { 44 | err = setFlagFromEnv(fs, prefix, f.Name, usedEnvKey, alreadySet) 45 | if err != nil { 46 | log.Error("setFlagFromEnv failed", zap.Error(err)) 47 | } 48 | }) 49 | 50 | return errors.Trace(err) 51 | } 52 | 53 | type flagSetter interface { 54 | Set(fk string, fv string) error 55 | } 56 | 57 | func setFlagFromEnv(fs flagSetter, prefix, fname string, usedEnvKey, alreadySet map[string]bool) error { 58 | key := flagToEnv(prefix, fname) 59 | if !alreadySet[key] { 60 | val := os.Getenv(key) 61 | if val != "" { 62 | usedEnvKey[key] = true 63 | if serr := fs.Set(fname, val); serr != nil { 64 | return errors.Errorf("invalid environment value %q for %s: %v", val, key, serr) 65 | } 66 | log.Info("recognized and used environment variable", 67 | zap.String("key", key), 68 | zap.String("val", val), 69 | zap.String("flag", fname)) 70 | } 71 | } 72 | return nil 73 | } 74 | 75 | // URLsFromFlag returns a slices from url got from the flag. 76 | func URLsFromFlag(fs *flag.FlagSet, urlsFlagName string) []url.URL { 77 | return fs.Lookup(urlsFlagName).Value.(*URLsValue).URLSlice() 78 | } 79 | 80 | // URLStrsFromFlag returns a string slices from url got from the flag. 81 | func URLStrsFromFlag(fs *flag.FlagSet, urlsFlagName string) []string { 82 | return fs.Lookup(urlsFlagName).Value.(*URLsValue).StringSlice() 83 | } 84 | -------------------------------------------------------------------------------- /pkg/flags/urls_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package flags 15 | 16 | import ( 17 | . "github.com/pingcap/check" 18 | ) 19 | 20 | var _ = Suite(&testUrlsSuite{}) 21 | 22 | type testUrlsSuite struct{} 23 | 24 | func (t *testUrlsSuite) TestParseHostPortAddr(c *C) { 25 | urls := []string{ 26 | "127.0.0.1:2379", 27 | "127.0.0.1:2379,127.0.0.2:2379", 28 | "localhost:2379", 29 | "pump-1:8250,pump-2:8250", 30 | "http://127.0.0.1:2379", 31 | "https://127.0.0.1:2379", 32 | "http://127.0.0.1:2379,http://127.0.0.2:2379", 33 | "https://127.0.0.1:2379,https://127.0.0.2:2379", 34 | "unix:///home/tidb/tidb.sock", 35 | } 36 | 37 | expectUrls := [][]string{ 38 | {"127.0.0.1:2379"}, 39 | {"127.0.0.1:2379", "127.0.0.2:2379"}, 40 | {"localhost:2379"}, 41 | {"pump-1:8250", "pump-2:8250"}, 42 | {"http://127.0.0.1:2379"}, 43 | {"https://127.0.0.1:2379"}, 44 | {"http://127.0.0.1:2379", "http://127.0.0.2:2379"}, 45 | {"https://127.0.0.1:2379", "https://127.0.0.2:2379"}, 46 | {"unix:///home/tidb/tidb.sock"}, 47 | } 48 | 49 | for i, url := range urls { 50 | urlList, err := ParseHostPortAddr(url) 51 | c.Assert(err, Equals, nil) 52 | c.Assert(len(urlList), Equals, len(expectUrls[i])) 53 | for j, u := range urlList { 54 | c.Assert(u, Equals, expectUrls[i][j]) 55 | } 56 | } 57 | 58 | inValidUrls := []string{ 59 | "127.0.0.1", 60 | "http:///127.0.0.1:2379", 61 | "htt://127.0.0.1:2379", 62 | } 63 | 64 | for _, url := range inValidUrls { 65 | _, err := ParseHostPortAddr(url) 66 | c.Assert(err, NotNil) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /pkg/loader/README.md: -------------------------------------------------------------------------------- 1 | loader 2 | ====== 3 | 4 | A package to load data into MySQL in real-time, aimed to be used by *reparo*, *drainer* etc unified. 5 | 6 | 7 | ### Getting started 8 | - Example is available via [example_loader_test.go](./example_loader_test.go) 9 | 10 | You need to write a translator to use *Loader* like *SecondaryBinlogToTxn* in [translate.go](./translate.go) to translate upstream data format (e.g. binlog) into `Txn` objects. 11 | 12 | 13 | ## Overview 14 | Loader splits the upstream transaction DML events and concurrently (shared by primary key or unique key) loads data into MySQL. It respects causality with [causality.go](./causality.go). 15 | 16 | 17 | ## Optimization 18 | #### Large Operation 19 | Instead of executing DML one by one, we can combine many small operations into a single large operation, like using INSERT statements with multiple VALUES lists to insert several rows at a time. This is [faster](https://medium.com/@benmorel/high-speed-inserts-with-mysql-9d3dcd76f723) than inserting one by one. 20 | 21 | #### Merge by Primary Key 22 | You may want to read [log-compaction](https://kafka.apache.org/documentation/#compaction) of Kafka. 23 | 24 | We can treat a table with Primary Key like a KV-store. To reload the table with the change history of the table, we only need the last value of every key. 25 | 26 | While synchronizing data into downstream at real-time, we can get DML events from upstream in batchs and merge by key. After merging, there's only one event for each key, so at downstream, we don't need to do as many events as upstream. This also help we to use batch insert operation. 27 | 28 | We should also consider secondary unique key here, see *execTableBatch* in [executor.go](./executor.go). Currently, we only merge by primary key and do batch operation if the table have primary key and no unique key. 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /pkg/loader/causality_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package loader 15 | 16 | import ( 17 | . "github.com/pingcap/check" 18 | ) 19 | 20 | type causalitySuite struct{} 21 | 22 | var _ = Suite(&causalitySuite{}) 23 | 24 | func (s *causalitySuite) TestCausality(c *C) { 25 | ca := NewCausality() 26 | caseData := []string{"test_1", "test_2", "test_3"} 27 | excepted := map[string]string{ 28 | "test_1": "test_1", 29 | "test_2": "test_1", 30 | "test_3": "test_1", 31 | } 32 | c.Assert(ca.Add(caseData), IsNil) 33 | c.Assert(ca.relations, DeepEquals, excepted) 34 | c.Assert(ca.Add([]string{"test_4"}), IsNil) 35 | excepted["test_4"] = "test_4" 36 | c.Assert(ca.relations, DeepEquals, excepted) 37 | conflictData := []string{"test_4", "test_3"} 38 | c.Assert(ca.DetectConflict(conflictData), IsTrue) 39 | c.Assert(ca.Add(conflictData), NotNil) 40 | ca.Reset() 41 | c.Assert(ca.relations, HasLen, 0) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/loader/example_loader_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package loader 15 | 16 | import "log" 17 | 18 | func Example() { 19 | // create sql.DB 20 | db, err := CreateDB("root", "", "localhost", 4000, nil /* *tls.Config */) 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | 25 | // init loader 26 | loader, err := NewLoader(db, WorkerCount(16), BatchSize(128)) 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | 31 | // get the success txn from loader 32 | go func() { 33 | // the return order will be the order you push into loader.Input() 34 | for txn := range loader.Successes() { 35 | log.Print("succ: ", txn) 36 | } 37 | }() 38 | 39 | // run loader 40 | go func() { 41 | // return non nil if encounter some case fail to load data the downstream 42 | // or nil when loader is closed when all data is loaded to downstream 43 | err := loader.Run() 44 | if err != nil { 45 | log.Fatal(err) 46 | } 47 | }() 48 | 49 | // push ddl txn 50 | loader.Input() <- NewDDLTxn("test", "test", "create table test(id primary key)") 51 | 52 | // push one insert dml txn 53 | values := map[string]interface{}{"id": 1} 54 | loader.Input() <- &Txn{ 55 | DMLs: []*DML{{Database: "test", Table: "test", Tp: InsertDMLType, Values: values}}, 56 | } 57 | 58 | // push one update dml txn 59 | newValues := map[string]interface{}{"id": 2} 60 | loader.Input() <- &Txn{ 61 | DMLs: []*DML{{Database: "test", Table: "test", Tp: UpdateDMLType, Values: newValues, OldValues: values}}, 62 | } 63 | 64 | // you can set safe mode or not at run time 65 | // which use replace for insert event and delete + replace for update make it be idempotent 66 | loader.SetSafeMode(true) 67 | 68 | // push one delete dml txn 69 | loader.Input() <- &Txn{ 70 | DMLs: []*DML{{Database: "test", Table: "test", Tp: DeleteDMLType, Values: newValues}}, 71 | } 72 | //... 73 | 74 | // Close the Loader. No more Txn can be push into Input() 75 | // Run will quit when all data is drained 76 | loader.Close() 77 | } 78 | -------------------------------------------------------------------------------- /pkg/node/node_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package node 15 | 16 | import ( 17 | . "github.com/pingcap/check" 18 | ) 19 | 20 | var _ = Suite(&testNodeSuite{}) 21 | 22 | type testNodeSuite struct{} 23 | 24 | func (s *testNodeSuite) TestClone(c *C) { 25 | status := NewStatus("nodeID", "localhost", Online, 100, 407775642342881, 407775645599649) 26 | status2 := CloneStatus(status) 27 | c.Assert(status, Not(Equals), status2) 28 | c.Assert(*status, Equals, *status2) 29 | } 30 | 31 | func (s *testNodeSuite) TestString(c *C) { 32 | status := NewStatus("nodeID", "localhost", Online, 100, 407775642342881, 407775645599649) 33 | str := status.String() 34 | c.Assert(str, Matches, "{NodeID: nodeID, Addr: localhost, State: online, MaxCommitTS: 407775642342881, UpdateTime: 1970-01-19 .*}") 35 | } 36 | -------------------------------------------------------------------------------- /pkg/types/urls.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package types 15 | 16 | import ( 17 | "net" 18 | "net/url" 19 | "sort" 20 | "strings" 21 | 22 | "github.com/pingcap/errors" 23 | ) 24 | 25 | // URLs defines a slice of URLs as a type 26 | type URLs []url.URL 27 | 28 | // NewURLs return a URLs from a slice of formatted URL strings 29 | func NewURLs(strs []string) (URLs, error) { 30 | all := make([]url.URL, len(strs)) 31 | if len(all) == 0 { 32 | return nil, errors.New("no valid URLs given") 33 | } 34 | for i, in := range strs { 35 | in = strings.TrimSpace(in) 36 | u, err := url.Parse(in) 37 | if err != nil { 38 | return nil, errors.Trace(err) 39 | } 40 | if u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "unix" && u.Scheme != "unixs" { 41 | return nil, errors.Errorf("URL scheme must be http, https, unix, or unixs: %s", in) 42 | } 43 | if _, _, err := net.SplitHostPort(u.Host); err != nil { 44 | return nil, errors.Errorf(`URL address does not have the form "host:port": %s`, in) 45 | } 46 | if u.Path != "" { 47 | return nil, errors.Errorf("URL must not contain a path: %s", in) 48 | } 49 | all[i] = *u 50 | } 51 | us := URLs(all) 52 | sort.Slice(us, func(i, j int) bool { return us[i].String() < us[j].String() }) 53 | 54 | return us, nil 55 | } 56 | 57 | // String return a string of list of URLs witch separated by comma 58 | func (us URLs) String() string { 59 | return strings.Join(us.StringSlice(), ",") 60 | } 61 | 62 | // StringSlice return a slice of formatted string of URL 63 | func (us URLs) StringSlice() []string { 64 | out := make([]string, len(us)) 65 | for i := range us { 66 | out[i] = us[i].String() 67 | } 68 | 69 | return out 70 | } 71 | -------------------------------------------------------------------------------- /pkg/types/urls_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package types 15 | 16 | import ( 17 | "strings" 18 | "testing" 19 | 20 | . "github.com/pingcap/check" 21 | ) 22 | 23 | func Test(t *testing.T) { 24 | TestingT(t) 25 | } 26 | 27 | var _ = Suite(&testTypesSuite{}) 28 | 29 | type testTypesSuite struct{} 30 | 31 | func (s *testTypesSuite) TestURLs(c *C) { 32 | urlstrs := []string{ 33 | "http://www.google.com:12306", 34 | "http://192.168.199.111:1080", 35 | "http://hostname:9000", 36 | } 37 | sorted := []string{ 38 | "http://192.168.199.111:1080", 39 | "http://hostname:9000", 40 | "http://www.google.com:12306", 41 | } 42 | 43 | urls, err := NewURLs(urlstrs) 44 | c.Assert(err, IsNil) 45 | c.Assert(urls.String(), Equals, strings.Join(sorted, ",")) 46 | } 47 | 48 | func (s *testTypesSuite) TestBadURLs(c *C) { 49 | badurls := [][]string{ 50 | {"http://192.168.199.111"}, 51 | {"127.0.0.1:1080"}, 52 | {"http://192.168.199.112:8080/api/v1"}, 53 | } 54 | 55 | for _, badurl := range badurls { 56 | _, err := NewURLs(badurl) 57 | c.Assert(err, NotNil) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/util/duration.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "encoding" 18 | "encoding/json" 19 | "fmt" 20 | "strconv" 21 | "time" 22 | 23 | "github.com/pingcap/errors" 24 | ) 25 | 26 | var empty = "" 27 | 28 | var _ encoding.TextMarshaler = Duration(empty) 29 | var _ encoding.TextUnmarshaler = (*Duration)(&empty) 30 | var _ json.Marshaler = Duration(empty) 31 | var _ json.Unmarshaler = (*Duration)(&empty) 32 | 33 | // Duration is a wrapper of time.Duration for TOML and JSON. 34 | // it can be parsed to both integer and string 35 | // integer 7 will be parsed to 7*24h 36 | // string 10m will be parsed to 10m 37 | type Duration string 38 | 39 | // NewDuration creates a Duration from time.Duration. 40 | func NewDuration(duration time.Duration) Duration { 41 | return Duration(duration.String()) 42 | } 43 | 44 | // MarshalJSON returns the duration as a JSON string. 45 | func (d Duration) MarshalJSON() ([]byte, error) { 46 | return []byte(fmt.Sprintf(`"%s"`, d)), nil 47 | } 48 | 49 | // UnmarshalJSON parses a JSON string into the duration. 50 | func (d *Duration) UnmarshalJSON(text []byte) error { 51 | s, err := strconv.Unquote(string(text)) 52 | if err != nil { 53 | return errors.WithStack(err) 54 | } 55 | td := Duration(s) 56 | _, err = td.ParseDuration() 57 | if err != nil { 58 | return errors.WithStack(err) 59 | } 60 | *d = Duration(s) 61 | return nil 62 | } 63 | 64 | // UnmarshalText parses a TOML string into the duration. 65 | func (d *Duration) UnmarshalText(text []byte) error { 66 | var err error 67 | td := Duration(text) 68 | _, err = td.ParseDuration() 69 | if err != nil { 70 | return errors.WithStack(err) 71 | } 72 | *d = Duration(text) 73 | return nil 74 | } 75 | 76 | // MarshalText returns the duration as a JSON string. 77 | func (d Duration) MarshalText() ([]byte, error) { 78 | return []byte(d), nil 79 | } 80 | 81 | // ParseDuration parses gc durations. The default unit is day. 82 | func (d Duration) ParseDuration() (time.Duration, error) { 83 | gc := string(d) 84 | t, err := strconv.ParseUint(gc, 10, 64) 85 | if err == nil { 86 | return time.Duration(t) * 24 * time.Hour, nil 87 | } 88 | gcDuration, err := time.ParseDuration(gc) 89 | if err != nil { 90 | return 0, errors.Annotatef(err, "unsupported gc time %s, etc: use 7 for 7 day, 7h for 7 hour", gc) 91 | } 92 | return gcDuration, nil 93 | } 94 | -------------------------------------------------------------------------------- /pkg/util/duration_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "time" 18 | 19 | . "github.com/pingcap/check" 20 | ) 21 | 22 | type durationSuite struct{} 23 | 24 | var _ = Suite(&durationSuite{}) 25 | 26 | func (s *durationSuite) TestParseDuration(c *C) { 27 | gc := Duration("7") 28 | expectDuration := 7 * 24 * time.Hour 29 | duration, err := gc.ParseDuration() 30 | c.Assert(err, IsNil) 31 | c.Assert(duration, Equals, expectDuration) 32 | 33 | gc = "30m" 34 | expectDuration = 30 * time.Minute 35 | duration, err = gc.ParseDuration() 36 | c.Assert(err, IsNil) 37 | c.Assert(duration, Equals, expectDuration) 38 | 39 | gc = "7d" 40 | _, err = gc.ParseDuration() 41 | c.Assert(err, NotNil) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/util/http.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "fmt" 18 | 19 | "github.com/pingcap/log" 20 | ) 21 | 22 | const ( 23 | statusOK = 200 24 | // nolint 25 | statusWrongParameter = 1 26 | statusNotFound = 2 27 | statusOtherError = 3 28 | ) 29 | 30 | // Response represents message that returns to client 31 | type Response struct { 32 | Message string `json:"message"` 33 | Code int `json:"code"` 34 | Data interface{} `json:"data"` 35 | } 36 | 37 | // SuccessResponse returns a success response. 38 | func SuccessResponse(message string, data interface{}) *Response { 39 | return &Response{ 40 | Code: statusOK, 41 | Data: data, 42 | Message: message, 43 | } 44 | } 45 | 46 | // NotFoundResponsef returns a not found response. 47 | func NotFoundResponsef(format string, args ...interface{}) *Response { 48 | format = format + " not found" 49 | return &Response{ 50 | Code: statusNotFound, 51 | Message: fmt.Sprintf(format, args...), 52 | } 53 | } 54 | 55 | // ErrResponsef returns a error response. 56 | func ErrResponsef(format string, args ...interface{}) *Response { 57 | errMsg := fmt.Sprintf(format, args...) 58 | log.Warn(errMsg) 59 | return &Response{ 60 | Code: statusOtherError, 61 | Message: errMsg, 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /pkg/util/http_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | . "github.com/pingcap/check" 18 | ) 19 | 20 | type httpSuite struct{} 21 | 22 | var _ = Suite(&httpSuite{}) 23 | 24 | func (s *httpSuite) TestSuccessResponse(c *C) { 25 | resp := SuccessResponse("cool", 42) 26 | c.Assert(resp.Message, Equals, "cool") 27 | c.Assert(resp.Code, Equals, 200) 28 | c.Assert(resp.Data.(int), Equals, 42) 29 | } 30 | 31 | func (s *httpSuite) TestNotFoundResponsef(c *C) { 32 | resp := NotFoundResponsef("%d %s", 1984, "Hero") 33 | c.Assert(resp.Message, Equals, "1984 Hero not found") 34 | c.Assert(resp.Code, Equals, statusNotFound) 35 | } 36 | 37 | func (s *httpSuite) TestErrResponsef(c *C) { 38 | resp := ErrResponsef("doctor %d", 42) 39 | c.Assert(resp.Message, Equals, "doctor 42") 40 | c.Assert(resp.Code, Equals, statusOtherError) 41 | } 42 | -------------------------------------------------------------------------------- /pkg/util/kafka.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "sync" 18 | 19 | "github.com/Shopify/sarama" 20 | "github.com/pingcap/errors" 21 | "github.com/pingcap/log" 22 | metrics "github.com/rcrowley/go-metrics" 23 | "github.com/rcrowley/go-metrics/exp" 24 | "go.uber.org/zap" 25 | ) 26 | 27 | // don't use directly, call GetParentMetricsRegistry to get it 28 | var metricRegistry metrics.Registry 29 | var metricRegistryOnce sync.Once 30 | 31 | func initMetrics() { 32 | metricRegistry = metrics.NewRegistry() 33 | // can't call Exp multi time 34 | exp.Exp(metricRegistry) 35 | } 36 | 37 | // GetParentMetricsRegistry get the metrics registry and expose the metrics while /debug/metrics 38 | func GetParentMetricsRegistry() metrics.Registry { 39 | metricRegistryOnce.Do(initMetrics) 40 | return metricRegistry 41 | } 42 | 43 | // NewSaramaConfig return the default config and set the according version and metrics 44 | func NewSaramaConfig(kafkaVersion string, metricsPrefix string) (*sarama.Config, error) { 45 | config := sarama.NewConfig() 46 | 47 | version, err := sarama.ParseKafkaVersion(kafkaVersion) 48 | if err != nil { 49 | return nil, errors.Trace(err) 50 | } 51 | 52 | config.ClientID = "tidb_binlog" 53 | config.Version = version 54 | log.Debug("kafka consumer", zap.Stringer("version", version)) 55 | config.MetricRegistry = metrics.NewPrefixedChildRegistry(GetParentMetricsRegistry(), metricsPrefix) 56 | 57 | return config, nil 58 | } 59 | -------------------------------------------------------------------------------- /pkg/util/kafka_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | . "github.com/pingcap/check" 18 | ) 19 | 20 | type kafkaSuite struct{} 21 | 22 | var _ = Suite(&kafkaSuite{}) 23 | 24 | func (s *kafkaSuite) TestNewSaramaConfig(c *C) { 25 | cfg, err := NewSaramaConfig("0.8.2.0", "testing") 26 | c.Assert(err, IsNil) 27 | c.Assert(cfg, NotNil) 28 | c.Assert(cfg.Version.String(), Equals, "0.8.2.0") 29 | c.Assert(cfg.ClientID, Equals, "tidb_binlog") 30 | } 31 | -------------------------------------------------------------------------------- /pkg/util/log_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "path" 18 | "time" 19 | 20 | . "github.com/pingcap/check" 21 | "github.com/pingcap/log" 22 | "go.uber.org/zap/zapcore" 23 | ) 24 | 25 | type logSuite struct{} 26 | 27 | var _ = Suite(&logSuite{}) 28 | 29 | func (s *logSuite) TestLog(c *C) { 30 | logger := NewLog() 31 | logger.Add("drainer", time.Millisecond) 32 | t0 := time.Now() 33 | 34 | var counter int 35 | callback := func() { 36 | counter++ 37 | } 38 | 39 | for i := 0; i < 1000; i++ { 40 | logger.Print("drainer", callback) 41 | } 42 | 43 | nInterval := time.Since(t0) / time.Millisecond 44 | 45 | c.Assert(counter, Equals, int(nInterval)+1) 46 | } 47 | 48 | func (s *logSuite) TestInitLogger(c *C) { 49 | f := path.Join(c.MkDir(), "test") 50 | err := InitLogger("error", f) 51 | c.Assert(err, IsNil) 52 | c.Assert(log.GetLevel(), Equals, zapcore.ErrorLevel) 53 | } 54 | -------------------------------------------------------------------------------- /pkg/util/net.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "crypto/tls" 18 | "net" 19 | "net/url" 20 | 21 | "github.com/pingcap/errors" 22 | ) 23 | 24 | // Listen return the listener from tls.Listen if tlsConfig is NOT Nil. 25 | func Listen(network, addr string, tlsConfig *tls.Config) (listener net.Listener, err error) { 26 | URL, err := url.Parse(addr) 27 | if err != nil { 28 | return nil, errors.Annotatef(err, "invalid listening socket addr (%s)", addr) 29 | } 30 | 31 | if tlsConfig != nil { 32 | listener, err = tls.Listen(network, URL.Host, tlsConfig) 33 | if err != nil { 34 | return nil, errors.Annotatef(err, "fail to start %s on %s", network, URL.Host) 35 | } 36 | } else { 37 | listener, err = net.Listen(network, URL.Host) 38 | if err != nil { 39 | return nil, errors.Annotatef(err, "fail to start %s on %s", network, URL.Host) 40 | } 41 | } 42 | 43 | return listener, nil 44 | } 45 | -------------------------------------------------------------------------------- /pkg/util/net_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "github.com/pingcap/check" 18 | ) 19 | 20 | type netSuite struct{} 21 | 22 | var _ = check.Suite(&netSuite{}) 23 | 24 | func (n *netSuite) TestListen(c *check.C) { 25 | // wrong addr 26 | _, err := Listen("unix", "://asdf:1231:123:12", nil) 27 | c.Assert(err, check.ErrorMatches, ".*invalid .* socket addr.*") 28 | 29 | // unbindable addr 30 | _, err = Listen("tcp", "http://asdf;klj:7979/12", nil) 31 | c.Assert(err, check.ErrorMatches, ".*fail to start.*") 32 | 33 | // return listener 34 | l, err := Listen("tcp", "http://localhost:17979", nil) 35 | c.Assert(err, check.IsNil) 36 | c.Assert(l, check.NotNil) 37 | } 38 | -------------------------------------------------------------------------------- /pkg/util/p8s.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "context" 18 | "time" 19 | 20 | "github.com/pingcap/log" 21 | "github.com/prometheus/client_golang/prometheus" 22 | "github.com/prometheus/client_golang/prometheus/push" 23 | "go.uber.org/zap" 24 | ) 25 | 26 | var ( 27 | addToPusher = addFromGatherer 28 | ) 29 | 30 | // NewMetricClient returns a pointer to a MetricClient 31 | func NewMetricClient(addr string, interval time.Duration, registry *prometheus.Registry) *MetricClient { 32 | return &MetricClient{addr: addr, interval: interval, registry: registry} 33 | } 34 | 35 | // MetricClient manage the periodic push to the Prometheus Pushgateway. 36 | type MetricClient struct { 37 | addr string 38 | interval time.Duration 39 | registry *prometheus.Registry 40 | } 41 | 42 | // Start run a loop of pushing metrics to Prometheus Pushgateway. 43 | func (mc MetricClient) Start(ctx context.Context, grouping map[string]string) { 44 | log.Debug("Start prometheus metrics client", 45 | zap.String("addr", mc.addr), 46 | zap.Float64("interval second", mc.interval.Seconds()), 47 | ) 48 | for { 49 | select { 50 | case <-ctx.Done(): 51 | return 52 | case <-time.After(mc.interval): 53 | if err := addToPusher("binlog", grouping, mc.addr, mc.registry); err != nil { 54 | log.Error("push metrics to Prometheus Pushgateway failed", zap.Error(err)) 55 | } 56 | } 57 | } 58 | } 59 | 60 | func addFromGatherer(job string, grouping map[string]string, url string, g prometheus.Gatherer) error { 61 | pusher := push.New(url, job) 62 | // add grouping 63 | for k, v := range grouping { 64 | pusher = pusher.Grouping(k, v) 65 | } 66 | pusher = pusher.Gatherer(g) 67 | return pusher.Add() 68 | } 69 | -------------------------------------------------------------------------------- /pkg/util/p8s_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "context" 18 | "time" 19 | 20 | . "github.com/pingcap/check" 21 | "github.com/prometheus/client_golang/prometheus" 22 | ) 23 | 24 | type p8sSuite struct{} 25 | 26 | var _ = Suite(&p8sSuite{}) 27 | 28 | func (s *p8sSuite) TestCanBeStopped(c *C) { 29 | mc := NewMetricClient("localhost:9999", time.Millisecond, prometheus.NewRegistry()) 30 | signal := make(chan struct{}) 31 | ctx, cancel := context.WithCancel(context.Background()) 32 | go func() { 33 | mc.Start(ctx, map[string]string{"instance": "drainer-1"}) 34 | close(signal) 35 | }() 36 | cancel() 37 | select { 38 | case <-signal: 39 | case <-time.After(time.Second): 40 | c.Fatal("Doesn't stop in time") 41 | } 42 | } 43 | 44 | func (s *p8sSuite) TestAddToPusher(c *C) { 45 | mc := NewMetricClient("localhost:9999", 10*time.Millisecond, prometheus.NewRegistry()) 46 | 47 | // Set up the mock function 48 | var nCalled int 49 | orig := addToPusher 50 | addToPusher = func(job string, grouping map[string]string, url string, g prometheus.Gatherer) error { 51 | c.Assert(job, Equals, "binlog") 52 | c.Assert(grouping, DeepEquals, map[string]string{"instance": "pump-1"}) 53 | c.Assert(url, Equals, mc.addr) 54 | c.Assert(g, DeepEquals, mc.registry) 55 | nCalled++ 56 | return nil 57 | } 58 | defer func() { 59 | addToPusher = orig 60 | }() 61 | 62 | ctx, cancel := context.WithTimeout(context.Background(), 5*mc.interval) 63 | defer cancel() 64 | go mc.Start(ctx, map[string]string{"instance": "pump-1"}) 65 | <-ctx.Done() 66 | c.Assert(nCalled, GreaterEqual, 4) 67 | c.Assert(nCalled, LessEqual, 6) 68 | } 69 | -------------------------------------------------------------------------------- /pkg/util/signal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "os" 18 | "os/signal" 19 | "runtime" 20 | "syscall" 21 | 22 | stdlog "log" 23 | 24 | "github.com/pingcap/log" 25 | "go.uber.org/zap" 26 | ) 27 | 28 | // SetupSignalHandler setup signal handler 29 | func SetupSignalHandler(shudownFunc func(sig os.Signal)) { 30 | usrDefSignalChan := make(chan os.Signal, 1) 31 | 32 | signal.Notify(usrDefSignalChan, syscall.SIGUSR1) 33 | go func() { 34 | buf := make([]byte, 1<<16) 35 | for { 36 | sig := <-usrDefSignalChan 37 | if sig == syscall.SIGUSR1 { 38 | stackLen := runtime.Stack(buf, true) 39 | stdlog.Printf("\n=== Got signal [%s] to dump goroutine stack. ===\n%s\n=== Finished dumping goroutine stack. ===\n", sig, buf[:stackLen]) 40 | } 41 | } 42 | }() 43 | 44 | closeSignalChan := make(chan os.Signal, 1) 45 | signal.Notify(closeSignalChan, 46 | syscall.SIGHUP, 47 | syscall.SIGINT, 48 | syscall.SIGTERM, 49 | syscall.SIGQUIT) 50 | 51 | go func() { 52 | sig := <-closeSignalChan 53 | log.Info("Got signal to exit.", zap.Stringer("signal", sig)) 54 | shudownFunc(sig) 55 | }() 56 | } 57 | -------------------------------------------------------------------------------- /pkg/util/signal_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "bytes" 18 | "log" 19 | "os" 20 | "strings" 21 | "syscall" 22 | "time" 23 | 24 | . "github.com/pingcap/check" 25 | ) 26 | 27 | type signalSuite struct{} 28 | 29 | var _ = Suite(&signalSuite{}) 30 | 31 | func (s *signalSuite) TestShouldCallFunc(c *C) { 32 | received := make(chan os.Signal, 2) 33 | 34 | SetupSignalHandler(func(s os.Signal) { 35 | received <- s 36 | }) 37 | 38 | pid := syscall.Getpid() 39 | err := syscall.Kill(pid, syscall.SIGINT) 40 | c.Assert(err, IsNil) 41 | 42 | select { 43 | case s := <-received: 44 | c.Assert(s, Equals, syscall.SIGINT, Commentf("Received wrong signal")) 45 | case <-time.After(2 * time.Second): 46 | c.Fatal("Timeout waiting for the signal") 47 | } 48 | } 49 | 50 | func (s *signalSuite) TestShouldDumpStack(c *C) { 51 | var buf bytes.Buffer 52 | log.SetOutput(&buf) 53 | log.SetFlags(0) 54 | defer func() { 55 | log.SetOutput(os.Stderr) 56 | }() 57 | 58 | SetupSignalHandler(func(s os.Signal) {}) 59 | 60 | pid := syscall.Getpid() 61 | err := syscall.Kill(pid, syscall.SIGUSR1) 62 | c.Assert(err, IsNil) 63 | 64 | time.Sleep(time.Second) 65 | 66 | record := buf.String() 67 | parts := strings.Split(strings.TrimSpace(record), "\n") 68 | c.Log(record) 69 | c.Assert(len(parts), Greater, 1) 70 | c.Assert(parts[0], Matches, ".*Got signal.*to dump.*") 71 | } 72 | -------------------------------------------------------------------------------- /pkg/util/ts.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "time" 18 | 19 | "github.com/pingcap/errors" 20 | "github.com/pingcap/log" 21 | "github.com/pingcap/tipb/go-binlog" 22 | "github.com/tikv/client-go/v2/oracle" 23 | pd "github.com/tikv/pd/client" 24 | "go.uber.org/zap" 25 | "golang.org/x/net/context" 26 | ) 27 | 28 | var ( 29 | slowDist = 30 * time.Millisecond 30 | physicalShiftBits uint = 18 31 | ) 32 | 33 | // GetApproachTS get a approach ts by ts and time 34 | func GetApproachTS(ts int64, tm time.Time) int64 { 35 | if ts == 0 { 36 | return 0 37 | } 38 | second := int64(time.Since(tm).Seconds()) 39 | return ts + (second*1000)< slowDist { 51 | log.Warn("get timestamp too slow", zap.Duration("take", dist)) 52 | } 53 | 54 | ts := int64(oracle.ComposeTS(physical, logical)) 55 | 56 | return ts, nil 57 | } 58 | 59 | // GenFakeBinlog generates a fake binlog from given tso 60 | func GenFakeBinlog(ts int64) *binlog.Binlog { 61 | return &binlog.Binlog{ 62 | StartTs: ts, 63 | Tp: binlog.BinlogType_Rollback, 64 | CommitTs: ts, 65 | } 66 | } 67 | 68 | // TSOToRoughTime translates tso to rough time that used to display 69 | func TSOToRoughTime(ts int64) time.Time { 70 | t := time.Unix(ts>>18/1000, 0) 71 | return t 72 | } 73 | -------------------------------------------------------------------------------- /pkg/util/ts_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "context" 18 | "errors" 19 | "time" 20 | 21 | . "github.com/pingcap/check" 22 | pd "github.com/tikv/pd/client" 23 | ) 24 | 25 | type tsSuite struct{} 26 | 27 | var _ = Suite(&tsSuite{}) 28 | 29 | func (s *tsSuite) TestGetApproachTS(c *C) { 30 | c.Assert(GetApproachTS(0, time.Now()), Equals, int64(0)) 31 | 32 | t := time.Now().Add(-1 * time.Second) 33 | ats := GetApproachTS(10, t) 34 | c.Assert(ats, Equals, int64(10+1000< 0 means some unread binlog was purged", 47 | }, []string{"id"}) 48 | ) 49 | 50 | var registry = prometheus.NewRegistry() 51 | 52 | func init() { 53 | storage.InitMetircs(registry) 54 | 55 | registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) 56 | registry.MustRegister(collectors.NewGoCollector()) 57 | 58 | registry.MustRegister(rpcHistogram) 59 | registry.MustRegister(lossBinlogCacheCounter) 60 | } 61 | -------------------------------------------------------------------------------- /pump/status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package pump 15 | 16 | import ( 17 | "encoding/json" 18 | "net/http" 19 | 20 | "github.com/pingcap/log" 21 | "go.uber.org/zap" 22 | 23 | "github.com/pingcap/tidb-binlog/pkg/node" 24 | pb "github.com/pingcap/tipb/go-binlog" 25 | ) 26 | 27 | // HTTPStatus exposes current status of all pumps via HTTP 28 | type HTTPStatus struct { 29 | StatusMap map[string]*node.Status `json:"status"` 30 | CommitTS int64 `json:"CommitTS"` 31 | CheckPoint pb.Pos `json:"Checkpoint"` 32 | ErrMsg string `json:"ErrMsg"` 33 | } 34 | 35 | // Status implements http.ServeHTTP interface 36 | func (s *HTTPStatus) Status(w http.ResponseWriter, r *http.Request) { 37 | err := json.NewEncoder(w).Encode(s) 38 | if err != nil { 39 | log.Error("Encode JSON status", zap.Any("status", s), zap.Error(err)) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /pump/storage/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package storage 15 | 16 | import "github.com/pingcap/errors" 17 | 18 | var ( 19 | // ErrWrongMagic means the magic number mismatch 20 | ErrWrongMagic = errors.New("wrong magic") 21 | ) 22 | -------------------------------------------------------------------------------- /pump/storage/helper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | // Copy from https://github.com/pingcap/tidb/blob/71def9c7263432c0dfa6a5960f6db824775177c9/store/helper/helper.go#L47 15 | // we can use it directly if we upgrade to the latest version of TiDB dependency. 16 | 17 | package storage 18 | 19 | import ( 20 | "context" 21 | "time" 22 | 23 | "github.com/pingcap/errors" 24 | "github.com/pingcap/kvproto/pkg/kvrpcpb" 25 | "github.com/pingcap/log" 26 | "github.com/pingcap/tidb/kv" 27 | "github.com/tikv/client-go/v2/tikv" 28 | "github.com/tikv/client-go/v2/tikvrpc" 29 | "go.uber.org/zap" 30 | ) 31 | 32 | // Helper is a middleware to get some information from tikv/pd. 33 | type Helper struct { 34 | Store tikv.Storage 35 | RegionCache *tikv.RegionCache 36 | } 37 | 38 | // GetMvccByEncodedKey get the MVCC value by the specific encoded key. 39 | func (h *Helper) GetMvccByEncodedKey(encodedKey kv.Key) (*kvrpcpb.MvccGetByKeyResponse, error) { 40 | keyLocation, err := h.RegionCache.LocateKey(tikv.NewBackoffer(context.Background(), 500), encodedKey) 41 | if err != nil { 42 | return nil, errors.Trace(err) 43 | } 44 | 45 | tikvReq := tikvrpc.NewRequest( 46 | tikvrpc.CmdMvccGetByKey, 47 | &kvrpcpb.MvccGetByKeyRequest{ 48 | Key: encodedKey, 49 | }, 50 | ) 51 | kvResp, err := h.Store.SendReq(tikv.NewBackoffer(context.Background(), 500), tikvReq, keyLocation.Region, time.Minute) 52 | if err != nil { 53 | log.Info("get MVCC by encoded key failed", 54 | zap.Binary("encodeKey", encodedKey), 55 | zap.Reflect("region", keyLocation.Region), 56 | zap.Binary("startKey", keyLocation.StartKey), 57 | zap.Binary("endKey", keyLocation.EndKey), 58 | zap.Reflect("kvResp", kvResp), 59 | zap.Error(err)) 60 | return nil, errors.Trace(err) 61 | } 62 | return kvResp.Resp.(*kvrpcpb.MvccGetByKeyResponse), nil 63 | } 64 | -------------------------------------------------------------------------------- /pump/storage/log_default.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | //go:build !linux 15 | // +build !linux 16 | 17 | package storage 18 | 19 | func (lf *logFile) fdatasync() error { 20 | return lf.fd.Sync() 21 | } 22 | -------------------------------------------------------------------------------- /pump/storage/log_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | //go:build linux 15 | // +build linux 16 | 17 | package storage 18 | 19 | import "syscall" 20 | 21 | // fdatasync() is similar to fsync(), but does not flush modified metadata unless that metadata is needed in order to allow a subsequent data retrieval to be correctly handled 22 | // https://linux.die.net/man/2/fdatasync 23 | // in some os don't support fdatasync, we can just use sync 24 | func (lf *logFile) fdatasync() error { 25 | return syscall.Fdatasync(int(lf.fd.Fd())) 26 | } 27 | -------------------------------------------------------------------------------- /pump/storage/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package storage 15 | 16 | import ( 17 | "encoding/binary" 18 | "sync/atomic" 19 | 20 | "github.com/dustin/go-humanize" 21 | "github.com/pingcap/errors" 22 | ) 23 | 24 | var tsKeyPrefix = []byte("ts:") 25 | 26 | func decodeTSKey(key []byte) int64 { 27 | // check bound 28 | _ = key[len(tsKeyPrefix)+8-1] 29 | 30 | return int64(binary.BigEndian.Uint64(key[len(tsKeyPrefix):])) 31 | } 32 | 33 | func encodeTSKey(ts int64) []byte { 34 | buf := make([]byte, 8+len(tsKeyPrefix)) 35 | copy(buf, tsKeyPrefix) 36 | 37 | b := buf[len(tsKeyPrefix):] 38 | 39 | binary.BigEndian.PutUint64(b, uint64(ts)) 40 | 41 | return buf 42 | } 43 | 44 | // HumanizeBytes is used for humanize configure 45 | type HumanizeBytes uint64 46 | 47 | // Uint64 return bytes 48 | func (b HumanizeBytes) Uint64() uint64 { 49 | return uint64(b) 50 | } 51 | 52 | // UnmarshalText implements UnmarshalText 53 | func (b *HumanizeBytes) UnmarshalText(text []byte) error { 54 | var err error 55 | 56 | if len(text) == 0 { 57 | *b = 0 58 | return nil 59 | } 60 | 61 | n, err := humanize.ParseBytes(string(text)) 62 | if err != nil { 63 | return errors.Annotatef(err, "text: %s", string(text)) 64 | } 65 | 66 | *b = HumanizeBytes(n) 67 | return nil 68 | } 69 | 70 | // test helper 71 | type memOracle struct { 72 | ts int64 73 | } 74 | 75 | func newMemOracle() *memOracle { 76 | return &memOracle{ 77 | ts: 0, 78 | } 79 | } 80 | 81 | func (o *memOracle) getTS() int64 { 82 | return atomic.AddInt64(&o.ts, 1) 83 | } 84 | -------------------------------------------------------------------------------- /pump/storage/util_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package storage 15 | 16 | import ( 17 | "bytes" 18 | "sort" 19 | "testing" 20 | 21 | "github.com/BurntSushi/toml" 22 | "github.com/pingcap/check" 23 | ) 24 | 25 | // Hook up gocheck into the "go test" runner. 26 | func Test(t *testing.T) { check.TestingT(t) } 27 | 28 | type EncodeTSKeySuite struct{} 29 | 30 | var _ = check.Suite(&EncodeTSKeySuite{}) 31 | 32 | func (e *EncodeTSKeySuite) TestEncodeTSKey(c *check.C) { 33 | var tsSlice = []int64{401603357443358721, 40160311937754726, 401605694141759490, 401605694129438725} 34 | 35 | sort.Slice(tsSlice, func(i int, j int) bool { 36 | return tsSlice[i] < tsSlice[j] 37 | }) 38 | 39 | var encodes [][]byte 40 | 41 | for _, ts := range tsSlice { 42 | data := encodeTSKey(ts) 43 | encodes = append(encodes, data) 44 | 45 | decodedTS := decodeTSKey(data) 46 | c.Assert(ts, check.Equals, decodedTS) 47 | } 48 | 49 | // the encode way must be sorted like origin integer ts 50 | sorted := sort.SliceIsSorted(encodes, func(i int, j int) bool { 51 | return bytes.Compare(encodes[i], encodes[j]) < 0 52 | }) 53 | 54 | c.Assert(sorted, check.IsTrue) 55 | } 56 | 57 | type UtilSuite struct{} 58 | 59 | var _ = check.Suite(&UtilSuite{}) 60 | 61 | func (u *UtilSuite) TestHumanizeBytes(c *check.C) { 62 | var s = struct { 63 | DiskSize HumanizeBytes `toml:"disk_size" json:"disk_size"` 64 | }{} 65 | 66 | tomlData := ` 67 | disk_size = "42 MB" 68 | 69 | ` 70 | 71 | _, err := toml.Decode(tomlData, &s) 72 | c.Assert(err, check.IsNil) 73 | c.Assert(s.DiskSize.Uint64(), check.Equals, uint64(42*1000*1000)) 74 | } 75 | -------------------------------------------------------------------------------- /reparo/README.md: -------------------------------------------------------------------------------- 1 | Reparo is a TiDB binlog recovery tool which is named from Harry Potter, a famous fiction by J. K. Rowling. 2 | -------------------------------------------------------------------------------- /reparo/ddl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package reparo 15 | 16 | import ( 17 | "github.com/pingcap/errors" 18 | "github.com/pingcap/log" 19 | "github.com/pingcap/tidb-binlog/pkg/filter" 20 | "github.com/pingcap/tidb/parser" 21 | "github.com/pingcap/tidb/parser/ast" 22 | "go.uber.org/zap" 23 | ) 24 | 25 | func parseDDL(sql string) (node ast.Node, table filter.TableName, err error) { 26 | nodes, _, err := parser.New().Parse(sql, "", "") 27 | if err != nil { 28 | return nil, table, errors.Trace(err) 29 | } 30 | 31 | // we assume ddl in the following format: 32 | // 1. use db; ddl 33 | // 2. ddl (no use statement) 34 | // and we assume ddl has single schema change. 35 | for _, n := range nodes { 36 | if useStmt, ok := n.(*ast.UseStmt); ok { 37 | table.Schema = useStmt.DBName 38 | continue 39 | } 40 | 41 | node = n 42 | //FIXME: does it needed? 43 | _, isDDL := n.(ast.DDLNode) 44 | if !isDDL { 45 | log.Warn("node is not ddl, unexpected!", zap.Reflect("node", n)) 46 | continue 47 | } 48 | switch v := n.(type) { 49 | case *ast.CreateDatabaseStmt: 50 | setSchemaIfExists(&table, v.Name.O, "") 51 | case *ast.DropDatabaseStmt: 52 | setSchemaIfExists(&table, v.Name.O, "") 53 | case *ast.CreateTableStmt: 54 | setSchemaIfExists(&table, v.Table.Schema.O, v.Table.Name.O) 55 | case *ast.DropTableStmt: 56 | setSchemaIfExists(&table, v.Tables[0].Schema.O, v.Tables[0].Name.O) 57 | case *ast.AlterTableStmt: 58 | setSchemaIfExists(&table, v.Table.Schema.O, v.Table.Name.O) 59 | case *ast.RenameTableStmt: 60 | setSchemaIfExists(&table, v.TableToTables[0].OldTable.Schema.O, v.TableToTables[0].OldTable.Name.O) 61 | case *ast.TruncateTableStmt: 62 | setSchemaIfExists(&table, v.Table.Schema.O, v.Table.Name.O) 63 | case *ast.CreateIndexStmt: 64 | setSchemaIfExists(&table, v.Table.Schema.O, v.Table.Name.O) 65 | case *ast.DropIndexStmt: 66 | setSchemaIfExists(&table, v.Table.Schema.O, v.Table.Name.O) 67 | case *ast.CreateViewStmt: 68 | setSchemaIfExists(&table, v.ViewName.Schema.O, v.ViewName.Name.O) 69 | } 70 | } 71 | 72 | return 73 | } 74 | 75 | func setSchemaIfExists(table *filter.TableName, schemaName string, tableName string) { 76 | if schemaName != "" { 77 | table.Schema = schemaName 78 | } 79 | if tableName != "" { 80 | table.Table = tableName 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /reparo/ddl_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package reparo 15 | 16 | import ( 17 | "github.com/pingcap/check" 18 | "github.com/pingcap/tidb-binlog/pkg/filter" 19 | ) 20 | 21 | type testDDLSuite struct{} 22 | 23 | var _ = check.Suite(&testDDLSuite{}) 24 | 25 | func (s *testDDLSuite) TestParseDDL(c *check.C) { 26 | tests := map[string]filter.TableName{ 27 | "create database db1": {Schema: "db1", Table: ""}, 28 | "drop database db1": {Schema: "db1", Table: ""}, 29 | 30 | "use db1; create table table1(id int)": {Schema: "db1", Table: "table1"}, 31 | "create table table1(id int)": {Schema: "", Table: "table1"}, 32 | 33 | "use db1; drop table table1": {Schema: "db1", Table: "table1"}, 34 | "drop table table1": {Schema: "", Table: "table1"}, 35 | 36 | "use db1; alter table table1 drop column v1": {Schema: "db1", Table: "table1"}, 37 | "alter table table1 drop column v1": {Schema: "", Table: "table1"}, 38 | 39 | "use db1; truncate table table1": {Schema: "db1", Table: "table1"}, 40 | "truncate table table1": {Schema: "", Table: "table1"}, 41 | 42 | "use db1; create index idx on table1(id)": {Schema: "db1", Table: "table1"}, 43 | "create index idx on table1(id)": {Schema: "", Table: "table1"}, 44 | 45 | "use db1; alter table table1 drop index index_name": {Schema: "db1", Table: "table1"}, 46 | "alter table table1 drop index index_name": {Schema: "", Table: "table1"}, 47 | 48 | "use db1;rename table table1 to table2": {Schema: "db1", Table: "table1"}, 49 | "rename table table1 to table2": {Schema: "", Table: "table1"}, 50 | } 51 | 52 | for sql, table := range tests { 53 | _, parseTable, err := parseDDL(sql) 54 | c.Assert(err, check.IsNil) 55 | c.Assert(parseTable, check.DeepEquals, table, check.Commentf("sql: %s", sql)) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /reparo/decode.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package reparo 15 | 16 | import ( 17 | "io" 18 | 19 | "github.com/pingcap/errors" 20 | "github.com/pingcap/tidb-binlog/pkg/binlogfile" 21 | pb "github.com/pingcap/tidb-binlog/proto/binlog" 22 | ) 23 | 24 | // Decode decodes binlog from protobuf content. 25 | // return *pb.Binlog and how many bytes read from reader 26 | func Decode(r io.Reader) (*pb.Binlog, int64, error) { 27 | payload, length, err := binlogfile.Decode(r) 28 | if err != nil { 29 | return nil, 0, errors.Trace(err) 30 | } 31 | 32 | binlog := &pb.Binlog{} 33 | err = binlog.Unmarshal(payload) 34 | if err != nil { 35 | return nil, 0, errors.Trace(err) 36 | } 37 | return binlog, length, nil 38 | } 39 | -------------------------------------------------------------------------------- /reparo/decode_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package reparo 15 | 16 | import ( 17 | "bytes" 18 | 19 | "github.com/pingcap/check" 20 | "github.com/pingcap/tidb-binlog/pkg/binlogfile" 21 | pb "github.com/pingcap/tidb-binlog/proto/binlog" 22 | ) 23 | 24 | type testDecodeSuite struct{} 25 | 26 | var _ = check.Suite(&testDecodeSuite{}) 27 | 28 | func (s *testDecodeSuite) TestDecode(c *check.C) { 29 | binlog := &pb.Binlog{ 30 | Tp: pb.BinlogType_DDL, 31 | CommitTs: 1000000000, 32 | } 33 | 34 | data, err := binlog.Marshal() 35 | c.Assert(err, check.IsNil) 36 | 37 | data = binlogfile.Encode(data) 38 | reader := bytes.NewReader(data) 39 | 40 | decodeBinlog, n, err := Decode(reader) 41 | c.Assert(err, check.IsNil) 42 | c.Assert(int(n), check.Equals, len(data)) 43 | c.Assert(decodeBinlog, check.DeepEquals, binlog) 44 | } 45 | -------------------------------------------------------------------------------- /reparo/file_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package reparo 15 | 16 | import ( 17 | "time" 18 | 19 | . "github.com/pingcap/check" 20 | "github.com/pingcap/tidb-binlog/pkg/binlogfile" 21 | pb "github.com/pingcap/tidb-binlog/proto/binlog" 22 | gb "github.com/pingcap/tipb/go-binlog" 23 | "github.com/tikv/client-go/v2/oracle" 24 | ) 25 | 26 | var _ = Suite(&testFileSuite{}) 27 | 28 | type testFileSuite struct{} 29 | 30 | func (s *testFileSuite) TestIsAcceptableBinlogFile(c *C) { 31 | binlogDir := c.MkDir() 32 | baseTS := int64(oracle.ComposeTS(time.Now().Unix()*1000, 0)) 33 | 34 | // set SegmentSizeBytes to 1 can rotate binlog file after every binlog write 35 | segmentSizeBytes := binlogfile.SegmentSizeBytes 36 | binlogfile.SegmentSizeBytes = 1 37 | defer func() { 38 | binlogfile.SegmentSizeBytes = segmentSizeBytes 39 | }() 40 | 41 | // create binlog file 42 | for i := 0; i < 10; i++ { 43 | binlog := &pb.Binlog{ 44 | CommitTs: baseTS + int64(i), 45 | } 46 | binlogData, err := binlog.Marshal() 47 | c.Assert(err, IsNil) 48 | 49 | binloger, err := binlogfile.OpenBinlogger(binlogDir, binlogfile.SegmentSizeBytes) 50 | c.Assert(err, IsNil) 51 | _, err = binloger.WriteTail(&gb.Entity{Payload: binlogData}) 52 | c.Assert(err, IsNil) 53 | err = binloger.Close() 54 | c.Assert(err, IsNil) 55 | } 56 | 57 | reparos := []*Reparo{ 58 | { 59 | cfg: &Config{ 60 | Dir: binlogDir, 61 | StartTSO: baseTS, 62 | StopTSO: baseTS + 9, 63 | }, 64 | }, 65 | { 66 | cfg: &Config{ 67 | Dir: binlogDir, 68 | StartTSO: baseTS + 1, 69 | StopTSO: baseTS + 2, 70 | }, 71 | }, 72 | { 73 | cfg: &Config{ 74 | Dir: binlogDir, 75 | StartTSO: baseTS + 2, 76 | }, 77 | }, 78 | { 79 | cfg: &Config{ 80 | Dir: binlogDir, 81 | StopTSO: baseTS + 2, 82 | }, 83 | }, 84 | } 85 | expectFileNums := []int{10, 2, 8, 3} 86 | 87 | allFiles, err := searchFiles(binlogDir) 88 | c.Assert(err, IsNil) 89 | 90 | for i, r := range reparos { 91 | files, err := filterFiles(allFiles, r.cfg.StartTSO, r.cfg.StopTSO) 92 | c.Assert(err, IsNil) 93 | c.Assert(files, HasLen, expectFileNums[i]) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /reparo/syncer/memory.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package syncer 15 | 16 | // execute sql to mysql/tidb 17 | 18 | import ( 19 | pb "github.com/pingcap/tidb-binlog/proto/binlog" 20 | ) 21 | 22 | // MemSyncer just save the pb.Binlog in memory, for test only 23 | type MemSyncer struct { 24 | binlogs []*pb.Binlog 25 | } 26 | 27 | var _ Syncer = &MemSyncer{} 28 | 29 | func newMemSyncer() (*MemSyncer, error) { 30 | return &MemSyncer{}, nil 31 | } 32 | 33 | // Sync implement interface of Syncer 34 | func (m *MemSyncer) Sync(pbBinlog *pb.Binlog, cb func(binlog *pb.Binlog)) error { 35 | m.binlogs = append(m.binlogs, pbBinlog) 36 | cb(pbBinlog) 37 | 38 | return nil 39 | } 40 | 41 | // Close implement interface of Syncer 42 | func (m *MemSyncer) Close() error { 43 | return nil 44 | } 45 | 46 | // GetBinlogs return binlogs receive 47 | func (m *MemSyncer) GetBinlogs() []*pb.Binlog { 48 | return m.binlogs 49 | } 50 | -------------------------------------------------------------------------------- /reparo/syncer/memory_test.go: -------------------------------------------------------------------------------- 1 | package syncer 2 | 3 | import ( 4 | "github.com/pingcap/check" 5 | ) 6 | 7 | type testMemorySuite struct{} 8 | 9 | var _ = check.Suite(&testMemorySuite{}) 10 | 11 | func (s *testMemorySuite) TestMemorySyncer(c *check.C) { 12 | syncer, err := newMemSyncer() 13 | c.Assert(err, check.IsNil) 14 | 15 | syncTest(c, Syncer(syncer)) 16 | 17 | binlog := syncer.GetBinlogs() 18 | c.Assert(binlog, check.HasLen, 2) 19 | 20 | err = syncer.Close() 21 | c.Assert(err, check.IsNil) 22 | } 23 | -------------------------------------------------------------------------------- /reparo/syncer/syncer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package syncer 15 | 16 | import ( 17 | "fmt" 18 | 19 | pb "github.com/pingcap/tidb-binlog/proto/binlog" 20 | ) 21 | 22 | // Syncer is the interface for executing binlog event to the target. 23 | type Syncer interface { 24 | // Sync the binlog into target database. 25 | Sync(pbBinlog *pb.Binlog, successCB func(binlog *pb.Binlog)) error 26 | 27 | // Close closes the Syncer 28 | Close() error 29 | } 30 | 31 | // New creates a new executor based on the name. 32 | func New(name string, cfg *DBConfig, worker int, batchSize int, safemode bool) (Syncer, error) { 33 | switch name { 34 | case "mysql": 35 | return newMysqlSyncer(cfg, worker, batchSize, safemode) 36 | case "print": 37 | return newPrintSyncer() 38 | case "memory": 39 | return newMemSyncer() 40 | } 41 | panic(fmt.Sprintf("unknown syncer %s", name)) 42 | } 43 | -------------------------------------------------------------------------------- /reparo/syncer/syncer_test.go: -------------------------------------------------------------------------------- 1 | package syncer 2 | 3 | import ( 4 | "reflect" 5 | 6 | "github.com/pingcap/check" 7 | ) 8 | 9 | type testSyncerSuite struct{} 10 | 11 | var _ = check.Suite(&testSyncerSuite{}) 12 | 13 | func (s *testSyncerSuite) TestNewSyncer(c *check.C) { 14 | cfg := new(DBConfig) 15 | 16 | testCases := []struct { 17 | typeStr string 18 | tp reflect.Type 19 | checker check.Checker 20 | }{ 21 | { 22 | "print", 23 | reflect.TypeOf(new(printSyncer)), 24 | check.Equals, 25 | }, { 26 | "memory", 27 | reflect.TypeOf(new(MemSyncer)), 28 | check.Equals, 29 | }, { 30 | "print", 31 | reflect.TypeOf(new(MemSyncer)), 32 | check.Not(check.Equals), 33 | }, 34 | } 35 | 36 | for _, testCase := range testCases { 37 | syncer, err := New(testCase.typeStr, cfg, 16, 20, false) 38 | c.Assert(err, check.IsNil) 39 | c.Assert(reflect.TypeOf(syncer), testCase.checker, testCase.tp) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /reparo/syncer/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package syncer 15 | 16 | import ( 17 | "fmt" 18 | 19 | "github.com/pingcap/tidb/parser/mysql" 20 | "github.com/pingcap/tidb/types" 21 | ) 22 | 23 | func formatValueToString(data types.Datum, tp byte) string { 24 | val := data.GetValue() 25 | switch tp { 26 | case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeNewDate, mysql.TypeTimestamp, mysql.TypeDuration, mysql.TypeNewDecimal, mysql.TypeVarchar, mysql.TypeString, mysql.TypeJSON: 27 | if val != nil { 28 | return fmt.Sprintf("%s", val) 29 | } 30 | fallthrough 31 | default: 32 | return fmt.Sprintf("%v", val) 33 | } 34 | } 35 | 36 | func formatValue(value types.Datum, tp byte) types.Datum { 37 | if value.GetValue() == nil { 38 | return value 39 | } 40 | 41 | switch tp { 42 | case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeNewDate, mysql.TypeTimestamp, mysql.TypeDuration, mysql.TypeNewDecimal, mysql.TypeVarchar, mysql.TypeString, mysql.TypeJSON: 43 | value = types.NewDatum(fmt.Sprintf("%s", value.GetValue())) 44 | case mysql.TypeEnum: 45 | value = types.NewDatum(value.GetMysqlEnum().Value) 46 | case mysql.TypeSet: 47 | value = types.NewDatum(value.GetMysqlSet().Value) 48 | case mysql.TypeBit: 49 | // see drainer/translator/mysql.go formatData 50 | value = types.NewDatum(value.GetUint64()) 51 | } 52 | 53 | return value 54 | } 55 | -------------------------------------------------------------------------------- /reparo/syncer/util_test.go: -------------------------------------------------------------------------------- 1 | package syncer 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/pingcap/check" 7 | "github.com/pingcap/tidb/parser/mysql" 8 | "github.com/pingcap/tidb/types" 9 | ) 10 | 11 | type testUtilSuite struct{} 12 | 13 | var _ = check.Suite(&testUtilSuite{}) 14 | 15 | func (s *testUtilSuite) TestFormatValue(c *check.C) { 16 | datetime, err := time.Parse("20060102150405", "20190415121212") 17 | c.Assert(err, check.IsNil) 18 | bitVal := uint64(12) 19 | 20 | testCases := []struct { 21 | value interface{} 22 | tp byte 23 | expectStr string 24 | expectVal interface{} 25 | }{ 26 | { 27 | value: 1, 28 | tp: mysql.TypeInt24, 29 | expectStr: "1", 30 | expectVal: int64(1), 31 | }, 32 | { 33 | value: 1.11, 34 | tp: mysql.TypeFloat, 35 | expectStr: "1.11", 36 | expectVal: float64(1.11), 37 | }, 38 | { 39 | value: 1.11, 40 | tp: mysql.TypeDouble, 41 | expectStr: "1.11", 42 | expectVal: float64(1.11), 43 | }, 44 | { 45 | value: "a", 46 | tp: mysql.TypeVarchar, 47 | expectStr: "a", 48 | expectVal: "a", 49 | }, 50 | { 51 | value: "a", 52 | tp: mysql.TypeString, 53 | expectStr: "a", 54 | expectVal: "a", 55 | }, 56 | { 57 | value: datetime, 58 | tp: mysql.TypeDatetime, 59 | expectStr: "2019-04-15 12:12:12 +0000 UTC", 60 | expectVal: "2019-04-15 12:12:12 +0000 UTC", 61 | }, 62 | { 63 | value: time.Duration(time.Second), 64 | tp: mysql.TypeDuration, 65 | expectStr: "1s", 66 | expectVal: "1s", 67 | }, 68 | { 69 | value: types.Enum{Name: "a", Value: 1}, 70 | tp: mysql.TypeEnum, 71 | expectStr: "a", 72 | expectVal: uint64(1), 73 | }, 74 | { 75 | value: types.Set{Name: "a", Value: 1}, 76 | tp: mysql.TypeSet, 77 | expectStr: "a", 78 | expectVal: uint64(1), 79 | }, 80 | { 81 | value: bitVal, 82 | tp: mysql.TypeBit, 83 | expectStr: "12", 84 | expectVal: uint64(12), 85 | }, 86 | } 87 | 88 | for _, testCase := range testCases { 89 | datum := types.NewDatum(testCase.value) 90 | str := formatValueToString(datum, testCase.tp) 91 | c.Assert(str, check.Equals, testCase.expectStr) 92 | 93 | fv := formatValue(datum, testCase.tp) 94 | c.Assert(fv.GetValue(), check.DeepEquals, testCase.expectVal) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This folder contains all tests which relies on external service such as TiDB. 4 | 5 | ## Preprations 6 | 7 | 1. The following three executables must be copied or linked into these locations: 8 | 9 | - `bin/pd-server` 10 | - `bin/tikv-server` 11 | - `bin/tidb-server` 12 | - `bin/binlogctl` 13 | - `bin/sync_diff_inspector` 14 | 15 | 2. The following programs must be installed: 16 | 17 | - `mysql`(the CLI client) 18 | - `kafka` working on default port 9092 19 | 20 | 3. The user executing the tests must have permission to create the folder 21 | 22 | `/tmp/tidb_binlog_test/pump`. All test artifacts will be written into this folder. 23 | 24 | ## Running 25 | 26 | Run `make integration_test` to execute the integration tests. This command will 27 | 28 | 1. Check that all executables exist. 29 | 2. Build `pump` and `drainer` 30 | 3. Execute `tests/run.sh` 31 | 32 | If the first two steps are done before, you could also run `tests/run.sh directly. 33 | 34 | The scrip will 35 | 36 | 1. Start PD, TiKV, TiDB, Pump, Drainer in backgroud 37 | 38 | 2. Find out all `tests/*/run.sh` and run it. 39 | 40 | Run `tests/run.sh --debug` to pause immediately after all servers are started. 41 | 42 | ## Writing new tests 43 | 44 | New integration tests can be written as shell script in `tests/TEST_NAME/run.sh`. 45 | 46 | The script should exit with a nonzero error code on failure. 47 | 48 | Serveral convenient commands are provided: 49 | 50 | - `run_drainer` Starts `drainer` using tests/TEST_NAME/drainer.toml (notice it may continue at the checkpoint from the last test case) 51 | - `run_sql ` Executes an SQL query on the TiDB database(port 4000) 52 | - `down_run_sql ` Executes an SQL query on the downstream TiDB database(port 3306) 53 | 54 | - `check_contains ` — Checks if the previous `run_sql` result contains the given text 55 | (in `-E` format) 56 | - `check_not_contains ` — Checks if the previous `run_sql` result does not contain the given 57 | text (in `-E` format) 58 | - `check_status ` — Checks if the node's status is STATUS, if NODE_TYPE is 'drainer', don't need set the NODE_ID 59 | - `check_data ` - Checks data between TiDB and downstream by sync_diff_inspector, this tool can download from [tidb-enterprise-tools-latest-linux-amd64.tar.gz](https://download.pingcap.org/tidb-enterprise-tools-latest-linux-amd64.tar.gz) -------------------------------------------------------------------------------- /tests/_utils/check_contains: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eu 4 | OUT_DIR=/tmp/tidb_binlog_test 5 | 6 | if ! grep -Fq "$1" "$OUT_DIR/sql_res.$TEST_NAME.txt"; then 7 | echo "TEST FAILED: OUTPUT DOES NOT CONTAIN '$1'" 8 | echo "____________________________________" 9 | cat "$OUT_DIR/sql_res.$TEST_NAME.txt" 10 | echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" 11 | exit 1 12 | fi 13 | -------------------------------------------------------------------------------- /tests/_utils/check_data: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | OUT_DIR=/tmp/tidb_binlog_test 4 | 5 | CONFIG_FILE=$1 6 | cp $CONFIG_FILE $OUT_DIR/diff.toml 7 | rm -rf $OUT_DIR/sync_diff_$TEST_NAME 8 | sed -i "s/test-name-placeholder/$TEST_NAME/g" $OUT_DIR/diff.toml 9 | sync_diff_inspector --log-level debug --config=$OUT_DIR/diff.toml > $OUT_DIR/diff.log 2>&1 10 | if [ $? -ne 0 ]; then 11 | cat $OUT_DIR/diff.log 12 | exit 1 13 | fi 14 | -------------------------------------------------------------------------------- /tests/_utils/check_not_contains: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eu 4 | OUT_DIR=/tmp/tidb_binlog_test 5 | 6 | if grep -Fq "$1" "$OUT_DIR/sql_res.$TEST_NAME.txt"; then 7 | echo "TEST FAILED: OUTPUT CONTAINS '$1'" 8 | echo "____________________________________" 9 | cat "$OUT_DIR/sql_res.$TEST_NAME.txt" 10 | echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" 11 | exit 1 12 | fi 13 | -------------------------------------------------------------------------------- /tests/_utils/check_status: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eu 4 | 5 | NODE_KIND=$1 6 | if [ $NODE_KIND == "pumps" ]; then 7 | NODE_ID=$2 8 | STATE=$3 9 | else 10 | NODE_ID="drainer" 11 | STATE=$2 12 | fi 13 | 14 | OUT_DIR=/tmp/tidb_binlog_test 15 | STATUS_LOG="${OUT_DIR}/status.log" 16 | max_commit_ts_old=0 17 | max_commit_ts_new=0 18 | 19 | for i in {1..15} 20 | do 21 | binlogctl -ssl-ca $OUT_DIR/cert/ca.pem \ 22 | -ssl-cert $OUT_DIR/cert/client.pem \ 23 | -ssl-key $OUT_DIR/cert/client.key \ 24 | -pd-urls https://127.0.0.1:2379 -cmd $NODE_KIND -show-offline-nodes > $STATUS_LOG 2>&1 25 | cat $STATUS_LOG 26 | 27 | if [ $NODE_KIND == "pumps" ]; then 28 | count=`grep "$NODE_ID" $STATUS_LOG | grep -c "$STATE" || true` 29 | else 30 | count=`grep -c "$STATE" $STATUS_LOG || true` 31 | fi 32 | 33 | if [ $i -eq 1 ]; then 34 | max_commit_ts_old=`cat $STATUS_LOG | sed 's/.*MaxCommitTS: \([0-9]*\), .*/\1/g'` 35 | else 36 | max_commit_ts_new=`cat $STATUS_LOG | sed 's/.*MaxCommitTS: \([0-9]*\), .*/\1/g'` 37 | fi 38 | 39 | # if status is online, will check the max commit ts, the new max commit ts should greater than the old one. 40 | if [ $count -ne 1 ] || ([ $STATE == "online" ] && [ $max_commit_ts_new -le $max_commit_ts_old ]); then 41 | if [ $i -eq 15 ]; then 42 | echo "${NODE_ID}'s status is not $STATE, or max commit ts is not update, old max commit ts is $max_commit_ts_old, new max commit ts is $max_commit_ts_new" 43 | exit 2 44 | else 45 | sleep 1 46 | fi 47 | else 48 | break 49 | fi 50 | done 51 | -------------------------------------------------------------------------------- /tests/_utils/down_run_sql: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eu 4 | 5 | OUT_DIR=/tmp/tidb_binlog_test 6 | 7 | echo "[$(date)] Executing SQL: $1" > "$OUT_DIR/sql_res.$TEST_NAME.txt" 8 | mysql -uroot -h127.0.0.1 -P3306 --default-character-set utf8 -E -e "$1" > "$OUT_DIR/sql_res.$TEST_NAME.txt" 9 | -------------------------------------------------------------------------------- /tests/_utils/run_drainer: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ue 4 | 5 | OUT_DIR=/tmp/tidb_binlog_test 6 | 7 | # kill drainer, util no drainer process is running 8 | while : 9 | do 10 | drainer_num=`ps aux > temp && grep "drainer -log-file" temp | wc -l && rm temp` 11 | if [ $drainer_num -ne 0 ]; then 12 | killall drainer || true 13 | sleep 1 14 | else 15 | break 16 | fi 17 | done 18 | 19 | config=${TEST_DIR-.}/drainer.toml 20 | 21 | echo "[$(date)] <<<<<< START IN TEST ${TEST_NAME-} FOR: $config >>>>>>" >> "$OUT_DIR/drainer.log" 22 | 23 | if [ -f "$config" ] 24 | then 25 | rm -f $OUT_DIR/drainer-config-tmp.toml 26 | cp $config $OUT_DIR/drainer-config-tmp.toml 27 | fi 28 | 29 | # Append the TLS config 30 | cat - >> "$OUT_DIR/drainer-config-tmp.toml" <> $OUT_DIR/drainer.log 2>&1 40 | -------------------------------------------------------------------------------- /tests/_utils/run_pump: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ue 4 | 5 | PORT=${1-8250} 6 | 7 | OUT_DIR=/tmp/tidb_binlog_test 8 | 9 | # kill pump, util no pump process is running 10 | while : 11 | do 12 | pump_num=`ps aux > temp && grep "pump -log-file ${OUT_DIR}/pump_${PORT}.log" temp | wc -l && rm temp` 13 | if [ $pump_num -ne 0 ]; then 14 | echo "try pause pump" 15 | binlogctl -ssl-ca $OUT_DIR/cert/ca.pem \ 16 | -ssl-cert $OUT_DIR/cert/client.pem \ 17 | -ssl-key $OUT_DIR/cert/client.key \ 18 | -pd-urls https://127.0.0.1:2379 -cmd pause-pump -node-id pump:$PORT || true 19 | sleep 1 20 | else 21 | break 22 | fi 23 | done 24 | 25 | echo "[$(date)] <<<<<< RUNNING pump >>>>>>" >> "$OUT_DIR/pump_$PORT.log" 26 | 27 | cat - > "$OUT_DIR/pump-config.toml" <> $OUT_DIR/pump_$PORT.log 2>&1 41 | -------------------------------------------------------------------------------- /tests/_utils/run_reparo: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ue 4 | 5 | OUT_DIR=/tmp/tidb_binlog_test 6 | 7 | killall reparo || true 8 | 9 | 10 | config=${TEST_DIR-.}/reparo.toml 11 | log=$OUT_DIR/reparo.log 12 | 13 | echo "[$(date)] <<<<<< START IN TEST ${TEST_NAME-} FOR: $config >>>>>>" >> "$log" 14 | 15 | if [ -f "$config" ] 16 | then 17 | reparo -config $config -log-file $log >> $log 2>&1 18 | else 19 | reapro -log-file $log >> $log 2>&1 20 | fi 21 | -------------------------------------------------------------------------------- /tests/_utils/run_sql: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eu 4 | 5 | OUT_DIR=/tmp/tidb_binlog_test 6 | 7 | echo "[$(date)] Executing SQL: $1" > "$OUT_DIR/sql_res.$TEST_NAME.txt" 8 | mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -E -e "$1" > "$OUT_DIR/sql_res.$TEST_NAME.txt" 9 | -------------------------------------------------------------------------------- /tests/attributes/drainer.toml: -------------------------------------------------------------------------------- 1 | data-dir = '/tmp/tidb_binlog_test/data.drainer' 2 | 3 | [syncer] 4 | txn-batch = 1 5 | worker-count = 1 6 | safe-mode = false 7 | db-type = 'mysql' 8 | replicate-do-db = ['attr'] 9 | 10 | [syncer.to] 11 | host = '127.0.0.1' 12 | user = 'root' 13 | password = '' 14 | port = 3306 15 | -------------------------------------------------------------------------------- /tests/attributes/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | run_drainer & 8 | 9 | sleep 3 10 | 11 | run_sql "CREATE DATABASE attr;" 12 | run_sql "CREATE TABLE attr.attributes_t1 (id INT PRIMARY KEY, name VARCHAR(50));" 13 | 14 | run_sql "ALTER TABLE attr.attributes_t1 ATTRIBUTES='merge_option=deny';" 15 | run_sql "INSERT INTO attr.attributes_t1 (id, name) VALUES (1, 'test1');" 16 | 17 | run_sql "CREATE TABLE attr.attributes_t2 (id INT PRIMARY KEY, name VARCHAR(50)) PARTITION BY RANGE (id) (PARTITION p0 VALUES LESS THAN (10000), PARTITION p1 VALUES LESS THAN (MAXVALUE));" 18 | run_sql "ALTER TABLE attr.attributes_t2 ATTRIBUTES='merge_option=deny';" 19 | run_sql "ALTER TABLE attr.attributes_t2 PARTITION p0 ATTRIBUTES='merge_option=allow';" 20 | run_sql "INSERT INTO attr.attributes_t2 (id, name) VALUES (2, 'test2');" 21 | 22 | run_sql "DROP TABLE attr.attributes_t1;" 23 | run_sql "RECOVER TABLE attr.attributes_t1;" 24 | run_sql "TRUNCATE TABLE attr.attributes_t1;" 25 | run_sql "FLASHBACK TABLE attr.attributes_t1 TO attributes_t1_back;" 26 | run_sql "RENAME TABLE attr.attributes_t1_back TO attr.attributes_t1_new;" 27 | run_sql "ALTER TABLE attr.attributes_t2 DROP PARTITION p0;" 28 | run_sql "ALTER TABLE attr.attributes_t2 TRUNCATE PARTITION p1;" 29 | 30 | sleep 3 31 | 32 | down_run_sql "SELECT count(TABLE_NAME) FROM information_schema.TABLES WHERE TABLE_SCHEMA='attr';" 33 | check_contains "count(TABLE_NAME): 3" 34 | 35 | run_sql "DROP TABLE attr.attributes_t1;" 36 | run_sql "DROP TABLE attr.attributes_t1_new;" 37 | run_sql "DROP TABLE attr.attributes_t2;" 38 | 39 | killall drainer 40 | -------------------------------------------------------------------------------- /tests/binlog/binlog.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "flag" 18 | "os" 19 | 20 | _ "github.com/go-sql-driver/mysql" 21 | "github.com/pingcap/errors" 22 | "github.com/pingcap/log" 23 | "github.com/pingcap/tidb-binlog/tests/dailytest" 24 | "github.com/pingcap/tidb-binlog/tests/util" 25 | ) 26 | 27 | func main() { 28 | cfg := util.NewConfig() 29 | err := cfg.Parse(os.Args[1:]) 30 | switch errors.Cause(err) { 31 | case nil: 32 | case flag.ErrHelp: 33 | os.Exit(0) 34 | default: 35 | log.S().Errorf("parse cmd flags err %s\n", err) 36 | os.Exit(2) 37 | } 38 | 39 | sourceDB, err := util.CreateDB(cfg.SourceDBCfg) 40 | if err != nil { 41 | log.S().Fatal(err) 42 | } 43 | defer func() { 44 | if err := util.CloseDB(sourceDB); err != nil { 45 | log.S().Errorf("Failed to close source database: %s\n", err) 46 | } 47 | }() 48 | 49 | targetDB, err := util.CreateDB(cfg.TargetDBCfg) 50 | if err != nil { 51 | log.S().Fatal(err) 52 | } 53 | defer func() { 54 | if err := util.CloseDB(targetDB); err != nil { 55 | log.S().Errorf("Failed to close target database: %s\n", err) 56 | } 57 | }() 58 | 59 | sourceDBs, err := util.CreateSourceDBs() 60 | if err != nil { 61 | log.S().Fatal(err) 62 | } 63 | defer func() { 64 | if err := util.CloseDBs(sourceDBs); err != nil { 65 | log.S().Errorf("Failed to close source databases: %s\n", err) 66 | } 67 | }() 68 | 69 | dailytest.RunMultiSource(sourceDBs, targetDB, cfg.SourceDBCfg.Name) 70 | dailytest.Run(sourceDB, targetDB, cfg.SourceDBCfg.Name, cfg.WorkerCount, cfg.JobCount, cfg.Batch) 71 | } 72 | -------------------------------------------------------------------------------- /tests/binlog/config.toml: -------------------------------------------------------------------------------- 1 | # Importer Configuration. 2 | 3 | log-level = "info" 4 | 5 | worker-count = 10 6 | job-count = 1000 7 | batch = 10 8 | 9 | [source-db] 10 | host = "127.0.0.1" 11 | user = "root" 12 | password = "" 13 | name = "test" 14 | port = 4000 15 | 16 | [target-db] 17 | host = "127.0.0.1" 18 | user = "root" 19 | password = "" 20 | name = "test" 21 | port = 3306 22 | 23 | [diff] 24 | equal-index = true 25 | equal-create-table = true 26 | equal-row-count = true 27 | equal-data = true 28 | -------------------------------------------------------------------------------- /tests/binlog/drainer.toml: -------------------------------------------------------------------------------- 1 | # drainer Configuration. 2 | 3 | # addr (i.e. 'host:port') to listen on for drainer connections 4 | # will register this addr into etcd 5 | # addr = "127.0.0.1:8249" 6 | 7 | # the interval time (in seconds) of detect pumps' status 8 | detect-interval = 10 9 | 10 | # drainer meta data directory path 11 | data-dir = "data.drainer" 12 | 13 | # a comma separated list of PD endpoints 14 | pd-urls = "http://127.0.0.1:2379" 15 | 16 | #[security] 17 | # Path of file that contains list of trusted SSL CAs for connection with cluster components. 18 | # ssl-ca = "/path/to/ca.pem" 19 | # Path of file that contains X509 certificate in PEM format for connection with cluster components. 20 | # ssl-cert = "/path/to/pump.pem" 21 | # Path of file that contains X509 key in PEM format for connection with cluster components. 22 | # ssl-key = "/path/to/pump-key.pem" 23 | 24 | # syncer Configuration. 25 | [syncer] 26 | 27 | 28 | # just for test compatible 29 | disable-dispatch = false 30 | enable-dispatch = true 31 | disable-detect = false 32 | enable-detect = true 33 | 34 | # disable sync these schema 35 | ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" 36 | 37 | # number of binlog events in a transaction batch 38 | txn-batch = 200 39 | 40 | # work count to execute binlogs 41 | worker-count = 20 42 | 43 | # safe mode will split update to delete and insert 44 | safe-mode = false 45 | 46 | # downstream storage, equal to --dest-db-type 47 | # valid values are "mysql", "file", "tidb", "kafka" 48 | db-type = "mysql" 49 | 50 | ##replicate-do-db priority over replicate-do-table if have same db name 51 | ##and we support regex expression , start with '~' declare use regex expression. 52 | # 53 | #replicate-do-db = ["~^b.*","s1"] 54 | #[[syncer.replicate-do-table]] 55 | #db-name ="test" 56 | #tbl-name = "log" 57 | 58 | #[[syncer.replicate-do-table]] 59 | #db-name ="test" 60 | #tbl-name = "~^a.*" 61 | 62 | # the downstream mysql protocol database 63 | [syncer.to] 64 | host = "127.0.0.1" 65 | user = "root" 66 | password = "" 67 | port = 3306 68 | [syncer.to.checkpoint] 69 | #schema = "tidb_binlog" 70 | 71 | # Uncomment this if you want to use file as db-type. 72 | #[syncer.to] 73 | #dir = "data.drainer" 74 | 75 | 76 | # when db-type is kafka, you can uncomment this to config the down stream kafka, it will be the globle config kafka default 77 | #[syncer.to] 78 | # only need config one of zookeeper-addrs and kafka-addrs, will get kafka address if zookeeper-addrs is configed. 79 | # zookeeper-addrs = "127.0.0.1:2181" 80 | # kafka-addrs = "127.0.0.1:9092" 81 | # kafka-version = "0.8.2.0" 82 | # kafka-max-messages = 1024 83 | # kafka-client-id = "tidb_binlog" 84 | # 85 | # 86 | # the topic name drainer will push msg, the default name is _obinlog 87 | # be careful don't use the same name if run multi drainer instances 88 | # topic-name = "" 89 | -------------------------------------------------------------------------------- /tests/binlog/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | function revert_timezone() { 8 | run_sql "SET @@global.time_zone = 'SYSTEM';" 9 | down_run_sql "SET @@global.time_zone = 'SYSTEM';" 10 | } 11 | 12 | # set timezone to others before drainer starts 13 | trap revert_timezone EXIT 14 | run_sql "SET @@global.time_zone = 'Asia/Tokyo';" 15 | down_run_sql "SET @@global.time_zone = 'EST';" 16 | 17 | run_drainer & 18 | 19 | GO111MODULE=on go build -o out 20 | 21 | ./out -config ./config.toml > ${OUT_DIR-/tmp}/$TEST_NAME.out 2>&1 22 | 23 | killall drainer 24 | -------------------------------------------------------------------------------- /tests/binlogfilter/drainer.toml: -------------------------------------------------------------------------------- 1 | # drainer Configuration. 2 | 3 | # addr (i.e. 'host:port') to listen on for drainer connections 4 | # will register this addr into etcd 5 | # addr = "127.0.0.1:8249" 6 | 7 | # the interval time (in seconds) of detect pumps' status 8 | detect-interval = 10 9 | 10 | # drainer meta data directory path 11 | data-dir = "/tmp/tidb_binlog_test/data.drainer" 12 | 13 | # a comma separated list of PD endpoints 14 | pd-urls = "http://127.0.0.1:2379" 15 | 16 | # syncer Configuration. 17 | [syncer] 18 | 19 | # number of binlog events in a transaction batch 20 | txn-batch = 1 21 | 22 | # work count to execute binlogs 23 | worker-count = 1 24 | 25 | # safe mode will split update to delete and insert 26 | safe-mode = false 27 | 28 | # downstream storage, equal to --dest-db-type 29 | # valid values are "mysql", "file", "tidb", "kafka" 30 | db-type = "mysql" 31 | 32 | # disable sync these schema 33 | ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" 34 | 35 | [[syncer.table-migrate-rule]] 36 | binlog-filter-rule = ["truncate-table-filter","add-column-aaa-filter","delete-filter"] 37 | source = { schema = "do_not_truncate_database*", table = "do_not_truncate_table*"} 38 | 39 | [[syncer.table-migrate-rule]] 40 | binlog-filter-rule = ["add-column-aaa-filter"] 41 | source = { schema = "do_not_add_col_database*", table = "do_not_add_col_table*" } 42 | 43 | [[syncer.table-migrate-rule]] 44 | binlog-filter-rule = ["delete-filter"] 45 | source = { schema = "do_not_delete_database*", table = "do_not_delete_table*" } 46 | 47 | [syncer.binlog-filter-rule] 48 | 49 | [syncer.binlog-filter-rule.truncate-table-filter] 50 | ignore-event = ["truncate table"] 51 | ignore-sql = [] 52 | 53 | [syncer.binlog-filter-rule.add-column-aaa-filter] 54 | ignore-event = [] 55 | ignore-sql = ["alter table .* add column aaa int"] 56 | 57 | [syncer.binlog-filter-rule.delete-filter] 58 | ignore-event = ["delete"] 59 | ignore-sql = [] 60 | 61 | 62 | # the downstream mysql protocol database 63 | [syncer.to] 64 | host = "127.0.0.1" 65 | user = "root" 66 | password = "" 67 | port = 3306 68 | [syncer.to.checkpoint] 69 | #schema = "tidb_binlog" 70 | 71 | 72 | -------------------------------------------------------------------------------- /tests/cache_table/drainer.toml: -------------------------------------------------------------------------------- 1 | data-dir = '/tmp/tidb_binlog_test/data.drainer' 2 | 3 | [syncer] 4 | txn-batch = 1 5 | worker-count = 1 6 | safe-mode = false 7 | db-type = 'mysql' 8 | replicate-do-db = ['cache_test'] 9 | 10 | [syncer.to] 11 | host = '127.0.0.1' 12 | user = 'root' 13 | password = '' 14 | port = 3306 15 | 16 | [syncer.to.checkpoint] 17 | schema = "cache_table_checkpoint" 18 | -------------------------------------------------------------------------------- /tests/cache_table/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | run_drainer --compressor gzip & 8 | 9 | sleep 3 10 | 11 | run_sql 'CREATE DATABASE cache_test;' 12 | run_sql 'CREATE TABLE cache_test.t1( a int, b varchar(128));' 13 | run_sql "INSERT INTO cache_test.t1 (a,b) VALUES(1,'a'),(2,'b'),(3,'c'),(4,'d');" 14 | run_sql "ALTER TABLE cache_test.t1 CACHE;" 15 | run_sql "INSERT INTO cache_test.t1 (a,b) VALUES(5,'e'),(6,'f'),(7,'g'),(8,'h');" 16 | run_sql "ALTER TABLE cache_test.t1 NOCACHE" 17 | 18 | sleep 3 19 | 20 | down_run_sql 'SELECT a, b FROM cache_test.t1 order by a' 21 | check_contains 'a: 7' 22 | check_contains 'b: g' 23 | check_contains 'a: 8' 24 | check_contains 'b: h' 25 | 26 | run_sql 'DROP DATABASE cache_test' 27 | 28 | killall drainer 29 | -------------------------------------------------------------------------------- /tests/dailytest/dailytest.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package dailytest 15 | 16 | import ( 17 | "database/sql" 18 | 19 | "github.com/pingcap/log" 20 | ) 21 | 22 | // RunMultiSource runs the test that need multi instance TiDB, one instance for one *sql.DB* in srcs 23 | func RunMultiSource(srcs []*sql.DB, targetDB *sql.DB, schema string) { 24 | runDDLTest(srcs, targetDB, schema) 25 | } 26 | 27 | // Run runs the daily test 28 | func Run(sourceDB *sql.DB, targetDB *sql.DB, schema string, workerCount int, jobCount int, batch int) { 29 | 30 | TableSQLs := []string{` 31 | create table ptest( 32 | a int primary key, 33 | b double NOT NULL DEFAULT 2.0, 34 | c varchar(10) NOT NULL, 35 | d time unique 36 | ); 37 | `, 38 | ` 39 | create table itest( 40 | a int, 41 | b double NOT NULL DEFAULT 2.0, 42 | c varchar(10) NOT NULL, 43 | d time unique, 44 | PRIMARY KEY(a, b) 45 | ); 46 | `, 47 | ` 48 | create table ntest( 49 | a int, 50 | b double NOT NULL DEFAULT 2.0, 51 | c varchar(10) NOT NULL, 52 | d time unique 53 | ); 54 | `} 55 | 56 | // run the simple test case 57 | RunCase(sourceDB, targetDB, schema) 58 | 59 | RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { 60 | // generate insert/update/delete sqls and execute 61 | RunDailyTest(sourceDB, TableSQLs, workerCount, jobCount, batch) 62 | }) 63 | 64 | RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { 65 | // truncate test data 66 | TruncateTestTable(sourceDB, TableSQLs) 67 | }) 68 | 69 | RunTest(sourceDB, targetDB, schema, func(src *sql.DB) { 70 | // drop test table 71 | DropTestTable(sourceDB, TableSQLs) 72 | }) 73 | 74 | log.S().Info("test pass!!!") 75 | 76 | } 77 | -------------------------------------------------------------------------------- /tests/dailytest/exector.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package dailytest 15 | 16 | import ( 17 | "database/sql" 18 | "fmt" 19 | "sync" 20 | 21 | "github.com/pingcap/log" 22 | ) 23 | 24 | // RunDailyTest generates insert/update/delete sqls and execute 25 | func RunDailyTest(db *sql.DB, tableSQLs []string, workerCount int, jobCount int, batch int) { 26 | var wg sync.WaitGroup 27 | wg.Add(len(tableSQLs)) 28 | 29 | for i := range tableSQLs { 30 | go func(i int) { 31 | defer wg.Done() 32 | 33 | table := newTable() 34 | err := parseTableSQL(table, tableSQLs[i]) 35 | if err != nil { 36 | log.S().Fatal(err) 37 | } 38 | 39 | err = execSQL(db, tableSQLs[i]) 40 | if err != nil { 41 | log.S().Fatal(err) 42 | } 43 | 44 | doProcess(table, db, jobCount, workerCount, batch) 45 | }(i) 46 | } 47 | 48 | wg.Wait() 49 | } 50 | 51 | // TruncateTestTable truncates test data 52 | func TruncateTestTable(db *sql.DB, tableSQLs []string) { 53 | for i := range tableSQLs { 54 | table := newTable() 55 | err := parseTableSQL(table, tableSQLs[i]) 56 | if err != nil { 57 | log.S().Fatal(err) 58 | } 59 | 60 | err = execSQL(db, fmt.Sprintf("truncate table %s", table.name)) 61 | if err != nil { 62 | log.S().Fatal(err) 63 | } 64 | } 65 | } 66 | 67 | // DropTestTable drops test table 68 | func DropTestTable(db *sql.DB, tableSQLs []string) { 69 | for i := range tableSQLs { 70 | table := newTable() 71 | err := parseTableSQL(table, tableSQLs[i]) 72 | if err != nil { 73 | log.S().Fatal(err) 74 | } 75 | 76 | err = execSQL(db, fmt.Sprintf("drop table %s", table.name)) 77 | if err != nil { 78 | log.S().Fatal(err) 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /tests/filter/drainer.toml: -------------------------------------------------------------------------------- 1 | # drainer Configuration. 2 | 3 | # addr (i.e. 'host:port') to listen on for drainer connections 4 | # will register this addr into etcd 5 | # addr = "127.0.0.1:8249" 6 | 7 | # the interval time (in seconds) of detect pumps' status 8 | detect-interval = 10 9 | 10 | # drainer meta data directory path 11 | data-dir = "/tmp/tidb_binlog_test/data.drainer" 12 | 13 | # a comma separated list of PD endpoints 14 | pd-urls = "http://127.0.0.1:2379" 15 | 16 | # syncer Configuration. 17 | [syncer] 18 | 19 | # number of binlog events in a transaction batch 20 | txn-batch = 1 21 | 22 | # work count to execute binlogs 23 | worker-count = 1 24 | 25 | # safe mode will split update to delete and insert 26 | safe-mode = false 27 | 28 | # downstream storage, equal to --dest-db-type 29 | # valid values are "mysql", "file", "tidb", "kafka" 30 | db-type = "mysql" 31 | 32 | # disable sync these schema 33 | ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" 34 | 35 | ##replicate-do-db priority over replicate-do-table if have same db name 36 | ##and we support regex expression , start with '~' declare use regex expression. 37 | # 38 | replicate-do-db = ["~^do_start.*","do_name"] 39 | 40 | [[syncer.replicate-do-table]] 41 | db-name ="test" 42 | tbl-name = "do_name" 43 | 44 | [[syncer.replicate-do-table]] 45 | db-name ="test" 46 | tbl-name = "do_ignore_name" 47 | 48 | [[syncer.replicate-do-table]] 49 | db-name ="test" 50 | tbl-name = "~^do_start.*" 51 | 52 | [[syncer.ignore-table]] 53 | db-name = "test" 54 | tbl-name = "do_ignore_name" 55 | 56 | # the downstream mysql protocol database 57 | [syncer.to] 58 | host = "127.0.0.1" 59 | user = "root" 60 | password = "" 61 | port = 3306 62 | [syncer.to.checkpoint] 63 | #schema = "tidb_binlog" 64 | 65 | 66 | -------------------------------------------------------------------------------- /tests/gencol/drainer.toml: -------------------------------------------------------------------------------- 1 | detect-interval = 10 2 | data-dir = '/tmp/tidb_binlog_test/data.drainer' 3 | pd-urls = 'http://127.0.0.1:2379' 4 | 5 | [syncer] 6 | ignore-schemas = 'INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql' 7 | txn-batch = 1 8 | worker-count = 1 9 | safe-mode = false 10 | db-type = 'mysql' 11 | replicate-do-db = ['gencol'] 12 | 13 | [syncer.to] 14 | host = '127.0.0.1' 15 | user = 'root' 16 | password = '' 17 | port = 3306 18 | -------------------------------------------------------------------------------- /tests/gencol/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | run_drainer & 8 | 9 | run_sql 'DROP TABLE IF EXISTS gencol.gct;' 10 | run_sql 'DROP DATABASE IF EXISTS gencol;' 11 | run_sql 'CREATE DATABASE gencol;' 12 | run_sql 'CREATE TABLE gencol.gct(a INT PRIMARY KEY, b INT GENERATED ALWAYS AS (a*a) VIRTUAL, c INT);' 13 | 14 | # Verify INSERT statements works with generated columns... 15 | 16 | run_sql 'INSERT INTO gencol.gct (a, c) VALUES (1, 10), (2, 12), (3, 32);' 17 | 18 | sleep 5 19 | 20 | down_run_sql 'SELECT count(*), sum(a), sum(b), sum(c) FROM gencol.gct;' 21 | check_contains 'count(*): 3' 22 | check_contains 'sum(a): 6' 23 | check_contains 'sum(b): 14' 24 | check_contains 'sum(c): 54' 25 | 26 | # Verify UPDATE statements works with generated columns... 27 | 28 | run_sql 'UPDATE gencol.gct SET a = 7, c = 8 WHERE b = 1;' 29 | 30 | sleep 5 31 | 32 | down_run_sql 'SELECT count(*), sum(a), sum(b), sum(c) FROM gencol.gct;' 33 | check_contains 'count(*): 3' 34 | check_contains 'sum(a): 12' 35 | check_contains 'sum(b): 62' 36 | check_contains 'sum(c): 52' 37 | 38 | run_sql 'UPDATE gencol.gct SET c = b WHERE a = 7;' 39 | 40 | sleep 5 41 | 42 | down_run_sql 'SELECT count(*), sum(a), sum(b), sum(c) FROM gencol.gct;' 43 | check_contains 'count(*): 3' 44 | check_contains 'sum(a): 12' 45 | check_contains 'sum(b): 62' 46 | check_contains 'sum(c): 93' 47 | 48 | # Verify DELETE statements works with generated columns... 49 | 50 | run_sql 'DELETE FROM gencol.gct WHERE b = 9;' 51 | 52 | sleep 5 53 | 54 | down_run_sql 'SELECT count(*), sum(a), sum(b), sum(c) FROM gencol.gct;' 55 | check_contains 'count(*): 2' 56 | check_contains 'sum(a): 9' 57 | check_contains 'sum(b): 53' 58 | check_contains 'sum(c): 61' 59 | 60 | killall drainer 61 | -------------------------------------------------------------------------------- /tests/kafka/drainer.toml: -------------------------------------------------------------------------------- 1 | # drainer meta data directory path 2 | data-dir = "/tmp/tidb_binlog_test/data.drainer" 3 | 4 | # a comma separated list of PD endpoints 5 | pd-urls = "http://127.0.0.1:2379" 6 | 7 | # syncer Configuration. 8 | [syncer] 9 | 10 | # disable sync these schema 11 | ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" 12 | 13 | # number of binlog events in a transaction batch 14 | txn-batch = 1 15 | 16 | # work count to execute binlogs 17 | worker-count = 1 18 | 19 | # safe mode will split update to delete and insert 20 | safe-mode = false 21 | 22 | # downstream storage, equal to --dest-db-type 23 | # valid values are "mysql", "file", "tidb", "kafka" 24 | db-type = "kafka" 25 | 26 | [syncer.to.checkpoint] 27 | #schema = "tidb_binlog" 28 | type = "mysql" 29 | host = "127.0.0.1" 30 | user = "root" 31 | password = "" 32 | port = 4000 33 | 34 | [syncer.to] 35 | topic-name = "binlog_test_topic" 36 | -------------------------------------------------------------------------------- /tests/kafka/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | args="-initial-commit-ts=-1" 8 | 9 | kafka_addr=${KAFKA_ADDRS-127.0.0.1:9092} 10 | 11 | run_drainer "$args" & 12 | 13 | GO111MODULE=on go build -o out 14 | 15 | ./out -offset=-1 -topic=binlog_test_topic -kafkaAddr=$kafka_addr -> ${OUT_DIR-/tmp}/$TEST_NAME.out 2>&1 16 | 17 | killall drainer 18 | -------------------------------------------------------------------------------- /tests/partition/drainer.toml: -------------------------------------------------------------------------------- 1 | data-dir = '/tmp/tidb_binlog_test/data.drainer' 2 | 3 | [syncer] 4 | txn-batch = 1 5 | worker-count = 1 6 | safe-mode = false 7 | db-type = 'mysql' 8 | replicate-do-db = ['partition_test'] 9 | 10 | [syncer.to] 11 | host = '127.0.0.1' 12 | user = 'root' 13 | password = '' 14 | port = 3306 15 | 16 | [syncer.to.checkpoint] 17 | schema = "partition_test_checkpoint" 18 | -------------------------------------------------------------------------------- /tests/partition/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | run_drainer & 8 | 9 | sleep 3 10 | 11 | run_sql 'create database partition_test;' 12 | # range partition 13 | run_sql 'create table partition_test.t1( a int ,b varchar(128) ) partition by range (a) (partition p0 values less than (3),partition p1 values less than (7));' 14 | run_sql "insert into partition_test.t1 (a,b) values(1,'a'),(2,'b'),(3,'c'),(4,'d');" 15 | run_sql "alter table partition_test.t1 add partition (partition p2 values less than (10));" 16 | run_sql "insert into partition_test.t1 (a,b) values(5,'e'),(6,'f'),(7,'g'),(8,'h');" 17 | 18 | # hash partition 19 | run_sql 'create table partition_test.t2(a int ,b varchar(128) ) partition by hash (a) partitions 2;' 20 | run_sql "insert into partition_test.t2 (a,b) values(1,'a'),(2,'b'),(3,'c'),(4,'d');" 21 | run_sql 'truncate table partition_test.t2;' 22 | run_sql "insert into partition_test.t2 (a,b) values(5,'e'),(6,'f'),(7,'g'),(8,'h');" 23 | 24 | sleep 3 25 | 26 | down_run_sql 'SELECT a, b FROM partition_test.t1 partition(p2) order by a' 27 | check_contains 'a: 7' 28 | check_contains 'b: g' 29 | check_contains 'a: 8' 30 | check_contains 'b: h' 31 | 32 | down_run_sql 'select a, b from partition_test.t2 order by a limit 1' 33 | check_contains 'a: 5' 34 | check_contains 'b: e' 35 | 36 | run_sql 'DROP database partition_test' 37 | 38 | killall drainer 39 | -------------------------------------------------------------------------------- /tests/placement_rules/drainer.toml: -------------------------------------------------------------------------------- 1 | data-dir = '/tmp/tidb_binlog_test/data.drainer' 2 | 3 | [syncer] 4 | txn-batch = 1 5 | worker-count = 1 6 | safe-mode = false 7 | db-type = 'mysql' 8 | replicate-do-db = ['placement_test'] 9 | 10 | [syncer.to] 11 | host = '127.0.0.1' 12 | user = 'root' 13 | password = '' 14 | port = 3306 15 | 16 | [syncer.to.checkpoint] 17 | schema = "placement_rules_checkpoint" 18 | -------------------------------------------------------------------------------- /tests/placement_rules/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | run_drainer & 8 | 9 | sleep 3 10 | 11 | run_sql 'CREATE DATABASE placement_test;' 12 | run_sql 'CREATE PLACEMENT POLICY x1 FOLLOWERS=4' 13 | run_sql 'CREATE TABLE placement_test.t1 (a INT NOT NULL PRIMARY KEY, b VARCHAR(100)) PLACEMENT POLICY=x1;' 14 | run_sql "INSERT INTO placement_test.t1 (a,b) VALUES(1,'a'),(2,'b'),(3,'c'),(4,'d');" 15 | run_sql "INSERT INTO placement_test.t1 (a,b) VALUES(5,'e'),(6,'f'),(7,'g'),(8,'h');" 16 | 17 | sleep 3 18 | 19 | down_run_sql 'SHOW PLACEMENT' 20 | check_not_contains 'Target: POLICY x1' 21 | check_not_contains 'Target: TABLE placement_test.t1' 22 | down_run_sql 'SELECT a, b FROM placement_test.t1 order by a' 23 | check_contains 'a: 7' 24 | check_contains 'b: g' 25 | check_contains 'a: 8' 26 | check_contains 'b: h' 27 | 28 | run_sql 'DROP DATABASE placement_test' 29 | 30 | killall drainer 31 | -------------------------------------------------------------------------------- /tests/reparo/binlog.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "flag" 18 | "os" 19 | 20 | _ "github.com/go-sql-driver/mysql" 21 | "github.com/pingcap/errors" 22 | "github.com/pingcap/log" 23 | 24 | "github.com/pingcap/tidb-binlog/tests/dailytest" 25 | "github.com/pingcap/tidb-binlog/tests/util" 26 | ) 27 | 28 | func main() { 29 | cfg := util.NewConfig() 30 | err := cfg.Parse(os.Args[1:]) 31 | switch errors.Cause(err) { 32 | case nil: 33 | case flag.ErrHelp: 34 | os.Exit(0) 35 | default: 36 | log.S().Errorf("parse cmd flags err %s\n", err) 37 | os.Exit(2) 38 | } 39 | 40 | sourceDB, err := util.CreateDB(cfg.SourceDBCfg) 41 | if err != nil { 42 | log.S().Fatal(err) 43 | } 44 | defer func() { 45 | if err := util.CloseDB(sourceDB); err != nil { 46 | log.S().Errorf("Failed to close source database: %s\n", err) 47 | } 48 | }() 49 | 50 | tableSQLs := []string{` 51 | create table ptest( 52 | a int primary key, 53 | b double NOT NULL DEFAULT 2.0, 54 | c varchar(10) NOT NULL, 55 | d time unique 56 | ); 57 | `, 58 | ` 59 | create table itest( 60 | a int, 61 | b double NOT NULL DEFAULT 2.0, 62 | c varchar(10) NOT NULL, 63 | d time unique, 64 | PRIMARY KEY(a, b) 65 | ); 66 | `, 67 | ` 68 | create table ntest( 69 | a int, 70 | b double NOT NULL DEFAULT 2.0, 71 | c varchar(10) NOT NULL, 72 | d time unique 73 | ); 74 | `, 75 | ` 76 | create table ntest2( 77 | a int, 78 | b double NOT NULL DEFAULT 2.0, 79 | c varchar(10) NOT NULL, 80 | d bit(20) 81 | ); 82 | `} 83 | 84 | dailytest.RunDailyTest(sourceDB, tableSQLs, cfg.WorkerCount, cfg.JobCount, cfg.Batch) 85 | } 86 | -------------------------------------------------------------------------------- /tests/reparo/config.toml: -------------------------------------------------------------------------------- 1 | # daily test config 2 | 3 | log-level = "info" 4 | 5 | worker-count = 10 6 | job-count = 1000 7 | batch = 10 8 | 9 | [source-db] 10 | host = "127.0.0.1" 11 | user = "root" 12 | password = "" 13 | name = "reparo-test" 14 | port = 4000 15 | -------------------------------------------------------------------------------- /tests/reparo/drainer.toml: -------------------------------------------------------------------------------- 1 | # drainer Configuration. 2 | 3 | # addr (i.e. 'host:port') to listen on for drainer connections 4 | # will register this addr into etcd 5 | # addr = "127.0.0.1:8249" 6 | 7 | # the interval time (in seconds) of detect pumps' status 8 | detect-interval = 10 9 | 10 | # drainer meta data directory path 11 | data-dir = "/tmp/tidb_binlog_test/data.drainer" 12 | 13 | # a comma separated list of PD endpoints 14 | pd-urls = "http://127.0.0.1:2379" 15 | 16 | # syncer Configuration. 17 | [syncer] 18 | 19 | # disable sync these schema 20 | ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql,test" 21 | 22 | # number of binlog events in a transaction batch 23 | txn-batch = 200 24 | 25 | # work count to execute binlogs 26 | worker-count = 20 27 | 28 | # safe mode will split update to delete and insert 29 | safe-mode = false 30 | 31 | # downstream storage, equal to --dest-db-type 32 | # valid values are "mysql", "file", "tidb", "kafka" 33 | db-type = "file" 34 | 35 | #[syncer.to] 36 | #dir = "/data/data.drainer" 37 | #compression = "gzip" 38 | 39 | 40 | -------------------------------------------------------------------------------- /tests/reparo/reparo.toml: -------------------------------------------------------------------------------- 1 | # drainer 输出的 protobuf 格式 binlog 文件的存储路径 2 | data-dir = "/tmp/tidb_binlog_test/data.drainer" 3 | 4 | # log-file = "" 5 | # log-rotate = "hour" 6 | log-level = "debug" 7 | 8 | # start-datetime and stop-datetime enable you to pick a range of binlog to reparo. 9 | # The datetime format is like '2018-02-28 12:12:12'. 10 | #start-datetime = "2018-10-24 00:00:00" 11 | #stop-datetime = "2018-11-26 00:00:00" 12 | 13 | # Start-tso is similar to start-datetime, but in pd-server tso format. e.g. 395181938313123110 14 | # Stop-tso is is similar to stop-datetime, but in pd-server tso format. e.g. 395181938313123110 15 | # start-tso = 0 16 | # stop-tso = 0 17 | 18 | # dest-type choose a destination, which value can be "mysql", "print". 19 | # for print, it just prints decoded value. 20 | dest-type = "mysql" 21 | 22 | # 如果指定的 dest-type 为 mysql 或 tidb，需要配置 dest-db。 23 | [dest-db] 24 | host = "127.0.0.1" 25 | port = 3306 26 | user = "root" 27 | password = "" -------------------------------------------------------------------------------- /tests/reparo/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | # use latest ts as initial-commit-ts, so we can skip binlog by previous test case 8 | args="-initial-commit-ts=-1" 9 | down_run_sql "DROP DATABASE IF EXISTS tidb_binlog" 10 | 11 | rm -rf /tmp/tidb_binlog_test/data.drainer 12 | 13 | run_drainer "$args" & 14 | 15 | GO111MODULE=on go build -o out 16 | 17 | sleep 8 18 | 19 | run_sql "CREATE DATABASE IF NOT EXISTS \`reparo-test\`" 20 | 21 | ./out -config ./config.toml > ${OUT_DIR-/tmp}/$TEST_NAME.out 2>&1 22 | 23 | sleep 5 24 | 25 | run_reparo & 26 | 27 | sleep 15 28 | 29 | check_data ./sync_diff_inspector.toml 30 | 31 | # clean up 32 | run_sql "DROP DATABASE IF EXISTS \`reparo-test\`" 33 | 34 | killall drainer 35 | -------------------------------------------------------------------------------- /tests/reparo/sync_diff_inspector.toml: -------------------------------------------------------------------------------- 1 | # diff Configuration. 2 | 3 | ######################### Global config ######################### 4 | 5 | # how many goroutines are created to check data 6 | check-thread-count = 1 7 | 8 | # set false if just want compare data by checksum, will skip select data when checksum is not equal. 9 | # set true if want compare all different rows, will slow down the total compare time. 10 | export-fix-sql = true 11 | 12 | # ignore check table's data 13 | check-struct-only = false 14 | 15 | ######################### Databases config ######################### 16 | [data-sources] 17 | [data-sources.source1] 18 | host = "127.0.0.1" 19 | port = 4000 20 | user = "root" 21 | password = "" 22 | 23 | [data-sources.target] 24 | host = "127.0.0.1" 25 | port = 3306 26 | user = "root" 27 | password = "" 28 | 29 | ######################### Task config ######################### 30 | [task] 31 | # 1 fix sql: fix-target-TIDB1.sql 32 | # 2 log: sync-diff.log 33 | # 3 summary: summary.txt 34 | # 4 checkpoint: a dir 35 | output-dir = "/tmp/tidb_binlog_test/sync_diff_test-name-placeholder" 36 | 37 | source-instances = ["source1"] 38 | 39 | target-instance = "target" 40 | 41 | # tables need to check. *Include `schema` and `table`. Use `.` to split* 42 | target-check-tables = ["/^reparo-test$/.*"] 43 | -------------------------------------------------------------------------------- /tests/resource_control/drainer.toml: -------------------------------------------------------------------------------- 1 | data-dir = '/tmp/tidb_binlog_test/data.drainer' 2 | 3 | [syncer] 4 | txn-batch = 1 5 | worker-count = 1 6 | safe-mode = false 7 | db-type = 'mysql' 8 | replicate-do-db = ['resource_control_test'] 9 | 10 | [syncer.to] 11 | host = '127.0.0.1' 12 | user = 'root' 13 | password = '' 14 | port = 3306 15 | 16 | [syncer.to.checkpoint] 17 | schema = "resource_control_checkpoint" 18 | -------------------------------------------------------------------------------- /tests/resource_control/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | run_drainer --compressor gzip & 8 | 9 | sleep 3 10 | 11 | run_sql 'CREATE DATABASE resource_control_test;' 12 | run_sql 'CREATE RESOURCE GROUP rg1 RU_PER_SEC=10000;' 13 | run_sql 'CREATE RESOURCE GROUP rg2 RU_PER_SEC=5000;' 14 | run_sql 'ALTER RESOURCE GROUP rg1 RU_PER_SEC=10000, BURSTABLE;' 15 | run_sql 'DROP RESOURCE GROUP rg2;' 16 | 17 | sleep 3 18 | 19 | run_sql 'DROP DATABASE resource_control_test;' 20 | 21 | killall drainer 22 | -------------------------------------------------------------------------------- /tests/restart/drainer.toml: -------------------------------------------------------------------------------- 1 | # drainer Configuration. 2 | 3 | # addr (i.e. 'host:port') to listen on for drainer connections 4 | # will register this addr into etcd 5 | # addr = "127.0.0.1:8249" 6 | 7 | # the interval time (in seconds) of detect pumps' status 8 | detect-interval = 10 9 | 10 | # drainer meta data directory path 11 | data-dir = "/tmp/tidb_binlog_test/data.drainer" 12 | 13 | # a comma separated list of PD endpoints 14 | pd-urls = "http://127.0.0.1:2379" 15 | 16 | # syncer Configuration. 17 | [syncer] 18 | 19 | # disable sync these schema 20 | ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql,test" 21 | 22 | # number of binlog events in a transaction batch 23 | txn-batch = 200 24 | 25 | # work count to execute binlogs 26 | worker-count = 20 27 | 28 | # safe mode will split update to delete and insert 29 | safe-mode = false 30 | 31 | # downstream storage, equal to --dest-db-type 32 | # valid values are "mysql", "file", "tidb", "kafka" 33 | db-type = "mysql" 34 | 35 | # the downstream mysql protocol database 36 | [syncer.to] 37 | host = "127.0.0.1" 38 | user = "root" 39 | password = "" 40 | port = 3306 41 | -------------------------------------------------------------------------------- /tests/restart/insert_data: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | run_sql "DROP DATABASE IF EXISTS RESTART_TEST;" 6 | run_sql "CREATE DATABASE RESTART_TEST;" 7 | run_sql "CREATE TABLE RESTART_TEST.TEST(a int);" 8 | 9 | # don't forget kill this process 10 | end=$((SECONDS+20)) 11 | while [ $SECONDS -lt $end ]; do 12 | run_sql "INSERT INTO RESTART_TEST.TEST VALUES(1);" 13 | sleep 0.05 14 | done 15 | -------------------------------------------------------------------------------- /tests/restart/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | OUT_DIR=/tmp/tidb_binlog_test 8 | STATUS_LOG="${OUT_DIR}/status.log" 9 | 10 | # run drainer, and drainer's status should be online 11 | # use latest ts as initial-commit-ts, so we can skip binlog by previous test case 12 | args="-initial-commit-ts=-1" 13 | down_run_sql "DROP DATABASE IF EXISTS tidb_binlog" 14 | rm -rf /tmp/tidb_binlog_test/data.drainer 15 | 16 | run_drainer "$args" & 17 | sleep 5 18 | 19 | # run a new pump 20 | run_pump 8251 & 21 | sleep 5 22 | 23 | ./insert_data & 24 | sleep 5 25 | 26 | # restart pumps 27 | run_pump 8251 & 28 | # make sure there is always one pump alive 29 | sleep 2 30 | check_status pumps "pump:8251" online 31 | run_pump 8250 & 32 | 33 | sleep 5 34 | 35 | echo "Verifying TiDB is alive..." 36 | mysql -uroot -h127.0.0.1 -P4000 --default-character-set utf8 -e 'select * from mysql.tidb;' 37 | if [ $? -ne 0 ]; then 38 | echo "TiDB is not alive!" 39 | exit 1 40 | fi 41 | 42 | killall insert_data || true 43 | sleep 2 44 | 45 | echo "after kill insert, check data" 46 | i=0 47 | while ! check_data ./sync_diff_inspector.toml; do 48 | i=$((i+1)) 49 | if [ "$i" -gt 20 ]; then 50 | echo 'data is not equal' 51 | exit 1 52 | fi 53 | sleep 2 54 | done 55 | 56 | echo "data is equal" 57 | 58 | # offline a pump 59 | binlogctl -ssl-ca $OUT_DIR/cert/ca.pem \ 60 | -ssl-cert $OUT_DIR/cert/client.pem \ 61 | -ssl-key $OUT_DIR/cert/client.key \ 62 | -pd-urls https://127.0.0.1:2379 -cmd offline-pump -node-id pump:8251 63 | sleep 1 64 | check_status pumps "pump:8251" offline 65 | -------------------------------------------------------------------------------- /tests/restart/sync_diff_inspector.toml: -------------------------------------------------------------------------------- 1 | # diff Configuration. 2 | 3 | ######################### Global config ######################### 4 | 5 | # how many goroutines are created to check data 6 | check-thread-count = 1 7 | 8 | # set false if just want compare data by checksum, will skip select data when checksum is not equal. 9 | # set true if want compare all different rows, will slow down the total compare time. 10 | export-fix-sql = true 11 | 12 | # ignore check table's data 13 | check-struct-only = false 14 | 15 | ######################### Databases config ######################### 16 | [data-sources] 17 | [data-sources.source1] 18 | host = "127.0.0.1" 19 | port = 4000 20 | user = "root" 21 | password = "" 22 | 23 | [data-sources.target] 24 | host = "127.0.0.1" 25 | port = 3306 26 | user = "root" 27 | password = "" 28 | 29 | ######################### Task config ######################### 30 | [task] 31 | # 1 fix sql: fix-target-TIDB1.sql 32 | # 2 log: sync-diff.log 33 | # 3 summary: summary.txt 34 | # 4 checkpoint: a dir 35 | output-dir = "/tmp/tidb_binlog_test/sync_diff_test-name-placeholder" 36 | 37 | source-instances = ["source1"] 38 | 39 | target-instance = "target" 40 | 41 | # tables need to check. *Include `schema` and `table`. Use `.` to split* 42 | target-check-tables = ["RESTART_TEST.TEST"] 43 | -------------------------------------------------------------------------------- /tests/sequence/drainer.toml: -------------------------------------------------------------------------------- 1 | data-dir = '/tmp/tidb_binlog_test/data.drainer' 2 | 3 | [syncer] 4 | txn-batch = 1 5 | worker-count = 1 6 | safe-mode = false 7 | db-type = 'mysql' 8 | replicate-do-db = ['seq'] 9 | 10 | [syncer.to] 11 | host = '127.0.0.1' 12 | user = 'root' 13 | password = '' 14 | port = 3306 15 | 16 | [syncer.to.checkpoint] 17 | schema = "seq_checkpoint" 18 | -------------------------------------------------------------------------------- /tests/sequence/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | 7 | run_drainer & 8 | 9 | sleep 3 10 | 11 | run_sql 'CREATE DATABASE seq;' 12 | run_sql 'CREATE SEQUENCE seq.sequence_name;' 13 | run_sql 'CREATE TABLE seq.table_name (id INT DEFAULT NEXT VALUE FOR seq.sequence_name, val int);' 14 | 15 | run_sql 'INSERT INTO seq.table_name(val) values(10);' 16 | 17 | sleep 5 18 | 19 | down_run_sql 'SELECT count(*), sum(id), sum(val) FROM seq.table_name;' 20 | check_contains 'count(*): 1' 21 | check_contains 'sum(id): 1' 22 | check_contains 'sum(val): 10' 23 | 24 | 25 | run_sql 'DROP TABLE seq.table_name;' 26 | run_sql 'DROP SEQUENCE seq.sequence_name;' 27 | 28 | 29 | killall drainer 30 | -------------------------------------------------------------------------------- /tests/status/drainer.toml: -------------------------------------------------------------------------------- 1 | # drainer Configuration. 2 | 3 | # addr (i.e. 'host:port') to listen on for drainer connections 4 | # will register this addr into etcd 5 | # addr = "127.0.0.1:8249" 6 | 7 | # the interval time (in seconds) of detect pumps' status 8 | detect-interval = 10 9 | 10 | # drainer meta data directory path 11 | data-dir = "/tmp/tidb_binlog_test/data.drainer" 12 | 13 | # a comma separated list of PD endpoints 14 | pd-urls = "http://127.0.0.1:2379" 15 | 16 | # syncer Configuration. 17 | [syncer] 18 | 19 | # disable sync these schema 20 | ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql" 21 | 22 | # number of binlog events in a transaction batch 23 | txn-batch = 1 24 | 25 | # work count to execute binlogs 26 | worker-count = 1 27 | 28 | # safe mode will split update to delete and insert 29 | safe-mode = false 30 | 31 | # downstream storage, equal to --dest-db-type 32 | # valid values are "mysql", "file", "tidb", "kafka" 33 | db-type = "file" 34 | 35 | #[syncer.to] 36 | #dir = "/data/data.drainer" 37 | #compression = "gzip" 38 | 39 | 40 | [syncer.to.checkpoint] 41 | schema = "status_checkpoint" 42 | -------------------------------------------------------------------------------- /tests/status/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | 4 | set -e 5 | 6 | cd "$(dirname "$0")" 7 | 8 | OUT_DIR=/tmp/tidb_binlog_test 9 | STATUS_LOG="${OUT_DIR}/status.log" 10 | 11 | # use latest ts as initial-commit-ts, so we can skip binlog by previous test case 12 | args="-initial-commit-ts=-1" 13 | # down_run_sql "DROP DATABASE IF EXISTS tidb_binlog" 14 | rm -rf /tmp/tidb_binlog_test/data.drainer 15 | 16 | drainerNodeID="drainer-id" 17 | # run drainer, and drainer's status should be online 18 | run_drainer "$args" & 19 | 20 | sleep 2 21 | echo "check drainer's status, should be online" 22 | check_status drainers online 23 | 24 | pumpNodeID="pump:8250" 25 | 26 | # pump's state should be online 27 | echo "check pump's status, should be online" 28 | check_status pumps $pumpNodeID online 29 | 30 | args="-ssl-ca $OUT_DIR/cert/ca.pem -ssl-cert $OUT_DIR/cert/client.pem -ssl-key $OUT_DIR/cert/client.key -pd-urls https://127.0.0.1:2379" 31 | 32 | # stop pump, and pump's state should be paused 33 | binlogctl $args -cmd pause-pump -node-id $pumpNodeID 34 | 35 | echo "check pump's status, should be paused" 36 | check_status pumps $pumpNodeID paused 37 | 38 | # offline pump, and pump's status should be offline 39 | run_pump & 40 | sleep 3 41 | binlogctl $args -cmd offline-pump -node-id $pumpNodeID 42 | 43 | echo "check pump's status, should be offline" 44 | check_status pumps $pumpNodeID offline 45 | 46 | # stop drainer, and drainer's state should be paused 47 | binlogctl $args -cmd pause-drainer -node-id $drainerNodeID 48 | 49 | echo "check drainer's status, should be paused" 50 | check_status drainers paused 51 | 52 | # offline drainer, and drainer's state should be offline 53 | run_drainer & 54 | sleep 3 55 | binlogctl $args -cmd offline-drainer -node-id $drainerNodeID 56 | 57 | echo "check drainer's status, should be offline" 58 | check_status drainers offline 59 | 60 | # update drainer's state to online, and then run pump, pump will notify drainer failed, pump's status will be paused 61 | binlogctl $args -cmd update-drainer -node-id $drainerNodeID -state online 62 | run_pump & 63 | 64 | echo "check pump's status, should be offline" 65 | check_status pumps $pumpNodeID offline 66 | 67 | # clean up 68 | binlogctl $args -cmd update-drainer -node-id $drainerNodeID -state paused 69 | run_pump & 70 | rm $STATUS_LOG || true 71 | -------------------------------------------------------------------------------- /tests/util/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 PingCAP, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "flag" 18 | "fmt" 19 | 20 | "github.com/BurntSushi/toml" 21 | "github.com/pingcap/errors" 22 | ) 23 | 24 | // NewConfig creates a new config. 25 | func NewConfig() *Config { 26 | cfg := &Config{} 27 | cfg.FlagSet = flag.NewFlagSet("binlogTest", flag.ContinueOnError) 28 | fs := cfg.FlagSet 29 | 30 | fs.StringVar(&cfg.configFile, "config", "", "Config file") 31 | fs.IntVar(&cfg.WorkerCount, "c", 1, "parallel worker count") 32 | fs.IntVar(&cfg.JobCount, "n", 1, "total job count") 33 | fs.IntVar(&cfg.Batch, "b", 1, "insert batch commit count") 34 | fs.StringVar(&cfg.LogLevel, "L", "info", "log level: debug, info, warn, error, fatal") 35 | 36 | return cfg 37 | } 38 | 39 | // Config is the configuration. 40 | type Config struct { 41 | *flag.FlagSet `json:"-"` 42 | 43 | LogLevel string `toml:"log-level" json:"log-level"` 44 | 45 | WorkerCount int `toml:"worker-count" json:"worker-count"` 46 | 47 | JobCount int `toml:"job-count" json:"job-count"` 48 | 49 | Batch int `toml:"batch" json:"batch"` 50 | 51 | SourceDBCfg DBConfig `toml:"source-db" json:"source-db"` 52 | 53 | TargetDBCfg DBConfig `toml:"target-db" json:"target-db"` 54 | 55 | configFile string 56 | } 57 | 58 | // Parse parses flag definitions from the argument list. 59 | func (c *Config) Parse(arguments []string) error { 60 | // Parse first to get config file. 61 | err := c.FlagSet.Parse(arguments) 62 | if err != nil { 63 | return errors.Trace(err) 64 | } 65 | 66 | // Load config file if specified. 67 | if c.configFile != "" { 68 | err = c.configFromFile(c.configFile) 69 | if err != nil { 70 | return errors.Trace(err) 71 | } 72 | } 73 | 74 | // Parse again to replace with command line options. 75 | err = c.FlagSet.Parse(arguments) 76 | if err != nil { 77 | return errors.Trace(err) 78 | } 79 | 80 | if len(c.FlagSet.Args()) != 0 { 81 | return errors.Errorf("'%s' is an invalid flag", c.FlagSet.Arg(0)) 82 | } 83 | 84 | return nil 85 | } 86 | 87 | func (c *Config) String() string { 88 | if c == nil { 89 | return "" 90 | } 91 | return fmt.Sprintf("Config(%+v)", *c) 92 | } 93 | 94 | // configFromFile loads config from file. 95 | func (c *Config) configFromFile(path string) error { 96 | _, err := toml.DecodeFile(path, c) 97 | return errors.Trace(err) 98 | } 99 | -------------------------------------------------------------------------------- /tools/check/check-tidy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | GO111MODULE=on go mod tidy 5 | 6 | if ! git diff-index --quiet HEAD --; then 7 | echo "Please run \`go mod tidy\` to clean up" 8 | exit 1 9 | fi 10 | -------------------------------------------------------------------------------- /tools/check/revive.toml: -------------------------------------------------------------------------------- 1 | ignoreGeneratedHeader = false 2 | severity = "error" 3 | confidence = 0.8 4 | errorCode = -1 5 | warningCode = -1 6 | 7 | [rule.blank-imports] 8 | [rule.context-as-argument] 9 | [rule.dot-imports] 10 | [rule.error-return] 11 | [rule.error-strings] 12 | [rule.error-naming] 13 | [rule.exported] 14 | [rule.if-return] 15 | [rule.var-naming] 16 | [rule.package-comments] 17 | Disabled = true 18 | [rule.range] 19 | [rule.receiver-naming] 20 | [rule.indent-error-flow] 21 | [rule.superfluous-else] 22 | [rule.modifies-parameter] 23 | 24 | # This can be checked by other tools like megacheck 25 | [rule.unreachable-code] 26 | 27 | 28 | # Currently this makes too much noise, but should add it in 29 | # and perhaps ignore it in a few files 30 | #[rule.confusing-naming] 31 | # severity = "warning" 32 | #[rule.confusing-results] 33 | # severity = "warning" 34 | #[rule.unused-parameter] 35 | # severity = "warning" 36 | #[rule.deep-exit] 37 | # severity = "warning" 38 | #[rule.flag-parameter] 39 | # severity = "warning" 40 | 41 | 42 | 43 | # Adding these will slow down the linter 44 | # They are already provided by megacheck 45 | # [rule.unexported-return] 46 | # [rule.time-naming] 47 | # [rule.errorf] 48 | 49 | # Adding these will slow down the linter 50 | # Not sure if they are already provided by megacheck 51 | # [rule.var-declaration] 52 | # [rule.context-keys-type] 53 | --------------------------------------------------------------------------------