├── .dev.env ├── .dockerignore ├── .env ├── .gitignore ├── .gometalinter.json ├── .travis.yml ├── Dockerfile.gravity ├── Dockerfile.gravity.race ├── Dockerfile.mongo.setup ├── Dockerfile.prometheus_sd ├── Dockerfile.test.gravity ├── LICENSE ├── Makefile ├── README-cn.md ├── README.md ├── cmd ├── dcp │ └── main.go ├── gravity │ └── main.go ├── padder │ └── main.go ├── prometheus_etcd_sd │ └── main.go └── verifier │ └── main.go ├── dcp ├── barrier │ ├── barrier.go │ ├── barrier_suite_test.go │ └── barrier_test.go ├── checker │ ├── alarm.go │ ├── buffer.go │ ├── checker.go │ ├── checker_suite_test.go │ ├── checker_test.go │ ├── result.go │ └── segment.go ├── collector │ ├── collector_suite_test.go │ ├── config.go │ ├── grpc.go │ ├── grpc_test.go │ ├── iface.go │ ├── mysql.go │ └── mysql_test.go ├── dcp_suite_test.go ├── local_server.go └── local_server_test.go ├── deploy └── grafana │ ├── pipeline-detail.json │ └── pipeline-overview.json ├── docker-compose-gravity-dev.yml ├── docker-compose-gravity-test.yml ├── docs ├── 2.0 │ ├── 00-arch-en.md │ ├── 00-arch.md │ ├── 01-quick-start-en.md │ ├── 01-quick-start.md │ ├── 02-config-index-en.md │ ├── 02-config-index.md │ ├── 03-inputs-en.md │ ├── 03-inputs.md │ ├── 04-outputs-en.md │ ├── 04-outputs.md │ ├── 05-filters-en.md │ ├── 05-filters.md │ ├── 06-scheduler-en.md │ ├── 06-scheduler.md │ ├── 07-matcher-en.md │ ├── example-mongo2kafka.toml │ ├── example-mysql2esmodel.toml │ ├── example-mysql2kafka.toml │ ├── example-mysql2mongo.toml │ ├── example-mysql2mysql-full.toml │ ├── example-mysql2mysql-inc.toml │ ├── k8s-160.png │ ├── product.png │ ├── single-process-160.png │ └── single-process.png ├── rfc │ └── rfc_schema_storage.md ├── rfc_schema_storage.md └── wip_developer_guide.md ├── entrypoint.sh ├── go.mod ├── go.sum ├── hooks └── pre-commit ├── integration_test ├── mongokafka │ ├── batch_test.go │ ├── main_test.go │ ├── replication_test.go │ └── stream_test.go ├── mongomysql │ ├── main_test.go │ └── replication_test.go ├── mysql_mysql_test.go └── mysqlelasticsearch │ ├── main_test.go │ └── replication_test.go ├── mock ├── binlog_checker │ └── mock.go ├── position_store │ └── mock.go └── sliding_window │ └── mock.go ├── mycnf └── mysql.cnf ├── padder ├── config │ ├── config.go │ ├── config_suite_test.go │ └── config_test.go ├── config_example │ └── padder.toml ├── job_processor │ ├── job.go │ ├── job_processor.go │ ├── job_processor_suite_test.go │ ├── job_processor_test.go │ ├── mysql_preview_worker.go │ └── mysql_worker.go ├── padder.go ├── padder_suite_test.go └── padder_test.go ├── pkg ├── app │ ├── server.go │ └── server_test.go ├── config │ ├── config.go │ ├── config_v2.go │ ├── config_v2_test.go │ ├── config_v3.go │ ├── kafka.go │ ├── limits.go │ ├── mysql.go │ └── table.go ├── consts │ └── gravity.go ├── core │ ├── emitter.go │ ├── encoding │ │ ├── encoding.go │ │ ├── mongo_json.go │ │ ├── pb.go │ │ ├── pb_test.go │ │ ├── rdb_json.go │ │ └── rdb_json_test.go │ ├── filter.go │ ├── input.go │ ├── matcher.go │ ├── msg.go │ ├── output.go │ ├── position_cache_creator.go │ ├── router.go │ └── scheduler.go ├── elasticsearch_test │ └── test.go ├── emitter │ ├── emitter.go │ └── emitter_test.go ├── env │ └── env.go ├── filters │ ├── accept_filter.go │ ├── accept_filter_test.go │ ├── base_filter.go │ ├── delete_dml_column_filter.go │ ├── delete_dml_column_filter_test.go │ ├── filters.go │ ├── grpc │ │ ├── client.go │ │ ├── server.go │ │ └── shared.go │ ├── grpc_sidecar_filter.go │ ├── grpc_sidecar_filter_test.go │ ├── reject_filter.go │ ├── reject_filter_test.go │ ├── rename_dml_column_filter.go │ ├── rename_dml_column_filter_test.go │ └── utils.go ├── inputs │ ├── dump_input.go │ ├── helper │ │ ├── binlog_checker │ │ │ ├── checker.go │ │ │ └── checker_test.go │ │ ├── mysql_common.go │ │ ├── mysql_common_test.go │ │ ├── two_stage_input.go │ │ ├── two_stage_input_test.go │ │ └── util.go │ ├── mongo │ │ └── input.go │ ├── mongobatch │ │ ├── input.go │ │ ├── input_test.go │ │ └── position_value.go │ ├── mongostream │ │ ├── input.go │ │ ├── oplog_checker.go │ │ ├── oplog_tailer.go │ │ ├── position_value.go │ │ └── position_value_test.go │ ├── mysql │ │ └── input.go │ ├── mysqlbatch │ │ ├── input.go │ │ ├── input_test.go │ │ ├── msg.go │ │ ├── mysql_table_scanner.go │ │ ├── mysql_table_scanner_test.go │ │ ├── position_value.go │ │ ├── position_value_test.go │ │ └── utils.go │ ├── mysqlstream │ │ ├── binlog_tailer.go │ │ ├── binlog_tailer_test.go │ │ ├── input.go │ │ ├── input_test.go │ │ ├── msg.go │ │ ├── msg_test.go │ │ ├── position_value.go │ │ ├── position_value_test.go │ │ └── utils.go │ ├── plugins.go │ └── tidb_kafka │ │ ├── binlog_tailer.go │ │ ├── input.go │ │ ├── position_value.go │ │ ├── position_value_test.go │ │ └── utils.go ├── kafka │ └── kafka.go ├── kafka_test │ └── test.go ├── logutil │ └── log.go ├── matchers │ ├── ddl_regex_matcher.go │ ├── ddl_regex_matcher_test.go │ ├── dml_operator_matcher.go │ ├── matchers.go │ ├── matchers_test.go │ ├── schema_matcher.go │ ├── table_matcher.go │ ├── table_matcher_test.go │ ├── table_regex_matcher.go │ └── table_regex_matcher_test.go ├── metrics │ └── metrics.go ├── mongo │ └── gtm │ │ ├── .gitignore │ │ ├── README.md │ │ ├── consistent │ │ └── consistent.go │ │ └── gtm.go ├── mongo_test │ └── test.go ├── mysql │ └── utils.go ├── mysql_test │ ├── data_generator.go │ ├── generator.go │ ├── test.go │ └── types │ │ ├── date_time.go │ │ ├── numeric.go │ │ ├── string.go │ │ ├── types.go │ │ └── wrapper.go ├── offsets │ ├── db.go │ └── offsets.go ├── outputs │ ├── async_kafka │ │ └── async_kafka.go │ ├── dump_output.go │ ├── elasticsearch │ │ ├── elasticsearch.go │ │ ├── elasticsearch_test.go │ │ ├── helper.go │ │ └── helper_test.go │ ├── esmodel │ │ ├── esmodel.go │ │ ├── esmodel_test.go │ │ ├── helper.go │ │ └── helper_test.go │ ├── mysql │ │ ├── add_missing_column.go │ │ ├── add_missing_column_test.go │ │ ├── cast_default_data.go │ │ ├── cast_default_data_test.go │ │ ├── mysql.go │ │ └── mysql_test.go │ ├── plugins.go │ ├── routers │ │ ├── elasticsearch_router.go │ │ ├── esmodel_router.go │ │ ├── esmodel_router_test.go │ │ ├── kafka_router.go │ │ ├── mysql_router.go │ │ ├── mysql_router_test.go │ │ ├── routers.go │ │ ├── utils.go │ │ └── utils_test.go │ └── stdout │ │ └── stdout.go ├── position_cache │ ├── cache.go │ └── cache_test.go ├── position_repos │ ├── mem_repo.go │ ├── mongo_repo.go │ ├── mongo_repo_test.go │ ├── mysql_repo.go │ ├── mysql_repo_test.go │ └── position_repos.go ├── protocol │ ├── dcp │ │ └── message.pb.go │ ├── meta.go │ ├── msgpb │ │ └── message.pb.go │ ├── protocol_suite_test.go │ └── tidb │ │ └── tidb.pb.go ├── registry │ ├── go_plugin_getter.go │ ├── go_plugin_getter_test.go │ ├── registry.go │ └── test_data │ │ ├── Makefile │ │ ├── dump_filter_plugin.darwin.so │ │ ├── dump_filter_plugin.go │ │ └── dump_filter_plugin.linux.so ├── sarama_cluster │ ├── LICENSE │ ├── README.md │ ├── balancer.go │ ├── client.go │ ├── cluster.go │ ├── config.go │ ├── consumer.go │ ├── doc.go │ ├── offsets.go │ ├── partitions.go │ └── util.go ├── schedulers │ ├── batch_table_scheduler │ │ ├── batch_table_scheduler.go │ │ ├── batch_table_scheduler_test.go │ │ ├── working_set.go │ │ └── working_set_test.go │ └── schedulers.go ├── schema_store │ ├── schema_store.go │ ├── schema_store_suite_test.go │ ├── simple_schema_store.go │ ├── simple_schema_store_test.go │ ├── utils.go │ └── utils_test.go ├── sliding_window │ ├── heap.go │ ├── sequence.go │ ├── sliding_window.go │ ├── sliding_window_suite_test.go │ ├── static_sliding_window.go │ └── static_sliding_window_test.go ├── sql_execution_engine │ ├── conflict_engine.go │ ├── conflict_engine_test.go │ ├── conflict_preview_engine.go │ ├── conflict_preview_engine_test.go │ ├── internal_txn_tagger.go │ ├── manual_engine.go │ ├── manual_engine_test.go │ ├── mysql_insert_ignore_engine.go │ ├── mysql_insert_ignore_engine_test.go │ ├── mysql_insert_on_dupkey_update_engine.go │ ├── mysql_replace_engine.go │ ├── mysql_replace_engine_test.go │ ├── sql_execution_engine.go │ ├── utils.go │ └── utils_test.go └── utils │ ├── db.go │ ├── db_test.go │ ├── etcd.go │ ├── getter.go │ ├── glob.go │ ├── glob_test.go │ ├── hash.go │ ├── labels.go │ ├── labels_test.go │ ├── life_cycle.go │ ├── mongo.go │ ├── mongo_test.go │ ├── printer.go │ ├── ptrs.go │ ├── rdb_internal_txn_tag.go │ ├── retry │ └── retry.go │ ├── struct.go │ ├── testing.go │ ├── type_cast.go │ └── utils_suite_test.go ├── protocol ├── dcp │ └── message.proto ├── msgpb │ └── message.proto └── tidb │ └── tidb.proto └── wait-for-it.sh /.dev.env: -------------------------------------------------------------------------------- 1 | export SOURCE_DB_HOST="127.0.0.1" 2 | export SOURCE_DB_USER="root" 3 | export SOURCE_DB_PORT=3306 4 | export SOURCE_DB_URL="$SOURCE_DB_USER:@tcp($SOURCE_DB_HOST:$SOURCE_DB_PORT)/?loc=Local&interpolateParams=true&readTimeout=30s&parseTime=true&collation=utf8mb4_general_ci" 5 | export TARGET_DB_HOST="127.0.0.1" 6 | export TARGET_DB_USER="root" 7 | export TARGET_DB_PORT=3306 8 | export TARGET_DB_URL="$TARGET_DB_USER:@tcp($TARGET_DB_HOST:$TARGET_DB_PORT)/?loc=Local&interpolateParams=true&readTimeout=30s&parseTime=true&collation=utf8mb4_general_ci" 9 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | DB_HOST=source-db 2 | DB_PORT=3306 3 | DB_USER=root -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .vscode 3 | *.out 4 | *.prof 5 | /bin 6 | */*.iml 7 | cmd/scratch 8 | profile 9 | *.idea 10 | *.meta 11 | 12 | 13 | *_dev.toml 14 | *_dev.json 15 | 16 | config_dev 17 | 18 | 19 | 20 | *.log 21 | .schema-store 22 | *.coverprofile 23 | 24 | *.retry 25 | 26 | test.sh 27 | test.sql 28 | tmp.py 29 | 30 | gravity.timestamp.toml 31 | gravity.timestamp.toml.backup 32 | 33 | nuclear/configdata/dev.toml 34 | nuclear/configdata/mysql-pb.toml 35 | 36 | etc/scripts/mongo_bike/mongo2mysql_* 37 | 38 | scripts/set_position.py 39 | 40 | default.etcd 41 | 42 | gravity_mongo.meta.toml 43 | 44 | private_plugins 45 | grpc-sidecar -------------------------------------------------------------------------------- /.gometalinter.json: -------------------------------------------------------------------------------- 1 | { 2 | "Enable": ["vet", "staticcheck", "errcheck", "gas", "safesql", "golint", "gosimple", "unconvert", "interfacer", "unused", "ineffassign"] 3 | } 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.13.3" 5 | 6 | install: true 7 | 8 | services: 9 | - docker 10 | 11 | jobs: 12 | include: 13 | - stage: "test" 14 | name: "build" 15 | script: 16 | - make build-linux 17 | - docker build -t moiot/gravity:${TRAVIS_COMMIT::8} -f Dockerfile.gravity . 18 | - name: "test" 19 | script: make test 20 | after_script: make test-down 21 | - stage: "publish" 22 | if: tag IS present 23 | script: 24 | - make build-linux 25 | - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin 26 | - docker build -t moiot/gravity:${TRAVIS_TAG} -f Dockerfile.gravity . 27 | - docker push moiot/gravity:${TRAVIS_TAG} 28 | - docker tag moiot/gravity:${TRAVIS_TAG} moiot/gravity:latest 29 | - docker push moiot/gravity:latest -------------------------------------------------------------------------------- /Dockerfile.gravity: -------------------------------------------------------------------------------- 1 | FROM frolvlad/alpine-glibc 2 | 3 | RUN apk update && apk upgrade && apk add bash && apk add tzdata 4 | 5 | RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo "Asia/Shanghai" > /etc/timezone 6 | 7 | WORKDIR / 8 | 9 | COPY bin/gravity-linux-amd64 /gravity 10 | 11 | CMD /gravity -config=/etc/gravity/config.toml -------------------------------------------------------------------------------- /Dockerfile.gravity.race: -------------------------------------------------------------------------------- 1 | FROM frolvlad/alpine-glibc 2 | 3 | RUN apk update && apk upgrade && apk add bash && apk add tzdata 4 | 5 | RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo "Asia/Shanghai" > /etc/timezone 6 | 7 | WORKDIR / 8 | 9 | COPY bin/gravity-race-linux-amd64 /gravity-race -------------------------------------------------------------------------------- /Dockerfile.mongo.setup: -------------------------------------------------------------------------------- 1 | FROM mongo:4.1 2 | 3 | RUN echo "rs.initiate();" > /docker-entrypoint-initdb.d/replica-init.js 4 | -------------------------------------------------------------------------------- /Dockerfile.prometheus_sd: -------------------------------------------------------------------------------- 1 | FROM frolvlad/alpine-glibc 2 | 3 | COPY alpine.repositories /etc/apk/repositories 4 | 5 | RUN apk update && apk upgrade && apk add bash && apk add tzdata 6 | 7 | RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime 8 | 9 | RUN echo "Asia/Shanghai" > /etc/timezone 10 | 11 | COPY bin/prometheus_sd-linux-amd64 /prometheus_sd 12 | 13 | COPY wait-for-it.sh /wait-for-it.sh 14 | 15 | WORKDIR / -------------------------------------------------------------------------------- /Dockerfile.test.gravity: -------------------------------------------------------------------------------- 1 | FROM golang:1.11.4 2 | 3 | WORKDIR /gravity 4 | 5 | COPY . /gravity 6 | 7 | CMD ["make go-test"] 8 | -------------------------------------------------------------------------------- /README-cn.md: -------------------------------------------------------------------------------- 1 | 2 | # Gravity 3 | ------------------------------------ 4 | 5 | [![Build Status](https://travis-ci.org/moiot/gravity.svg?branch=master)](https://travis-ci.org/moiot/gravity.svg?branch=master) 6 | 7 | ![2.0 Product](docs/2.0/product.png) 8 | 9 | 10 | Gravity 是一款数据复制组件，提供全量、增量数据同步，以及向消息队列发布数据更新。 11 | 12 | DRC 的设计目标是： 13 | - 支持多种数据源和目标的，可灵活定制的数据复制组件 14 | - 支持基于 Kubernetes 的 PaaS 平台，简化运维任务 15 | 16 | 17 | ### 使用场景 18 | 19 | - 大数据总线：发送 MySQL Binlog，Mongo Oplog 的数据变更到 kafka 供下游消费 20 | - 单向数据同步：MySQL --> MySQL 的全量、增量同步 21 | - 双向数据同步：MySQL <--> MySQL 的双向增量同步，同步过程中可以防止循环复制 22 | - 分库分表到合库的同步：MySQL 分库分表 --> 合库的同步，可以指定源表和目标表的对应关系 23 | - 在线数据变换：同步过程中，可支持对进行数据变换 24 | 25 | ### 功能列表 26 | 27 | - 数据源 28 | 29 | | | 是否支持 | 30 | |---|---| 31 | | MySQL Binlog | ✅ | 32 | | MySQL 全量 | ✅ | 33 | | Mongo Oplog | ✅ | 34 | | TiDB Binlog | 开发中 | 35 | | PostgreSQL WAL | 开发中 | 36 | 37 | - 数据输出 38 | 39 | | | 是否支持 | 40 | |---|---| 41 | | Kafka | ✅ | 42 | | MySQL/TiDB | ✅ | 43 | | Mongo DB | 开发中 | 44 | 45 | 46 | - 数据变换 47 | 48 | | | 是否支持 | 49 | |---|---| 50 | | 数据过滤 | ✅ | 51 | | 重命令列 | ✅ | 52 | | 删除列|✅| 53 | 54 | 55 | ### 文档 56 | 57 | [架构简介](docs/2.0/00-arch.md) 58 | 59 | [快速上手](docs/2.0/01-quick-start.md) 60 | 61 | [配置手册](docs/2.0/02-config-index.md) 62 | 63 | [集群部署](https://github.com/moiot/gravity-operator) 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # **Gravity** [简体中文](./README-cn.md) 2 | ------------------------- 3 | [![Build Status](https://travis-ci.org/moiot/gravity.svg?branch=master)](https://travis-ci.org/moiot/gravity) 4 | 5 | ![2.0 Product](docs/2.0/product.png) 6 | 7 | **Gravity** is used to replicate data between various inputs and outputs (databases, message queues). 8 | 9 | It is designed to be a customizable data replication tool that: 10 | 11 | - Supports multiple sources and destinations 12 | 13 | - Supports Kubernetes-based replication cluster 14 | 15 | ## Use Cases 16 | 17 | - Data Bus: Use Change Data Capture (MySQL binlog, MongoDB Oplog) and batch table scan to publish data to message queue like Kafka. 18 | - Unidirectional data replication: Replicates data from one MySQL cluster to another MySQL cluster. 19 | - Bidirectional data replication: Replicates data between two MySQL clusters bidirectionally. 20 | - Synchronization of shards to the merged table: Synchronizes MySQL sharded tables to the merged table. You can specify the corresponding relationship between the source table and the target table. 21 | - Online data mutation: Data can be changed during the replication. For example, rename the column, encrypt/decrypt data columns. 22 | ## Features 23 | 24 | ### Input support 25 | 26 | | Input | Status | 27 | |---|---| 28 | | MySQL Binlog | ✅ | 29 | | MySQL Scan | ✅ | 30 | | Mongo Oplog | ✅ | 31 | | Mongo Scan | ✅ | 32 | | TiDB Binlog | Doing | 33 | | PostgreSQL WAL | Doing | 34 | 35 | 36 | ### output support 37 | 38 | 39 | | Output | Status | 40 | |---|---| 41 | | MySQL/TiDB | ✅ | 42 | | Kafka | ✅ | 43 | | Elasticsearch | Beta | 44 | 45 | 46 | ### Data mutation support 47 | 48 | **Gravity** supports the following data mutations: 49 | 50 | - Ignore specific data 51 | - Renaming columns 52 | - Deleting columns 53 | 54 | ### Documentation 55 | 56 | - [Architecture](docs/2.0/00-arch-en.md) 57 | - [Quick Start](docs/2.0/01-quick-start-en.md) 58 | - [Configuration](docs/2.0/02-config-index-en.md) 59 | - [Cluster Deployment](https://github.com/moiot/gravity-operator) 60 | 61 | ----------- 62 | 63 | Special thanks to [@dantin](https://github.com/dantin), [@LiangShang](https://github.com/liangshang), and [@liwink](https://github.com/liwink) for the early support to this project, and thanks [@CaitinChen](https://github.com/CaitinChen) for the initial translation of the documentation. -------------------------------------------------------------------------------- /cmd/dcp/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | 9 | "github.com/moiot/gravity/dcp" 10 | "github.com/moiot/gravity/dcp/barrier" 11 | "github.com/moiot/gravity/dcp/checker" 12 | "github.com/moiot/gravity/dcp/collector" 13 | "github.com/moiot/gravity/pkg/utils" 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | func main() { 18 | dbConfig := utils.TestConfig() 19 | 20 | barrierConfig := barrier.Config{ 21 | Db: *dbConfig, 22 | TickerSeconds: 2, 23 | } 24 | 25 | collectorConfigs := []collector.Config{ 26 | &collector.MysqlConfig{ 27 | Db: collector.DbConfig{ 28 | Name: "db", 29 | Host: dbConfig.Host, 30 | Port: uint(dbConfig.Port), 31 | Username: dbConfig.Username, 32 | Password: dbConfig.Password, 33 | ServerId: 999, 34 | }, 35 | 36 | TagConfigs: []collector.TagConfig{ 37 | { 38 | Tag: "src", 39 | Tables: []collector.SchemaAndTable{ 40 | { 41 | 42 | Schema: "drc", 43 | Table: "src", 44 | PrimaryKeyIdx: 0, 45 | }, 46 | }, 47 | }, 48 | { 49 | Tag: "target", 50 | Tables: []collector.SchemaAndTable{ 51 | { 52 | 53 | Schema: "drc", 54 | Table: "target", 55 | PrimaryKeyIdx: 0, 56 | }, 57 | }, 58 | }, 59 | }, 60 | }, 61 | } 62 | 63 | checkerConfig := checker.Config{ 64 | SourceTag: "src", 65 | TargetTags: []string{"target"}, 66 | TimeoutSeconds: 2, 67 | } 68 | 69 | shutDown := make(chan struct{}) 70 | alarm := make(chan checker.Result, 10) 71 | closed := make(chan error) 72 | 73 | go func() { 74 | closed <- dcp.StartLocal(&barrierConfig, collectorConfigs, &checkerConfig, shutDown, alarm) 75 | }() 76 | 77 | sc := make(chan os.Signal, 1) 78 | signal.Notify(sc, 79 | syscall.SIGHUP, 80 | syscall.SIGINT, 81 | syscall.SIGTERM, 82 | syscall.SIGQUIT) 83 | 84 | loop: 85 | for { 86 | select { 87 | case a := <-alarm: 88 | fmt.Printf("%T\n", a) 89 | 90 | case sig := <-sc: 91 | log.Info(sig) 92 | close(shutDown) 93 | 94 | case err := <-closed: 95 | if err != nil { 96 | log.Error(err) 97 | } 98 | break loop 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /cmd/padder/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // main is the bootstrap. 4 | //func main() { 5 | // cfg := config.NewConfig() 6 | // err := cfg.ParseCmd(os.Args[1:]) 7 | // switch errors.Cause(err) { 8 | // case nil: 9 | // case flag.ErrHelp: 10 | // os.Exit(0) 11 | // default: 12 | // log.Fatalf("parse cmd flags errors: %s\n", err) 13 | // } 14 | // 15 | // if cfg.ConfigFile == "" { 16 | // log.Fatalf("config file is required") 17 | // } 18 | // 19 | // if err := cfg.CreateConfigFromFile(cfg.ConfigFile); err != nil { 20 | // log.Fatalf("failed to load config from file. %v", err) 21 | // } 22 | // if err = config.Validate(cfg.PadderConfig); err != nil { 23 | // log.Fatalf("config validation failed: %v", err) 24 | // } 25 | // logutil.MustInitLogger(&cfg.Log) 26 | // utils.LogRawInfo("padder") 27 | // if cfg.PreviewMode { 28 | // stats, err := padder.Preview(cfg.PadderConfig) 29 | // if err != nil { 30 | // log.Fatalf("pad preview bin log failed: %v", err) 31 | // } 32 | // statsJson, err := json.MarshalIndent(stats, "", " ") 33 | // if err != nil { 34 | // log.Fatalf("parse json failed: %v", err) 35 | // } 36 | // err = ioutil.WriteFile("stats.json", statsJson, 0644) 37 | // if err != nil { 38 | // log.Fatalf("export preview statistic failed: %v", err) 39 | // } 40 | // } else { 41 | // if err := padder.Pad(cfg.PadderConfig); err != nil { 42 | // log.Fatalf("pad bin log failed: %v", err) 43 | // } 44 | // } 45 | //} 46 | -------------------------------------------------------------------------------- /dcp/barrier/barrier_suite_test.go: -------------------------------------------------------------------------------- 1 | package barrier_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | 9 | log "github.com/sirupsen/logrus" 10 | 11 | "github.com/moiot/gravity/pkg/logutil" 12 | ) 13 | 14 | func TestBarrier(t *testing.T) { 15 | logutil.SetLogLevelFromEnv() 16 | log.SetOutput(GinkgoWriter) 17 | RegisterFailHandler(Fail) 18 | RunSpecs(t, "Barrier Suite") 19 | } 20 | -------------------------------------------------------------------------------- /dcp/barrier/barrier_test.go: -------------------------------------------------------------------------------- 1 | package barrier_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "time" 8 | 9 | "gopkg.in/DATA-DOG/go-sqlmock.v1" 10 | 11 | "github.com/moiot/gravity/dcp/barrier" 12 | ) 13 | 14 | var _ = Describe("Barrier", func() { 15 | It("should init meta table and periodically update offset", func() { 16 | db, mock, err := sqlmock.New() 17 | Expect(err).ShouldNot(HaveOccurred()) 18 | defer db.Close() 19 | 20 | mock.ExpectExec("CREATE database if not EXISTS drc").WillReturnResult(sqlmock.NewResult(1, 1)) 21 | mock.ExpectExec("CREATE TABLE IF NOT EXISTS drc.barrier").WillReturnResult(sqlmock.NewResult(1, 1)) 22 | mock.ExpectExec("INSERT IGNORE").WillReturnResult(sqlmock.NewResult(1, 0)) 23 | mock.ExpectExec("UPDATE").WillReturnResult(sqlmock.NewResult(0, 1)) 24 | mock.ExpectExec("UPDATE").WillReturnResult(sqlmock.NewResult(1, 1)) 25 | 26 | shutdown := make(chan struct{}) 27 | 28 | barrier.Start(&barrier.Config{ 29 | TestDB: db, 30 | TickerSeconds: 1, 31 | }, shutdown) 32 | 33 | time.Sleep(time.Millisecond * 1500) 34 | 35 | close(shutdown) 36 | 37 | Expect(mock.ExpectationsWereMet()).Should(BeNil()) 38 | }) 39 | }) 40 | -------------------------------------------------------------------------------- /dcp/checker/alarm.go: -------------------------------------------------------------------------------- 1 | package checker 2 | 3 | import log "github.com/sirupsen/logrus" 4 | 5 | type AlarmManager interface { 6 | Alarm(r Result) 7 | } 8 | 9 | type ConsoleAlarmManager struct { 10 | } 11 | 12 | func (ConsoleAlarmManager) Alarm(r Result) { 13 | log.Infof("result: %+v", r) 14 | } 15 | 16 | type ChanAlarmManager struct { 17 | Output chan Result 18 | } 19 | 20 | func (c ChanAlarmManager) Alarm(r Result) { 21 | c.Output <- r 22 | } 23 | -------------------------------------------------------------------------------- /dcp/checker/buffer.go: -------------------------------------------------------------------------------- 1 | package checker 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | "github.com/moiot/gravity/pkg/protocol/dcp" 6 | "github.com/sirupsen/logrus" 7 | ) 8 | 9 | type buffer struct { 10 | log *logrus.Entry 11 | tag string 12 | buffer []*dcp.Message 13 | inputChan chan *dcp.Message 14 | OutputChan chan *segment 15 | } 16 | 17 | func newBuffer(tag string) *buffer { 18 | buffer := &buffer{ 19 | logrus.WithField("buffer", tag), 20 | tag, 21 | make([]*dcp.Message, 0, 128), 22 | make(chan *dcp.Message), 23 | make(chan *segment), 24 | } 25 | go buffer.handler() 26 | return buffer 27 | } 28 | 29 | func (b *buffer) Write(msg *dcp.Message) { 30 | if msg.Tag != b.tag { 31 | panic(errors.Errorf("unmatched tag, expected %s, actual %s", b.tag, msg.Tag)) 32 | } 33 | b.inputChan <- msg 34 | } 35 | 36 | func (b *buffer) Close() { 37 | b.log.Info("closing buffer") 38 | close(b.inputChan) 39 | close(b.OutputChan) 40 | } 41 | 42 | func (b *buffer) handler() { 43 | for msg := range b.inputChan { 44 | b.log.Debugf("buffer handle message %v, buffer %v", msg, b.buffer) 45 | barrier, isBarrier := msg.Body.(*dcp.Message_Barrier) 46 | if len(b.buffer) == 0 && !isBarrier { 47 | panic(errors.Errorf("buffer[%s] is empty, expect barrier message but actual %s", b.tag, msg)) 48 | } 49 | 50 | if len(b.buffer) > 0 && isBarrier { 51 | if barrier.Barrier != b.buffer[0].GetBarrier()+1 { 52 | b.log.Panicf("jumping barrier, previous %d, current %d", b.buffer[0].GetBarrier(), barrier.Barrier) 53 | } 54 | b.OutputChan <- newSegment(b.tag, b.buffer[0].GetBarrier(), barrier.Barrier, b.buffer[1:]) 55 | b.buffer = b.buffer[:0] 56 | } 57 | 58 | b.buffer = append(b.buffer, msg) 59 | } 60 | 61 | b.log.Info("exit buffer handler") 62 | } 63 | -------------------------------------------------------------------------------- /dcp/checker/checker_suite_test.go: -------------------------------------------------------------------------------- 1 | package checker_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "net/http" 7 | _ "net/http/pprof" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | log "github.com/sirupsen/logrus" 12 | 13 | "github.com/moiot/gravity/pkg/logutil" 14 | ) 15 | 16 | func TestChecker(t *testing.T) { 17 | logutil.SetLogLevelFromEnv() 18 | log.SetOutput(GinkgoWriter) 19 | 20 | go http.ListenAndServe("0.0.0.0:8000", nil) 21 | 22 | RegisterFailHandler(Fail) 23 | RunSpecs(t, "Checker Suite") 24 | } 25 | -------------------------------------------------------------------------------- /dcp/checker/result.go: -------------------------------------------------------------------------------- 1 | package checker 2 | 3 | type Result interface { 4 | needAlert() bool 5 | } 6 | 7 | type Diff struct { 8 | sourceSeg *segment 9 | targetSeg *segment 10 | } 11 | 12 | func (Diff) needAlert() bool { 13 | return true 14 | } 15 | 16 | type Same struct { 17 | sourceSeg *segment 18 | targetSeg *segment 19 | } 20 | 21 | func (Same) needAlert() bool { 22 | return false 23 | } 24 | 25 | type Timeout struct { 26 | sourceSeg *segment 27 | targetTag string 28 | } 29 | 30 | func (Timeout) needAlert() bool { 31 | return true 32 | } 33 | 34 | // Ready is just for test 35 | type Ready struct { 36 | } 37 | 38 | func (Ready) needAlert() bool { 39 | return false 40 | } 41 | -------------------------------------------------------------------------------- /dcp/checker/segment.go: -------------------------------------------------------------------------------- 1 | package checker 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/juju/errors" 7 | "github.com/moiot/gravity/pkg/protocol/dcp" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | type segment struct { 12 | tag string 13 | startOffset uint64 14 | endOffset uint64 15 | messages map[string][]*dcp.Message //key is payload id 16 | } 17 | 18 | func newSegment(tag string, startOffset uint64, endOffset uint64, msgs []*dcp.Message) *segment { 19 | t := make(map[string][]*dcp.Message) 20 | 21 | for _, m := range msgs { 22 | payloadId := m.GetPayload().Id 23 | _, exists := t[payloadId] 24 | if !exists { 25 | t[payloadId] = make([]*dcp.Message, 0, 1) 26 | } 27 | t[payloadId] = append(t[payloadId], m) 28 | } 29 | 30 | for _, v := range t { 31 | sort.Slice(v, func(i, j int) bool { 32 | if v[i].Timestamp == v[j].Timestamp { 33 | return v[i].Id < v[j].Id 34 | } 35 | return v[i].Timestamp < v[j].Timestamp 36 | }) 37 | } 38 | 39 | s := segment{ 40 | tag, 41 | startOffset, 42 | endOffset, 43 | t, 44 | } 45 | 46 | log.Infof("new segment, tag=%s, startOffset=%d, msg cnt=%d", s.tag, s.startOffset, len(msgs)) 47 | 48 | return &s 49 | } 50 | 51 | func (s *segment) equal(s2 *segment) bool { 52 | if s == s2 || s.tag == s2.tag || s.startOffset != s2.startOffset || s.endOffset != s2.endOffset { 53 | panic(errors.Errorf("can't compare segment %+v with %+v", s, s2)) 54 | } 55 | 56 | if len(s.messages) != len(s2.messages) { 57 | return false 58 | } 59 | 60 | for id, m1 := range s.messages { 61 | m2, ok := s2.messages[id] 62 | if !ok || len(m1) != len(m2) { 63 | log.Error("length not equal ", m1, m2) 64 | return false 65 | } 66 | 67 | for i := 0; i < len(m1); i++ { 68 | if m1[i].GetChecksum() != m2[i].GetChecksum() { 69 | log.Error("checksum not equal, content: ", *m1[i].GetPayload(), *m2[i].GetPayload()) 70 | return false 71 | } 72 | } 73 | } 74 | 75 | return true 76 | } 77 | -------------------------------------------------------------------------------- /dcp/collector/collector_suite_test.go: -------------------------------------------------------------------------------- 1 | package collector_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | 9 | log "github.com/sirupsen/logrus" 10 | 11 | "github.com/moiot/gravity/pkg/logutil" 12 | ) 13 | 14 | func TestCollector(t *testing.T) { 15 | logutil.SetLogLevelFromEnv() 16 | log.SetOutput(GinkgoWriter) 17 | RegisterFailHandler(Fail) 18 | RunSpecs(t, "Collector Suite") 19 | } 20 | -------------------------------------------------------------------------------- /dcp/collector/config.go: -------------------------------------------------------------------------------- 1 | package collector 2 | 3 | type Config interface { 4 | foo() 5 | } 6 | 7 | type MysqlConfig struct { 8 | Db DbConfig 9 | TagConfigs []TagConfig 10 | } 11 | 12 | func (MysqlConfig) foo() { 13 | } 14 | 15 | type DbConfig struct { 16 | Name string `toml:"name" json:"name"` 17 | Host string `toml:"host" json:"host"` 18 | Username string `toml:"username" json:"username"` 19 | Password string `toml:"password" json:"password"` 20 | Port uint `toml:"port" json:"port"` 21 | ServerId uint32 22 | } 23 | 24 | type TagConfig struct { 25 | Tag string 26 | Tables []SchemaAndTable 27 | } 28 | 29 | type SchemaAndTable struct { 30 | Schema string 31 | Table string 32 | PrimaryKeyIdx int 33 | } 34 | 35 | type GrpcConfig struct { 36 | Port int 37 | } 38 | 39 | func (GrpcConfig) foo() { 40 | } 41 | -------------------------------------------------------------------------------- /dcp/collector/grpc.go: -------------------------------------------------------------------------------- 1 | package collector 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/moiot/gravity/pkg/protocol/dcp" 7 | 8 | "fmt" 9 | "net" 10 | 11 | log "github.com/sirupsen/logrus" 12 | "google.golang.org/grpc" 13 | ) 14 | 15 | type Grpc struct { 16 | port int 17 | server *grpc.Server 18 | output chan *dcp.Message 19 | } 20 | 21 | func NewGrpc(config *GrpcConfig) Interface { 22 | s := &Grpc{ 23 | port: config.Port, 24 | server: grpc.NewServer(), 25 | output: make(chan *dcp.Message, 100), 26 | } 27 | dcp.RegisterDCPServiceServer(s.server, s) 28 | return s 29 | } 30 | 31 | func (s *Grpc) Start() { 32 | lis, err := net.Listen("tcp", fmt.Sprintf(":%d", s.port)) 33 | if err != nil { 34 | log.Fatalf("failed to listen: %v", err) 35 | } 36 | go s.server.Serve(lis) 37 | } 38 | 39 | func (s *Grpc) Stop() { 40 | s.server.Stop() 41 | close(s.output) 42 | log.Info("collector.Grpc stopped") 43 | } 44 | 45 | func (s *Grpc) GetChan() chan *dcp.Message { 46 | return s.output 47 | } 48 | 49 | func (s *Grpc) Process(stream dcp.DCPService_ProcessServer) error { 50 | for { 51 | msg, err := stream.Recv() 52 | if err == io.EOF { 53 | log.Info("collector.Grpc receive EOF") 54 | return nil 55 | } 56 | 57 | if err != nil { 58 | log.Error("collector.Grpc ", err) 59 | return err 60 | } 61 | 62 | s.output <- msg 63 | 64 | err = stream.Send(&dcp.Response{ 65 | Id: msg.Id, 66 | }) 67 | 68 | if err != nil { 69 | log.Info("collector.Grpc send error ", err) 70 | return err 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /dcp/collector/grpc_test.go: -------------------------------------------------------------------------------- 1 | package collector_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "context" 8 | "fmt" 9 | "time" 10 | 11 | "google.golang.org/grpc" 12 | 13 | . "github.com/moiot/gravity/dcp/collector" 14 | "github.com/moiot/gravity/pkg/protocol/dcp" 15 | ) 16 | 17 | var _ = Describe("Grpc Collector", func() { 18 | 19 | config := GrpcConfig{ 20 | Port: 8888, 21 | } 22 | 23 | XIt("should serve client request", func() { 24 | server := NewGrpc(&config) 25 | server.Start() 26 | 27 | msg := &dcp.Message{Id: "1"} 28 | conn, err := grpc.Dial(fmt.Sprintf("localhost:%d", config.Port), grpc.WithInsecure()) 29 | Expect(err).ShouldNot(HaveOccurred()) 30 | defer conn.Close() 31 | 32 | client := dcp.NewDCPServiceClient(conn) 33 | stream, err := client.Process(context.Background()) 34 | Expect(err).ShouldNot(HaveOccurred()) 35 | 36 | err = stream.Send(msg) 37 | Expect(err).ShouldNot(HaveOccurred()) 38 | Eventually(server.GetChan()).Should(Receive(Equal(msg))) 39 | 40 | resp, err := stream.Recv() 41 | Expect(err).ShouldNot(HaveOccurred()) 42 | 43 | Expect(resp.Id).Should(Equal(msg.Id)) 44 | Expect(resp.Code).Should(Equal(int32(0))) 45 | 46 | go func() { 47 | for { 48 | err = stream.Send(msg) 49 | Expect(err).ShouldNot(HaveOccurred()) 50 | 51 | time.Sleep(time.Millisecond * 5) 52 | } 53 | }() 54 | 55 | server.Stop() 56 | Eventually(server.GetChan()).Should(BeClosed()) 57 | }) 58 | }) 59 | -------------------------------------------------------------------------------- /dcp/collector/iface.go: -------------------------------------------------------------------------------- 1 | package collector 2 | 3 | import "github.com/moiot/gravity/pkg/protocol/dcp" 4 | 5 | type Interface interface { 6 | GetChan() chan *dcp.Message 7 | Start() 8 | Stop() 9 | } 10 | -------------------------------------------------------------------------------- /dcp/dcp_suite_test.go: -------------------------------------------------------------------------------- 1 | package dcp_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | 9 | log "github.com/sirupsen/logrus" 10 | 11 | "github.com/moiot/gravity/pkg/logutil" 12 | ) 13 | 14 | func TestDcp(t *testing.T) { 15 | logutil.SetLogLevelFromEnv() 16 | log.SetOutput(GinkgoWriter) 17 | RegisterFailHandler(Fail) 18 | RunSpecs(t, "Dcp Suite") 19 | } 20 | -------------------------------------------------------------------------------- /dcp/local_server.go: -------------------------------------------------------------------------------- 1 | package dcp 2 | 3 | import ( 4 | _ "github.com/go-sql-driver/mysql" 5 | "github.com/moiot/gravity/dcp/barrier" 6 | "github.com/moiot/gravity/dcp/checker" 7 | "github.com/moiot/gravity/dcp/collector" 8 | "github.com/moiot/gravity/pkg/protocol/dcp" 9 | 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | // StartLocal starts collector, checker & barrier all in one process 14 | func StartLocal(barrierConfig *barrier.Config, collectorConfigs []collector.Config, checkerConfig *checker.Config, shutdown chan struct{}, alarm chan<- checker.Result) error { 15 | barrier.Start(barrierConfig, shutdown) 16 | 17 | collectors := make([]collector.Interface, 0, len(collectorConfigs)) 18 | allMsg := make(chan *dcp.Message, 100) 19 | for _, cfg := range collectorConfigs { 20 | var c collector.Interface 21 | switch cfg := cfg.(type) { 22 | case *collector.MysqlConfig: 23 | c = collector.NewMysqlCollector(cfg) 24 | case *collector.GrpcConfig: 25 | c = collector.NewGrpc(cfg) 26 | } 27 | c.Start() 28 | go func(c <-chan *dcp.Message) { 29 | for m := range c { 30 | allMsg <- m 31 | } 32 | }(c.GetChan()) 33 | collectors = append(collectors, c) 34 | } 35 | 36 | c := checker.New(checkerConfig) 37 | 38 | alarm <- &checker.Ready{} 39 | 40 | loop: 41 | for { 42 | select { 43 | case <-shutdown: 44 | log.Info("local server receive shutdown") 45 | break loop 46 | 47 | case r := <-c.ResultChan: 48 | alarm <- r 49 | 50 | case m := <-allMsg: 51 | c.Consume(m) 52 | } 53 | } 54 | 55 | for _, c := range collectors { 56 | c.Stop() 57 | } 58 | close(allMsg) 59 | c.Shutdown() 60 | 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /docker-compose-gravity-dev.yml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | services: 3 | source-db: 4 | image: mysql 5 | container_name: source-db-dev 6 | environment: 7 | - MYSQL_ALLOW_EMPTY_PASSWORD=yes 8 | ports: 9 | - 3478:3306 10 | volumes: 11 | - ./mycnf:/etc/mysql/conf.d -------------------------------------------------------------------------------- /docker-compose-gravity-test.yml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | services: 3 | source-db: 4 | image: mysql:5.7.18 5 | container_name: source-db-test 6 | environment: 7 | - MYSQL_ALLOW_EMPTY_PASSWORD=yes 8 | command: 9 | # test for binlog rotate case 10 | - --max-binlog-size=409600 11 | logging: 12 | driver: none 13 | volumes: 14 | - ./mycnf:/etc/mysql/conf.d 15 | 16 | target-db: 17 | image: mysql:5.7.18 18 | container_name: target-db-test 19 | environment: 20 | - MYSQL_ALLOW_EMPTY_PASSWORD=yes 21 | logging: 22 | driver: none 23 | volumes: 24 | - ./mycnf:/etc/mysql/conf.d 25 | 26 | mongo: 27 | container_name: mongo 28 | build: 29 | context: ./ 30 | dockerfile: Dockerfile.mongo.setup 31 | ports: 32 | - 27017 33 | logging: 34 | driver: none 35 | command: mongod --replSet rs1 36 | 37 | zookeeper: 38 | image: zookeeper:3.4.10 39 | container_name: zookeeper 40 | logging: 41 | driver: none 42 | 43 | kafka: 44 | image: confluentinc/cp-kafka:5.1.0 45 | container_name: kafka 46 | depends_on: 47 | - zookeeper 48 | environment: 49 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 50 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 51 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 52 | logging: 53 | driver: none 54 | 55 | elasticsearch: 56 | image: docker.elastic.co/elasticsearch/elasticsearch:6.3.2 57 | container_name: elasticsearch 58 | environment: 59 | - http.host=0.0.0.0 60 | - xpack.security.transport.filter.allow=0.0.0.0 61 | - cluster.name=gravity 62 | - "ES_JAVA_OPTS=-Xms750m -Xmx750m" 63 | logging: 64 | driver: none 65 | 66 | gravity-test: 67 | build: 68 | context: ./ 69 | dockerfile: Dockerfile.test.gravity 70 | depends_on: 71 | - mongo 72 | environment: 73 | - MONGO_HOST=mongo 74 | - KAFKA_BROKER=kafka:9092 75 | - ELASTICSEARCH_URLS=http://elasticsearch:9200 76 | command: ["bash", "./wait-for-it.sh", "source-db:3306","-t", "0", 77 | "--", "./wait-for-it.sh", "target-db:3306","-t", "0", 78 | "--", "./wait-for-it.sh", "mongo:27017", "-t", "0", 79 | "--", "./wait-for-it.sh", "kafka:9092", "-t", "0", 80 | "--", "./wait-for-it.sh", "elasticsearch:9200", "-t", "0", 81 | "--", "make", "go-test"] 82 | -------------------------------------------------------------------------------- /docs/2.0/00-arch-en.md: -------------------------------------------------------------------------------- 1 | # Gravity's Architecture 2 | 3 | Gravity can be deployed as a single process, and can also be deployed on Kubernetes as a cluster. 4 | 5 | ## Gravity Single Process architecture 6 | 7 | Gravity is designed with micro-kernel/plugin architecture. The message process pipeline is around the `core.Msg` data structure. 8 | 9 | Each plugin has its own configuration options. 10 | 11 | ![Gravity Single Process architecture](./single-process-160.png) 12 | 13 | Gravity consists of the following types of plugins: 14 | 15 | - **Input** 16 | 17 | It adapts various data sources. For example, it adapts MySQL binlog files and generates `core.Msg`. 18 | 19 | - **Filter** 20 | 21 | It mutates the data flow generated by Input, like filtering some data, renaming some columns and encrypting columns. 22 | 23 | - **Scheduler** 24 | 25 | It schedules the data flow generated by Input and writes the data to the target by Output. It defines the data consistency feature the current system supports ( The default scheduler supports modifying the data in the same row in order by default). 26 | 27 | - **Output** 28 | 29 | It writes data to the target, like Kafka and MySQL. During this process, the routing rules defined by **Router** are used. 30 | 31 | - **Matcher** 32 | 33 | It matches the data generated by Input. **Filter** and **Router** use **Matcher** to match data. 34 | 35 | 36 | You can develop plugins of the above types for your specific requirements. 37 | 38 | The `core.Msg` definition is as follows: 39 | 40 | ```golang 41 | 42 | type DDLMsg struct { 43 | Statement string 44 | } 45 | 46 | type DMLMsg struct { 47 | Operation DMLOp 48 | Data map[string]interface{} 49 | Old map[string]interface{} 50 | Pks map[string]interface{} 51 | PkColumns []string 52 | } 53 | 54 | type Msg struct { 55 | Type MsgType 56 | Host string 57 | Database string 58 | Table string 59 | Timestamp time.Time 60 | 61 | DdlMsg *DDLMsg 62 | DmlMsg *DMLMsg 63 | ... 64 | } 65 | ``` 66 | 67 | ## Gravity Cluster architecture 68 | 69 | Gravity Cluster supports cluster deployment on Kubernetes natively. Please see [here](https://github.com/moiot/gravity-operator). 70 | 71 | It provides Rest API to create the data synchronization task to report the task status. It manages each Gravity task in the Web interface (Gravity Admin). 72 | -------------------------------------------------------------------------------- /docs/2.0/00-arch.md: -------------------------------------------------------------------------------- 1 | ### 架构简介 2 | 3 | **单进程架构** 4 | 5 | 单进程的 Gravity 采用基于插件的微内核架构，由各个插件围绕系统里的 `core.Msg` 结构实现输入到输出的整个流程。 6 | 7 | 各个插件有各自独立的配置选项。 8 | 9 | ![2.0 Arch Image](./single-process-160.png) 10 | 11 | 如上图所示，系统总共由这几个插件组成： 12 | 13 | - **Input** 用来适配各种数据源，比如 MySQL 的 Binlog 并生成 `core.Msg` 14 | 15 | - **Filter** 用来对 Input 所生成的数据流做数据变换操作，比如过滤某些数据，重命名某些列，对列加密 16 | 17 | - **Output** 用来将数据写入目标，比如 Kafka, MySQL，**Output** 写入目标时，使用 **Router** 所定义的路由规则 18 | 19 | - **Scheduler** 用来对 Input 生成的数据流调度，并使用 Output 写入目标；Scheduler 定义了当前系统支持的一致性特性（_当前默认的 Scheduler 支持同一行数据的修改有序_） 20 | 21 | - **Matcher** 用来匹配 Input 生成的数据。**Filter** 和 **Router** 使用 **Matcher** 匹配数据 22 | 23 | 开发人员可以开发以上的几个插件类型，实现特定的需求。 24 | 25 | `core.Msg` 的定义如下 26 | 27 | ```golang 28 | 29 | type DDLMsg struct { 30 | Statement string 31 | } 32 | 33 | type DMLMsg struct { 34 | Operation DMLOp 35 | Data map[string]interface{} 36 | Old map[string]interface{} 37 | Pks map[string]interface{} 38 | PkColumns []string 39 | } 40 | 41 | type Msg struct { 42 | Type MsgType 43 | Host string 44 | Database string 45 | Table string 46 | Timestamp time.Time 47 | 48 | DdlMsg *DDLMsg 49 | DmlMsg *DMLMsg 50 | ... 51 | } 52 | ``` 53 | 54 | **集群架构** 55 | 56 | 集群版本的 Gravity 原生支持 Kubernetes 上的集群部署，请查看[这里](https://github.com/moiot/gravity-operator)。 57 | 58 | 集群版本 Gravity 提供 Rest API 创建创建数据同步任务，汇报状态。自带 Web 界面 (Gravity Admin) 管理各个任务。 59 | 60 | -------------------------------------------------------------------------------- /docs/2.0/02-config-index-en.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | This document introduces how to configure Gravity Single Process and Gravity Cluster respectively. 4 | 5 | ## Gravity Single Process 6 | 7 | To configure Gravity Single Process, use the configuration file. 8 | 9 | Gravity works in the plugin-based mode and each plugin has its own configuration. Currently, Gravity supports the configuration file in the `toml` and `json` formats. 10 | 11 | This section uses the configuration file in the `toml` format to describe configuration rules. 12 | 13 | ## Gravity Cluster 14 | 15 | To configure Gravity Cluster, use Rest API to start the task. Make sure the Rest API and the `json` format of the configuration file are consistent. 16 | 17 | For Gravity Cluster, the Web interface configuration is provided. For details about the options of Rest API, see the configuration file in the `toml` format. 18 | 19 | ## Note 20 | 21 | For the following configurations, the `Input` and `Output` configurations are required: 22 | 23 | - [Input Configuration](03-inputs-en.md) 24 | - [Output Configuration](04-outputs-en.md) 25 | - [Filter Configuration](05-filters-en.md) 26 | - [Scheduler Configuration](06-scheduler-en.md) 27 | - [Matcher Configuration](07-matcher-en.md) -------------------------------------------------------------------------------- /docs/2.0/02-config-index.md: -------------------------------------------------------------------------------- 1 | ### 配置文件 2 | 3 | 单进程的 Gravity 使用配置文件来配置。 4 | 5 | Gravity 是基于插件的微内核模式，各个插件有自己独立的配置。目前 Gravity 支持以 `toml` 格式和 `json` 格式作为配置文件来配置。 6 | 7 | 本节的描述中，为了方便起见，统一使用 `toml` 格式的配置文件描述配置规则。 8 | 9 | 10 | ### Rest API 11 | 12 | 集群方式部署的 Gravity 集群使用 Rest API 来启动任务，Rest API 和配置文件的 `json` 格式保持一致。 13 | 14 | 15 | 集群方式部署提供 Web 界面配置，因此本节不再描述 Rest API 的各个选项，请参考 `toml` 格式的配置文件描述即可。 16 | 17 | 18 | ------------------- 19 | 20 | 21 | 配置文件最少需要提供 `Input` 和 `Output` 的配置。 22 | 23 | - [Input 配置](03-inputs.md) 24 | - [Output 配置](04-outputs.md) 25 | - [Filter 配置](05-filters.md) 26 | - [Scheduler 配置](06-scheduler.md) 27 | - [Matcher 配置](07-matcher-en.md) -------------------------------------------------------------------------------- /docs/2.0/05-filters.md: -------------------------------------------------------------------------------- 1 | Filter 定义了对 Input 消息的一些列变换操作。 2 | 3 | Filter 是以数组的方式配置的，系统会按照顺序执行每一个 Filter。 4 | 5 | 当前支持如下几种 Filter： 6 | 7 | - **reject** 忽略匹配的源端消息 8 | - **delete-dml-column** 删除源端 DML 消息里的某些列 9 | - **rename-dml-column** 重命名源端 DML 消息里的某些列 10 | 11 | 12 | ### reject 13 | 14 | ```toml 15 | [[filters]] 16 | type = "reject" 17 | 18 | [filters.config] 19 | match-schema = "test" 20 | match-table = "test_table_*" 21 | ``` 22 | 23 | `reject` Filter 会拒绝所有匹配到的 Input 消息，这些消息不会发送到下一个 Filter，也不会发送给 Output。 24 | 25 | 上面的例子里，所有 `schema` 为 `test`，`table` 名字为 `test_table_*` 开头的消息都会被过滤掉。 26 | 27 | ```toml 28 | [[filters]] 29 | type = "reject" 30 | [filters.config] 31 | match-schema = "test" 32 | match-dml-op = "delete" 33 | ``` 34 | 35 | 上面的例子里，所有 `schema` 为 `test` 的 `delete` 类型 `DML` 都会被过滤掉。 36 | 37 | ### delete-dml-column 38 | ```toml 39 | [[filters]] 40 | type = "delete-dml-column" 41 | [filters.config] 42 | match-schema = "test" 43 | match-table = "test_table" 44 | columns = ["e", "f"] 45 | ``` 46 | 47 | `delete-dml-column` Filter 会删除匹配到的 Input 消息里的某些列。 48 | 49 | 上面的例子里，所有 `schema` 为 `test`, `table` 名字为 `test_table` 的 DML 消息，它们的 `e`, `f` 列都会被删除。 50 | 51 | ### rename-dml-column 52 | 53 | ```toml 54 | [[filters]] 55 | type = "rename-dml-column" 56 | [filters.config] 57 | match-schema = "test" 58 | match-table = "test_table" 59 | from = ["a", "b"] 60 | to = ["c", "d"] 61 | ``` 62 | 63 | ## `grpc-sidecar` 64 | 65 | `grpc-sidecar` Filter 会下载一个你指定的二进制文件并启动一个进程。你的这个程序需要实现一个 GRPC 的服务。一个 Golang 的例子在[这里](https://github.com/moiot/gravity-grpc-sidecar-filter-example) 66 | 67 | 目前 `grpc-sidecar` 只支持修改 `core.Msg.DmlMsg` 里的内容。 68 | 69 | GRPC 协议的定义在[这里](https://github.com/moiot/gravity/blob/master/protocol/msgpb/message.proto) 70 | ```toml 71 | [[filters]] 72 | type = "grp-sidecar" 73 | [filters.config] 74 | match-schema = "test" 75 | match-table = "test_table" 76 | binary-url = "binary url that stores the binary" 77 | name = "unique name of this plugin" 78 | ``` -------------------------------------------------------------------------------- /docs/2.0/06-scheduler-en.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Scheduler Configuration 3 | summary: Learn how to configure Scheduler. 4 | --- 5 | 6 | # Scheduler Configuration 7 | 8 | Currently, Gravity supports only one Scheduler plugin: 9 | 10 | - `batch-table-scheduler`: Guarantees that update operations on the data in the same row defined by the primary key are performed in order. 11 | 12 | ## `batch-table-scheduler` configuration 13 | 14 | ```toml 15 | # 16 | # `batch-table-scheduler` configuration 17 | # Optional 18 | # 19 | [scheduler] 20 | type = "batch-table-scheduler" 21 | 22 | [scheduler.config] 23 | # "100" by default 24 | nr-worker = 100 25 | 26 | # "1" by default 27 | batch-size = 1 28 | 29 | # "1024" by default 30 | queue-size = 1024 31 | 32 | # "10240" by default 33 | sliding-window-size = 10240 34 | ``` 35 | 36 | In the above configuration: 37 | 38 | - `batch-table-scheduler` uses the worker pool method to call the interface defined by Output. 39 | 40 | - `nr-worker` is the number of workers. 41 | 42 | - `batch-table-scheduler` uses Output in the unit of batch and guarantees that one batch has the same `core.Msg.Table`. 43 | 44 | - `batch-size` is the size of a batch. 45 | 46 | - `batch-table-scheduler` uses the sliding window to guarantee the positions in Input are saved in order. 47 | 48 | - `sliding-window-size` is the size of a sliding window. -------------------------------------------------------------------------------- /docs/2.0/06-scheduler.md: -------------------------------------------------------------------------------- 1 | 当前支持的 Scheduler Plugin 只有一种 2 | 3 | - **batch-table-scheduler** 保证一个表内，由主键内容定义的同一行数据的更新操作有序 4 | 5 | ### batch-table-scheduler 6 | 7 | ```toml 8 | # 9 | # batch-table-scheduler 配置 10 | # - 可选 11 | # 12 | [scheduler] 13 | type = "batch-table-scheduler" 14 | 15 | [scheduler.config] 16 | # 默认值 100 17 | nr-worker = 100 18 | 19 | # 默认值 1 20 | batch-size = 1 21 | 22 | # 默认值 1024 23 | queue-size = 1024 24 | 25 | # 默认值 10240 26 | sliding-window-size = 10240 27 | ``` 28 | 29 | batch-table-scheduler 使用 worker pool 的方式调用 Output 定义的接口。 30 | 31 | `nr-worker` 是 worker 的数目； 32 | 33 | batch-table-scheduler 按照 batch 来使用 Output，它保证一个 batch 有相同的 core.Msg.Table 34 | 35 | `batch-size` 是 batch 大小； 36 | 37 | 38 | batch-table-scheduler 使用 sliding window 保证 Input 的位点按顺序保存。 39 | 40 | `sliding-window-size` 是 sliding window 大小。 -------------------------------------------------------------------------------- /docs/2.0/07-matcher-en.md: -------------------------------------------------------------------------------- 1 | # Matcher 2 | 3 | `Matcher` is used in filter and router. Existing matchers list here. 4 | 5 | ```toml 6 | match-schema = "test" 7 | match-table = "test_table_*" 8 | match-table = ["a*", "b*"] 9 | match-table-regex = "^t_\\d+$" # pay attention to `^` and `$` 10 | match-table-regex = ["^a.*$", "^t_\\d+$"] 11 | match-dml-op = "delete" # rejects ddl 12 | match-dml-op = ["insert", "update", "delete"] 13 | match-ddl-regex = '(?i)^DROP\sTABLE' # rejects dml -------------------------------------------------------------------------------- /docs/2.0/example-mongo2kafka.toml: -------------------------------------------------------------------------------- 1 | name = "example-mongo2kafka" 2 | 3 | [input] 4 | type = "mongo" 5 | mode = "stream" 6 | 7 | [input.config.source] 8 | host = "127.0.0.1" 9 | port = 27017 10 | 11 | [output] 12 | type = "async-kafka" 13 | 14 | [output.config.kafka-global-config] 15 | broker-addrs = ["localhost:9092"] 16 | 17 | [[output.config.routes]] 18 | match-schema = "test" 19 | match-table = "test_table" 20 | dml-topic = "test.test_table" 21 | 22 | [scheduler] 23 | type = "batch-table-scheduler" 24 | [scheduler.config] 25 | nr-worker = 1 26 | batch-size = 2 27 | queue-size = 1024 28 | sliding-window-size = 1024 29 | -------------------------------------------------------------------------------- /docs/2.0/example-mysql2kafka.toml: -------------------------------------------------------------------------------- 1 | # 整个配置由 4 部分组成： 2 | # - input: 定义 input plugin 的配置 3 | # - filters: 定义 filters plugin 的配置，filter 用来对数据流做变更操作 4 | # - output: 定义 output plugin 的配置 5 | # - system: 定义系统级配置 6 | # 7 | # 围绕 core.Msg, 系统定义若干个 match 函数，在配置文件里使用 match 函数 8 | # 来匹配 filter 和 output 的路由，filter/output 里的每一个 match 函数 9 | # 都匹配才算满足匹配规则 10 | # 11 | name = "mysql2mysqlDemo" 12 | version = "1.0" 13 | 14 | [input] 15 | type = "mysql" 16 | mode = "stream" 17 | 18 | [input.config] 19 | ignore-bidirectional-data = true 20 | 21 | [input.config.source] 22 | host = "127.0.0.1" 23 | username = "root" 24 | password = "" 25 | port = 3306 26 | 27 | [[filters]] 28 | type = "reject" 29 | [filters.config] 30 | match-schema = "test_db" 31 | match-table = "test_table" 32 | 33 | [[filters]] 34 | type = "rename-dml-column" 35 | [filters.config] 36 | match-schema = "test" 37 | match-table = "test_table_2" 38 | from = ["b"] 39 | to = ["d"] 40 | 41 | [[filters]] 42 | type = "delete-dml-column" 43 | [filters.config] 44 | match-schema = "test" 45 | match-table = "test_table" 46 | columns = ["e", "f"] 47 | 48 | [[filters]] 49 | type = "dml-pk-override" 50 | [filters.config] 51 | match-schema = "test" 52 | match-table = "test_table" 53 | id = "another_id" 54 | 55 | [output] 56 | type = "async-kafka" 57 | [output.config.kafka-global-config] 58 | broker-addrs = ["1.2.3.4:9002"] 59 | mode = "async" 60 | 61 | [[output.config.routes]] 62 | match-schema = "test_db" 63 | match-table = "test_table_2" 64 | dml-topic = "binlog.test_db.test_table_2" 65 | 66 | [scheduler] 67 | type = "batch-table-scheduler" 68 | [scheduler.config] 69 | nr-worker = 1 70 | batch-size = 2 71 | queue-size = 1024 72 | sliding-window-size = 1024 73 | -------------------------------------------------------------------------------- /docs/2.0/example-mysql2mongo.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moiot/gravity/15226ce9f24891b52cfc3955fce6ae6bf7e3d32a/docs/2.0/example-mysql2mongo.toml -------------------------------------------------------------------------------- /docs/2.0/example-mysql2mysql-full.toml: -------------------------------------------------------------------------------- 1 | # 整个配置由 4 部分组成： 2 | # - input: 定义 input plugin 的配置 3 | # - filters: 定义 filters plugin 的配置，filter 用来对数据流做变更操作 4 | # - output: 定义 output plugin 的配置 5 | # - system: 定义系统级配置 6 | # 7 | # 围绕 core.Msg, 系统定义若干个 match 函数，在配置文件里使用 match 函数 8 | # 来匹配 filter 和 output 的路由，filter/output 里的每一个 match 函数 9 | # 都匹配才算满足匹配规则 10 | # 11 | name = "mysql2mysqlDemo" 12 | version = "1.0" 13 | 14 | [input] 15 | type = "mysql" 16 | mode = "replication" 17 | 18 | [input.config] 19 | ignore-bidirectional-data = true 20 | 21 | [input.config.source] 22 | host = "127.0.0.1" 23 | username = "root" 24 | password = "" 25 | port = 3306 26 | max-open = 50 # optional, max connections 27 | max-idle = 50 # optional, suggest to be the same as max-open 28 | 29 | [[filters]] 30 | type = "reject" 31 | [filters.config] 32 | match-schema = "test_db" 33 | match-table = "test_table" 34 | 35 | [[filters]] 36 | type = "rename-dml-column" 37 | [filters.config] 38 | match-schema = "test" 39 | match-table = "test_table_2" 40 | from = ["b"] 41 | to = ["d"] 42 | 43 | [[filters]] 44 | type = "delete-dml-column" 45 | [filters.config] 46 | match-schema = "test" 47 | match-table = "test_table" 48 | columns = ["e", "f"] 49 | 50 | [[filters]] 51 | type = "dml-pk-override" 52 | [filters.config] 53 | match-schema = "test" 54 | match-table = "test_table" 55 | id = "another_id" 56 | 57 | [output] 58 | type = "mysql" 59 | 60 | [output.config] 61 | enable-ddl = true 62 | 63 | [output.config.target] 64 | host = "127.0.0.1" 65 | username = "root" 66 | password = "" 67 | port = 3306 68 | max-open = 20 # optional, max connections 69 | max-idle = 20 # optional, suggest to be the same as max-open 70 | 71 | [output.config.sql-engine-config] 72 | type = "mysql-replace-engine" 73 | 74 | [output.config.sql-engine-config.config] 75 | tag-internal-txn = true 76 | 77 | [[output.config.routes]] 78 | match-schema = "test_db" 79 | match-table = "test_table" 80 | target-schema = "test_db" 81 | target-table = "*" 82 | 83 | [scheduler] 84 | type = "batch-table-scheduler" 85 | [scheduler.config] 86 | nr-worker = 20 87 | batch-size = 10 88 | queue-size = 1024 89 | sliding-window-size = 1024 90 | -------------------------------------------------------------------------------- /docs/2.0/example-mysql2mysql-inc.toml: -------------------------------------------------------------------------------- 1 | # 整个配置由 4 部分组成： 2 | # - input: 定义 input plugin 的配置 3 | # - filters: 定义 filters plugin 的配置，filter 用来对数据流做变更操作 4 | # - output: 定义 output plugin 的配置 5 | # - system: 定义系统级配置 6 | # 7 | # 围绕 core.Msg, 系统定义若干个 match 函数，在配置文件里使用 match 函数 8 | # 来匹配 filter 和 output 的路由，filter/output 里的每一个 match 函数 9 | # 都匹配才算满足匹配规则 10 | # 11 | name = "mysql2mysqlDemo" 12 | version = "1.0" 13 | 14 | [input] 15 | type = "mysql" 16 | mode = "stream" 17 | 18 | [input.config] 19 | ignore-bidirectional-data = true 20 | [input.config.source] 21 | host = "127.0.0.1" 22 | username = "root" 23 | password = "" 24 | port = 3306 25 | 26 | [[filters]] 27 | type = "reject" 28 | [filters.config] 29 | match-schema = "test_db" 30 | match-table = "test_table" 31 | 32 | [[filters]] 33 | type = "rename-dml-column" 34 | [filters.config] 35 | match-schema = "test_db" 36 | match-table = "test_table_2" 37 | from = ["b"] 38 | to = ["d"] 39 | 40 | [[filters]] 41 | type = "delete-dml-column" 42 | [filters.config] 43 | match-schema = "test_db" 44 | match-table = "test_table_3" 45 | columns = ["e", "f"] 46 | 47 | [[filters]] 48 | type = "dml-pk-override" 49 | [filters.config] 50 | match-schema = "test_db" 51 | match-table = "test_table" 52 | from = ["id"] 53 | to = ["another_id"] 54 | 55 | [output] 56 | type = "mysql" 57 | 58 | [output.config] 59 | enable-ddl = true 60 | 61 | [output.config.target] 62 | host = "127.0.0.1" 63 | username = "root" 64 | password = "" 65 | port = 3306 66 | max-open = 20 # optional, max connections 67 | max-idle = 20 # optional, suggest to be the same as max-open 68 | 69 | [output.config.sql-engine-config] 70 | type = "mysql-replace-engine" 71 | 72 | [output.config.sql-engine-config.config] 73 | # with following two config enabled, gravity will add /*abc*/ to every statement executed, which is useful to hint db proxies 74 | tag-internal-txn = true 75 | sql-annotation = "abc" 76 | 77 | [[output.config.routes]] 78 | match-schema = "test_db" 79 | match-table = "test_table" 80 | target-schema = "test_db" 81 | target-table = "test_target_db" 82 | 83 | [scheduler] 84 | type = "batch-table-scheduler" 85 | [scheduler.config] 86 | nr-worker = 20 87 | batch-size = 10 88 | queue-size = 1024 89 | sliding-window-size = 1024 90 | -------------------------------------------------------------------------------- /docs/2.0/k8s-160.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moiot/gravity/15226ce9f24891b52cfc3955fce6ae6bf7e3d32a/docs/2.0/k8s-160.png -------------------------------------------------------------------------------- /docs/2.0/product.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moiot/gravity/15226ce9f24891b52cfc3955fce6ae6bf7e3d32a/docs/2.0/product.png -------------------------------------------------------------------------------- /docs/2.0/single-process-160.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moiot/gravity/15226ce9f24891b52cfc3955fce6ae6bf7e3d32a/docs/2.0/single-process-160.png -------------------------------------------------------------------------------- /docs/2.0/single-process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moiot/gravity/15226ce9f24891b52cfc3955fce6ae6bf7e3d32a/docs/2.0/single-process.png -------------------------------------------------------------------------------- /docs/rfc/rfc_schema_storage.md: -------------------------------------------------------------------------------- 1 | # RFC： schema store 2 | 3 | 4 | ### 目的 5 | Schema Storage 的目的是为了保存源数据库当前的 Schema，主要的目的是两个： 6 | 1. 解析当前的 binlog event 之后，需要知道 Column 的值和 Column 的名字的对应关系，才能包含完整语义的数据到 kafka 7 | 2. gravity 重启之后，需要知道当前所对应的 binlog 位置对应的 Schema 8 | 9 | ### 设计 10 | 11 | 基于以上两个目的，考虑把源数据库的 Schema 信息存放在数据库里。由于 Schema 的信息会随着 DDL 操作的发生而变化，所有需要有地方存储 Schema 的信息。 12 | 13 | 注意不能直接从源数据库拿 Schema，当处理 DDL 操作的时候，从源数据库拿到的 Schema 是当前时间的最新的 Schema，但不一定是 下一个要处理的 DML 语句对应的 Schema。 14 | 15 | 本设计中，利用 `schema-tracker` 跟踪源数据库的 Schema。`schema-tracker` 是一个独立的 MySQL 数据库，每次 DDL 变更操作到来的时候，都会在 `schema-tracker` 上执行一次。当前 Schema 可以通过 `show create table` 语句获取。 16 | 17 | 18 | 19 | 20 | 以下为数据库表结构的设计： 21 | 22 | ``` 23 | CREATE TABLE `mysql_schema_history` ( 24 | `id` int(11) NOT NULL AUTO_INCREMENT, 25 | `previous_schema_id` int(11) 26 | `binlog_position_id` text NOT NULL, 27 | `schema` text NOT NULL, 28 | `create_table_statement` text NOT NULL, 29 | `source_uuid` varchar(255) NOT NULL, 30 | `database_name` varchar(255) NOT NULL, 31 | `table_name` varchar(255) NOT NULL, 32 | `created_at` 33 | `updated_at` 34 | PRIMARY KEY (`id`) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; 36 | ``` 37 | 38 | SchemaStore 接口： 39 | 40 | - getSchema(binlogPositionId int) schema 41 | 42 | 返回此 positionId 能找到的最新的 Schema。在 `mysql_schema_history` 中查找 `binlogPositionId` 对应的最新的 Schema 43 | 44 | - processDDL(sql string, binlogPositionId int) 45 | 46 | 存储此 `binlogPositionId` 对应的 Schema。此操作会先在 `schema-tracker` 里执行 DDL 语句，然后对相应的 table 查询 `show create table`， 47 | 将返回的 语句存入 `create_table_statement`，并通过 `information_schema` 得到当前 schema, 存入 `mysql._schema_history.schema`。 48 | 49 | `create_table_statement` 语句可以方便我们调试。 50 | 51 | 52 | `binlogPositionId` 连同 binlog 的位置保存在本地文件里持久化存储，每次更新 binlog 位置时会对 `binlogPositionId` 更新。`binlogPositionId`设计为自增的。 53 | 54 | 55 | 将来可以优化的地方： 56 | 57 | - 能够处理 DDL 失败的情况。可以在 DDL 操作前 dump 一次 `schema_tracker` 的 Schema，操作完成后删除这个 dump，如果 DDL 失败，那么下次重启的时候发现有 dump 存在，可以重新再来 58 | 59 | - binlogPosition 以及 binlog 的位置现在是存在文件里，实时更新。 60 | 可以考虑存在数据库里，并且存储 binlog 位置的历史版本 (CheckPoint)，这样可以通过历史版本的 binlog 位置以及 Schema 回放某个 binlog 位置开始的操作。 -------------------------------------------------------------------------------- /docs/rfc_schema_storage.md: -------------------------------------------------------------------------------- 1 | # RFC： schema store 2 | 3 | 4 | ### 目的 5 | Schema Storage 的目的是为了保存源数据库当前的 Schema，主要的目的是两个： 6 | 1. 解析当前的 binlog event 之后，需要知道 Column 的值和 Column 的名字的对应关系，才能包含完整语义的数据到 kafka 7 | 2. gravity 重启之后，需要知道当前所对应的 binlog 位置对应的 Schema 8 | 9 | ### 设计 10 | 11 | 基于以上两个目的，考虑把源数据库的 Schema 信息存放在数据库里。由于 Schema 的信息会随着 DDL 操作的发生而变化，所有需要有地方存储 Schema 的信息。 12 | 13 | 注意不能直接从源数据库拿 Schema，当处理 DDL 操作的时候，从源数据库拿到的 Schema 是当前时间的最新的 Schema，但不一定是 下一个要处理的 DML 语句对应的 Schema。 14 | 15 | 本设计中，利用 `schema-tracker` 跟踪源数据库的 Schema。`schema-tracker` 是一个独立的 MySQL 数据库，每次 DDL 变更操作到来的时候，都会在 `schema-tracker` 上执行一次。当前 Schema 可以通过 `show create table` 语句获取。 16 | 17 | 18 | 19 | 20 | 以下为数据库表结构的设计： 21 | 22 | ``` 23 | CREATE TABLE `mysql_schema_history` ( 24 | `id` int(11) NOT NULL AUTO_INCREMENT, 25 | `previous_schema_id` int(11) 26 | `binlog_position_id` text NOT NULL, 27 | `schema` text NOT NULL, 28 | `create_table_statement` text NOT NULL, 29 | `source_uuid` varchar(255) NOT NULL, 30 | `database_name` varchar(255) NOT NULL, 31 | `table_name` varchar(255) NOT NULL, 32 | `created_at` 33 | `updated_at` 34 | PRIMARY KEY (`id`) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; 36 | ``` 37 | 38 | SchemaStore 接口： 39 | 40 | - getSchema(binlogPositionId int) schema 41 | 42 | 返回此 positionId 能找到的最新的 Schema。在 `mysql_schema_history` 中查找 `binlogPositionId` 对应的最新的 Schema 43 | 44 | - processDDL(sql string, binlogPositionId int) 45 | 46 | 存储此 `binlogPositionId` 对应的 Schema。此操作会先在 `schema-tracker` 里执行 DDL 语句，然后对相应的 table 查询 `show create table`， 47 | 将返回的 语句存入 `create_table_statement`，并通过 `information_schema` 得到当前 schema, 存入 `mysql._schema_history.schema`。 48 | 49 | `create_table_statement` 语句可以方便我们调试。 50 | 51 | 52 | `binlogPositionId` 连同 binlog 的位置保存在本地文件里持久化存储，每次更新 binlog 位置时会对 `binlogPositionId` 更新。`binlogPositionId`设计为自增的。 53 | 54 | 55 | 将来可以优化的地方： 56 | 57 | - 能够处理 DDL 失败的情况。可以在 DDL 操作前 dump 一次 `schema_tracker` 的 Schema，操作完成后删除这个 dump，如果 DDL 失败，那么下次重启的时候发现有 dump 存在，可以重新再来 58 | 59 | - binlogPosition 以及 binlog 的位置现在是存在文件里，实时更新。 60 | 可以考虑存在数据库里，并且存储 binlog 位置的历史版本 (CheckPoint)，这样可以通过历史版本的 binlog 位置以及 Schema 回放某个 binlog 位置开始的操作。 -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Add local user 4 | # Either use the LOCAL_USER_ID if passed in at runtime or 5 | # fallback 6 | 7 | USER_ID=${LOCAL_USER_ID:-9001} 8 | 9 | echo "Starting with UID : $USER_ID" 10 | useradd --shell /bin/bash -u $USER_ID -o -c "" -m user 11 | export HOME=/home/user 12 | 13 | exec /usr/local/bin/gosu user "$@" -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/moiot/gravity 2 | 3 | require ( 4 | github.com/BurntSushi/toml v0.3.1 5 | github.com/OneOfOne/xxhash v1.2.5 6 | github.com/Shopify/sarama v1.19.0 7 | github.com/coreos/etcd v3.3.10+incompatible 8 | github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 9 | github.com/fsnotify/fsnotify v1.4.7 10 | github.com/go-sql-driver/mysql v1.5.0 11 | github.com/gogo/protobuf v1.3.2 12 | github.com/golang/mock v1.3.1 13 | github.com/golang/protobuf v1.4.3 14 | github.com/hashicorp/go-getter v0.0.0-20181213035916-be39683deade 15 | github.com/hashicorp/go-hclog v0.7.0 // indirect 16 | github.com/hashicorp/go-plugin v0.0.0-20190220160451-3f118e8ee104 17 | github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect 18 | github.com/jinzhu/gorm v1.9.12 19 | github.com/json-iterator/go v1.1.12 20 | github.com/juju/errors v0.0.0-20170703010042-c7d06af17c68 21 | github.com/juju/loggo v0.0.0-20190212223446-d976af380377 // indirect 22 | github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073 // indirect 23 | github.com/mitchellh/hashstructure v1.0.0 24 | github.com/mitchellh/mapstructure v1.1.2 25 | github.com/olivere/elastic v6.2.17+incompatible 26 | github.com/olivere/elastic/v7 v7.0.6 27 | github.com/onsi/ginkgo v1.11.0 28 | github.com/onsi/gomega v1.8.1 29 | github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 30 | github.com/pingcap/parser v0.0.0-20200623164729-3a18f1e5dceb 31 | github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c 32 | github.com/prometheus/client_golang v1.11.1 33 | github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect 34 | github.com/satori/go.uuid v1.2.0 35 | github.com/serialx/hashring v0.0.0-20170811022404-6a9381c5a83e 36 | github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 37 | github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 38 | github.com/siddontang/go-mysql v0.0.0-20190312052122-c6ab05a85eb8 39 | github.com/sirupsen/logrus v1.6.0 40 | github.com/stretchr/testify v1.5.1 41 | github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect 42 | github.com/xdg/stringprep v1.0.0 // indirect 43 | go.mongodb.org/mongo-driver v1.0.1 44 | golang.org/x/net v0.0.0-20201021035429-f5854403a974 45 | golang.org/x/sys v0.6.0 // indirect 46 | google.golang.org/grpc v1.26.0 47 | gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 48 | gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce 49 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 50 | ) 51 | 52 | go 1.13 53 | -------------------------------------------------------------------------------- /hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This file modified from k8s 3 | # https://github.com/kubernetes/kubernetes/blob/master/hooks/pre-commit 4 | 5 | # How to use this hook? 6 | # ln -s hooks/pre-commit .git/hooks/ 7 | # In case hook is not executable 8 | # chmod +x .git/hooks/pre-commit 9 | 10 | readonly reset=$(tput sgr0) 11 | readonly red=$(tput bold; tput setaf 1) 12 | readonly green=$(tput bold; tput setaf 2) 13 | 14 | exit_code=0 15 | 16 | # comment it by default. You can uncomment it. 17 | # echo -ne "Checking that it builds..." 18 | # if ! OUT=$(make 2>&1); then 19 | # echo 20 | # echo "${red}${OUT}" 21 | # exit_code=1 22 | # else 23 | # echo "${green}OK" 24 | # fi 25 | # echo "${reset}" 26 | 27 | echo -ne "Checking for files that need goimport... " 28 | files_need_gofmt=() 29 | files=($(git diff --cached --name-only --diff-filter ACM | grep "\.go" | grep -v -e "^vendor" | grep -v "pb.go")) 30 | for file in "${files[@]}"; do 31 | # Check for files that fail gofmt. 32 | diff="$(cat "${file}" | goimports -d 2>&1)" 33 | if [[ -n "$diff" ]]; then 34 | echo ${diff} 35 | files_need_gofmt+=("${file}") 36 | fi 37 | done 38 | 39 | if [[ "${#files_need_gofmt[@]}" -ne 0 ]]; then 40 | echo "${red}ERROR!" 41 | echo "Some files have not been gofmt'd. To fix these errors, " 42 | echo "copy and paste the following:" 43 | echo " goimports -local git.mobike.io -w ${files_need_gofmt[@]}" 44 | exit_code=1 45 | else 46 | echo "${green}OK" 47 | fi 48 | echo "${reset}" 49 | 50 | if [[ "${exit_code}" != 0 ]]; then 51 | echo "${red}Aborting commit${reset}" 52 | fi 53 | exit ${exit_code} 54 | -------------------------------------------------------------------------------- /integration_test/mongomysql/main_test.go: -------------------------------------------------------------------------------- 1 | package mongokafka_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/moiot/gravity/pkg/utils" 8 | 9 | mgo "gopkg.in/mgo.v2" 10 | 11 | "github.com/moiot/gravity/pkg/consts" 12 | 13 | "github.com/moiot/gravity/pkg/mongo_test" 14 | ) 15 | 16 | var session *mgo.Session 17 | var db *mgo.Database 18 | 19 | func TestMain(m *testing.M) { 20 | mongoCfg := mongo_test.TestConfig() 21 | s, err := utils.CreateMongoSession(&mongoCfg) 22 | if err != nil { 23 | panic(err) 24 | } 25 | session = s 26 | mongo_test.InitReplica(session) 27 | defer session.Close() 28 | 29 | db = s.DB("mongomysql") 30 | if err := db.DropDatabase(); err != nil { 31 | panic(err) 32 | } 33 | if err := s.DB(consts.GravityDBName).DropDatabase(); err != nil { 34 | panic(err) 35 | } 36 | 37 | os.Exit(m.Run()) 38 | } 39 | -------------------------------------------------------------------------------- /integration_test/mysqlelasticsearch/main_test.go: -------------------------------------------------------------------------------- 1 | package mysqlelasticsearch_test 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "os" 7 | "testing" 8 | 9 | "github.com/moiot/gravity/pkg/consts" 10 | "github.com/moiot/gravity/pkg/elasticsearch_test" 11 | "github.com/moiot/gravity/pkg/mysql_test" 12 | "github.com/olivere/elastic" 13 | ) 14 | 15 | func TestMain(m *testing.M) { 16 | db := mysql_test.MustCreateSourceDBConn() 17 | _, err := db.Exec("drop database if exists " + consts.GravityDBName) 18 | if err != nil { 19 | panic(err) 20 | } 21 | err = db.Close() 22 | if err != nil { 23 | panic(err) 24 | } 25 | 26 | client, err := elasticsearch_test.CreateTestClient() 27 | if err != nil { 28 | panic(err) 29 | } 30 | _, err = client.DeleteIndex(mysql_test.TestTableName).Do(context.Background()) 31 | if err != nil { 32 | e, ok := err.(*elastic.Error) 33 | if !ok || e.Status != http.StatusNotFound { 34 | panic(err) 35 | } 36 | } 37 | client.Stop() 38 | os.Exit(m.Run()) 39 | } 40 | -------------------------------------------------------------------------------- /mycnf/mysql.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | log-bin=mysql-bin 3 | server-id=100 4 | gtid_mode=ON 5 | enforce-gtid-consistency=true 6 | transaction-isolation=READ-COMMITTED 7 | explicit_defaults_for_timestamp=true 8 | sql_mode=STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION -------------------------------------------------------------------------------- /padder/config/config_suite_test.go: -------------------------------------------------------------------------------- 1 | package config_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestPadder(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Padder Config Suite") 13 | } 14 | -------------------------------------------------------------------------------- /padder/config_example/padder.toml: -------------------------------------------------------------------------------- 1 | [padder] 2 | enable-delete = true 3 | binlog-list = ["bin.001", "bin.002"] 4 | 5 | [padder.mysql.target] 6 | host = "localhost" 7 | username = "root" 8 | password = "" 9 | port = 3306 10 | schema = "test" 11 | location = "AST" 12 | 13 | [padder.mysql.start-position] 14 | binlog-name= "bin.001" 15 | binlog-pos= 1234 16 | 17 | -------------------------------------------------------------------------------- /padder/job_processor/job.go: -------------------------------------------------------------------------------- 1 | package job_processor 2 | 3 | import ( 4 | "time" 5 | 6 | gomysql "github.com/siddontang/go-mysql/mysql" 7 | 8 | "github.com/moiot/gravity/pkg/core" 9 | 10 | "github.com/moiot/gravity/pkg/schema_store" 11 | "github.com/moiot/gravity/pkg/utils" 12 | ) 13 | 14 | type Job struct { 15 | seqNum int64 16 | srcId string 17 | opType string 18 | JobMsg core.Msg 19 | pos gomysql.Position 20 | gtidSet gomysql.MysqlGTIDSet 21 | schemaStore schema_store.SchemaStore 22 | } 23 | 24 | func CreateJob(seqNum int64, srcId string, opType string, jobMsg core.Msg, pos gomysql.Position, gtidSet gomysql.MysqlGTIDSet) Job { 25 | return Job{ 26 | seqNum: seqNum, 27 | srcId: srcId, 28 | opType: opType, 29 | JobMsg: jobMsg, 30 | pos: pos, 31 | gtidSet: gtidSet, 32 | } 33 | } 34 | 35 | func (job Job) SlidingWindowKey() string { 36 | return job.srcId 37 | } 38 | 39 | func (job Job) TableKey() string { 40 | return utils.TableIdentity(job.JobMsg.Database, job.JobMsg.Table) 41 | } 42 | 43 | func (job Job) WorkerKey() string { 44 | return job.JobMsg.GetPkSign() 45 | } 46 | 47 | func (job Job) EventTime() time.Time { 48 | return job.JobMsg.Timestamp 49 | } 50 | 51 | func (job Job) SequenceNumber() int64 { 52 | return job.seqNum 53 | } 54 | 55 | func (job Job) BeforeWindowMoveForward() { 56 | return 57 | } 58 | 59 | func (job Job) SkipDownStream() bool { 60 | return false 61 | } 62 | 63 | func (job Job) DoneC() chan struct{} { 64 | return nil 65 | } 66 | 67 | func (job Job) Msg() core.Msg { 68 | return job.JobMsg 69 | } 70 | -------------------------------------------------------------------------------- /padder/job_processor/job_processor_suite_test.go: -------------------------------------------------------------------------------- 1 | package job_processor_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestJobProcessor(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "pad job worker Suite") 13 | } 14 | -------------------------------------------------------------------------------- /padder/job_processor/mysql_worker.go: -------------------------------------------------------------------------------- 1 | package job_processor 2 | 3 | // 4 | // import ( 5 | // "github.com/juju/errors" 6 | // 7 | // "github.com/moiot/gravity/pkg/core" 8 | // 9 | // "sync" 10 | // "time" 11 | // 12 | // "github.com/moiot/gravity/pkg/utils/retry" 13 | // "github.com/moiot/gravity/pkg/worker_pool" 14 | // "github.com/moiot/gravity/schema_store" 15 | // "github.com/moiot/gravity/sql_execution_engine" 16 | // ) 17 | // 18 | // type MySQLWorker struct { 19 | // targetSchemaStore schema_store.SchemaStore 20 | // sqlExecutionEngine sql_execution_engine.SQlExecutionEngine 21 | // wg sync.WaitGroup 22 | // } 23 | // 24 | // func NewMySQLWorker( 25 | // jobBatchC chan []worker_pool.Job, 26 | // schemaStore schema_store.SchemaStore, 27 | // sqlEngine sql_execution_engine.SQlExecutionEngine, 28 | // maxRetryCount int, 29 | // sleepDuration time.Duration, 30 | // ) *MySQLWorker { 31 | // w := MySQLWorker{ 32 | // sqlExecutionEngine: sqlEngine, 33 | // targetSchemaStore: schemaStore, 34 | // } 35 | // 36 | // w.wg.Add(1) 37 | // go func() { 38 | // defer w.wg.Done() 39 | // for jobBatch := range jobBatchC { 40 | // retry.Do(func() error { 41 | // return w.Execute(jobBatch) 42 | // }, maxRetryCount, sleepDuration) 43 | // } 44 | // }() 45 | // 46 | // return &w 47 | // } 48 | // 49 | // func (w *MySQLWorker) Wait() { 50 | // w.wg.Wait() 51 | // } 52 | // 53 | // func (w *MySQLWorker) Execute(jobBatch []worker_pool.Job) error { 54 | // var pbMsgBatch []*core.Msg 55 | // var tableDef *schema_store.Table 56 | // for i := range jobBatch { 57 | // job := jobBatch[i] 58 | // mysqlJob, ok := job.(Job) 59 | // if !ok { 60 | // return errors.Errorf("[padder_mysql_worker] failed type conversion") 61 | // } 62 | // 63 | // if tableDef == nil { 64 | // if schema, err := w.targetSchemaStore.GetSchema(mysqlJob.JobMsg.Database); err != nil { 65 | // return errors.Trace(err) 66 | // } else { 67 | // tableDef = schema[mysqlJob.JobMsg.Table] 68 | // } 69 | // 70 | // } 71 | // msg := job.Msg() 72 | // pbMsgBatch = append(pbMsgBatch, &msg) 73 | // } 74 | // 75 | // e := w.sqlExecutionEngine.Execute(pbMsgBatch, tableDef) 76 | // if e != nil { 77 | // return errors.Trace(e) 78 | // } 79 | // 80 | // return nil 81 | // } 82 | -------------------------------------------------------------------------------- /padder/padder_suite_test.go: -------------------------------------------------------------------------------- 1 | package padder 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestJobProcessor(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "padder suite") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/config/config_v2.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/BurntSushi/toml" 5 | "github.com/juju/errors" 6 | log "github.com/sirupsen/logrus" 7 | ) 8 | 9 | type PipelineConfigV2 struct { 10 | PipelineName string `mapstructure:"name" toml:"name" json:"name"` 11 | InputPlugins map[string]interface{} `toml:"input" json:"input" mapstructure:"input"` 12 | FilterPlugins []interface{} `mapstructure:"filters" toml:"filters" json:"filters,omitempty"` 13 | OutputPlugins map[string]interface{} `mapstructure:"output" toml:"output" json:"output"` 14 | SchedulerPlugins map[string]interface{} `mapstructure:"scheduler" toml:"scheduler" json:"scheduler,omitempty"` 15 | } 16 | 17 | func (c *PipelineConfigV2) IsV3() bool { 18 | _, ok := c.InputPlugins["type"] 19 | if !ok { 20 | log.Warn("received v2 config") 21 | } 22 | return ok 23 | } 24 | 25 | func (c *PipelineConfigV2) ToV3() PipelineConfigV3 { 26 | ret := PipelineConfigV3{ 27 | PipelineName: c.PipelineName, 28 | } 29 | 30 | for k, v := range c.InputPlugins { 31 | ret.InputPlugin.Type = k 32 | ret.InputPlugin.Config = v.(map[string]interface{}) 33 | if k == "mysql" { 34 | ret.InputPlugin.Mode = InputMode(ret.InputPlugin.Config["mode"].(string)) 35 | } else { 36 | ret.InputPlugin.Mode = Stream 37 | } 38 | } 39 | 40 | for _, f := range c.FilterPlugins { 41 | m := f.(map[string]interface{}) 42 | ff := GenericPluginConfig{ 43 | Type: m["type"].(string), 44 | } 45 | delete(m, "type") 46 | ff.Config = m 47 | ret.FilterPlugins = append(ret.FilterPlugins, ff) 48 | } 49 | 50 | for k, v := range c.OutputPlugins { 51 | ret.OutputPlugin.Type = k 52 | ret.OutputPlugin.Config = v.(map[string]interface{}) 53 | } 54 | 55 | for k, v := range c.SchedulerPlugins { 56 | ret.SchedulerPlugin = &GenericPluginConfig{ 57 | Type: k, 58 | Config: v.(map[string]interface{}), 59 | } 60 | } 61 | 62 | return ret 63 | } 64 | 65 | func DecodeTomlString(s string) (*PipelineConfigV2, error) { 66 | cfg := &PipelineConfigV2{} 67 | if _, err := toml.Decode(s, cfg); err != nil { 68 | return nil, errors.Trace(err) 69 | } 70 | 71 | return cfg, nil 72 | } 73 | -------------------------------------------------------------------------------- /pkg/config/config_v2_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/BurntSushi/toml" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestMapStructureDecodeEmptyStuff(t *testing.T) { 12 | r := require.New(t) 13 | 14 | t.Run("toml with empty key", func(tt *testing.T) { 15 | s := ` 16 | [some-random-key] 17 | ` 18 | 19 | m := make(map[string]interface{}) 20 | _, err := toml.Decode(s, &m) 21 | r.NoError(err) 22 | r.Equal(1, len(m)) 23 | _, ok := m["some-random-key"] 24 | r.True(ok) 25 | r.Equal(map[string]interface{}{}, m["some-random-key"]) 26 | }) 27 | 28 | t.Run("json with empty key", func(tt *testing.T) { 29 | s := ` 30 | { 31 | "some-random-key": {} 32 | } 33 | ` 34 | m := make(map[string]interface{}) 35 | err := json.Unmarshal([]byte(s), &m) 36 | r.NoError(err) 37 | r.Equal(1, len(m)) 38 | _, ok := m["some-random-key"] 39 | r.True(ok) 40 | r.Equal(map[string]interface{}{}, m["some-random-key"]) 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/config/config_v3.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/juju/errors" 7 | ) 8 | 9 | const PipelineConfigV3Version = "1.0" 10 | const defaultInternalDBName = "_gravity" 11 | 12 | type PipelineConfigV3 struct { 13 | PipelineName string `yaml:"name" toml:"name" json:"name"` 14 | InternalDBName string `yaml:"internal-db-name" toml:"internal-db-name" json:"internal-db-name"` 15 | Version string `yaml:"version" toml:"version" json:"version"` 16 | InputPlugin InputConfig `yaml:"input" toml:"input" json:"input"` 17 | FilterPlugins []GenericPluginConfig `yaml:"filters" toml:"filters" json:"filters,omitempty"` 18 | OutputPlugin GenericPluginConfig `yaml:"output" toml:"output" json:"output"` 19 | SchedulerPlugin *GenericPluginConfig `yaml:"scheduler" toml:"scheduler" json:"scheduler,omitempty"` 20 | } 21 | 22 | func (c *PipelineConfigV3) SetDefault() { 23 | if c.Version == "" { 24 | c.Version = PipelineConfigV3Version 25 | } 26 | 27 | if c.InternalDBName == "" { 28 | c.InternalDBName = defaultInternalDBName 29 | } 30 | } 31 | 32 | func (c *PipelineConfigV3) DeepCopy() PipelineConfigV3 { 33 | b, err := json.Marshal(c) 34 | if err != nil { 35 | panic(err) 36 | } 37 | ret := PipelineConfigV3{} 38 | if err = json.Unmarshal(b, &ret); err != nil { 39 | panic(err) 40 | } 41 | return ret 42 | } 43 | 44 | const ( 45 | Unknown InputMode = "unknown" 46 | Batch InputMode = "batch" 47 | Stream InputMode = "stream" 48 | Replication InputMode = "replication" // scan + binlog 49 | ) 50 | 51 | type InputMode string 52 | 53 | func (mode InputMode) Valid() error { 54 | if mode == Batch || mode == Stream || mode == Replication { 55 | return nil 56 | } else { 57 | return errors.Errorf("invalid mode: %v", mode) 58 | } 59 | } 60 | 61 | type InputConfig struct { 62 | Type string `yaml:"type" json:"type" toml:"type"` 63 | Mode InputMode `yaml:"mode" json:"mode" toml:"mode"` 64 | Config map[string]interface{} `yaml:"config" json:"config" toml:"config"` 65 | } 66 | 67 | type GenericPluginConfig struct { 68 | Type string `yaml:"type" json:"type" toml:"type"` 69 | Config map[string]interface{} `yaml:"config" json:"config,omitempty" toml:"config,omitempty"` 70 | } 71 | -------------------------------------------------------------------------------- /pkg/config/limits.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | var ( 4 | TxnBufferLimit = 1024 5 | MaxNrGravity = 16 6 | ) 7 | -------------------------------------------------------------------------------- /pkg/config/table.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type TableConfig struct { 4 | Schema string `toml:"schema" json:"schema"` 5 | Table string `toml:"table" json:"table"` 6 | 7 | RenameColumns map[string]string `toml:"rename-columns" json:"rename-columns"` 8 | IgnoreColumns []string `toml:"ignore-columns" json:"ignore-columns"` 9 | 10 | PkOverride []string `toml:"pk-override" json:"pk-override"` 11 | 12 | ScanColumn string `toml:"scan-column" json:"scan-column"` 13 | ScanType string `toml:"scan-type" json:"scan-type"` 14 | } 15 | -------------------------------------------------------------------------------- /pkg/consts/gravity.go: -------------------------------------------------------------------------------- 1 | package consts 2 | 3 | const ( 4 | MySQLInternalDBName = "mysql" 5 | OldDrcDBName = "drc" 6 | TxnTagTableName = "_gravity_txn_tags" 7 | DDLTag = "/*gravityDDL*/" 8 | ) 9 | 10 | var GravityDBName = "_gravity" 11 | 12 | func IsInternalDBTraffic(schema string) bool { 13 | return schema == OldDrcDBName || schema == GravityDBName 14 | } 15 | -------------------------------------------------------------------------------- /pkg/core/emitter.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type Emitter interface { 4 | // Emit use fs to modify messages and submit job to scheduler 5 | // msg is the message to send 6 | // 7 | Emit(msg *Msg) error 8 | Close() error 9 | } 10 | -------------------------------------------------------------------------------- /pkg/core/encoding/mongo_json.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/moiot/gravity/pkg/mongo/gtm" 7 | 8 | "github.com/moiot/gravity/pkg/core" 9 | ) 10 | 11 | type JsonMsg01 struct { 12 | Version string `json:"version"` 13 | Database string `json:"database"` 14 | Collection string `json:"collection"` 15 | UniqueSourceName string `json:"unique_source_name"` 16 | Oplog *gtm.Op `json:"oplog"` 17 | } 18 | 19 | type JsonMsgVersion20 struct { 20 | Version string `json:"version"` 21 | Database string `json:"database"` 22 | Collection string `json:"collection"` 23 | Data map[string]interface{} `json:"data"` 24 | Row map[string]interface{} `json:"row"` 25 | } 26 | 27 | type mongoJsonSerde struct { 28 | } 29 | 30 | func (s *mongoJsonSerde) Serialize(msg *core.Msg, version string) ([]byte, error) { 31 | switch version { 32 | case Version01: 33 | jsonMsg := JsonMsg01{} 34 | jsonMsg.Version = version 35 | jsonMsg.Database = msg.Oplog.GetDatabase() 36 | jsonMsg.Collection = msg.Oplog.GetCollection() 37 | jsonMsg.UniqueSourceName = msg.Host 38 | jsonMsg.Oplog = msg.Oplog 39 | return json.Marshal(jsonMsg) 40 | case Version20Alpha: 41 | jsonMsg := JsonMsgVersion20{} 42 | jsonMsg.Version = version 43 | jsonMsg.Database = msg.Database 44 | jsonMsg.Collection = msg.Table 45 | jsonMsg.Data = msg.Oplog.Data 46 | jsonMsg.Row = msg.Oplog.Row 47 | return json.Marshal(jsonMsg) 48 | default: 49 | return nil, nil 50 | } 51 | 52 | } 53 | 54 | func (*mongoJsonSerde) Deserialize(b []byte) (core.Msg, error) { 55 | panic("implement me") 56 | } 57 | -------------------------------------------------------------------------------- /pkg/core/encoding/pb_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * // Copyright 2019 , Beijing Mobike Technology Co., Ltd. 4 | * // 5 | * // Licensed under the Apache License, Version 2.0 (the "License"); 6 | * // you may not use this file except in compliance with the License. 7 | * // You may obtain a copy of the License at 8 | * // 9 | * // http://www.apache.org/licenses/LICENSE-2.0 10 | * // 11 | * // Unless required by applicable law or agreed to in writing, software 12 | * // distributed under the License is distributed on an "AS IS" BASIS, 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package encoding 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | func TestDataMapToPB(t *testing.T) { 27 | r := require.New(t) 28 | 29 | currentTime := time.Now() 30 | data := map[string]interface{}{ 31 | "a": "v1", 32 | "b": 2, 33 | "c": 1.234, 34 | "d": currentTime, 35 | "e": true, 36 | } 37 | 38 | pbMap, err := DataMapToPB(data) 39 | r.NoError(err) 40 | r.Equal(len(data), len(pbMap)) 41 | 42 | newData, err := PBToDataMap(pbMap) 43 | r.NoError(err) 44 | 45 | for k, v := range data { 46 | t, ok := v.(time.Time) 47 | if ok { 48 | kt := newData[k].(time.Time) 49 | r.True(t.Equal(kt)) 50 | } else { 51 | r.EqualValues(v, newData[k]) 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pkg/core/encoding/rdb_json_test.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | 9 | "github.com/moiot/gravity/pkg/core" 10 | ) 11 | 12 | func TestSerializeVersion20JsonMsg(t *testing.T) { 13 | assert := assert.New(t) 14 | 15 | tsString := "2018-01-01T22:35:35+08:00" 16 | ts, err := time.Parse(time.RFC3339Nano, tsString) 17 | if err != nil { 18 | assert.FailNow(err.Error()) 19 | } 20 | cases := []struct { 21 | name string 22 | msg core.Msg 23 | jsonString string 24 | }{ 25 | { 26 | "dml", 27 | core.Msg{ 28 | Type: core.MsgDML, 29 | DmlMsg: &core.DMLMsg{ 30 | Operation: core.Insert, 31 | Data: map[string]interface{}{ 32 | "id": 1, 33 | "name": nil, 34 | "ts": ts, 35 | "f": 10.345678, 36 | "a": "中文", 37 | "b": "😄", 38 | "c": true, 39 | "d": false, 40 | }, 41 | }, 42 | }, 43 | `{"version":"2.0.alpha","database":"","table":"","type":"insert","Data":{"a":"中文","b":"😄","c":true,"d":false,"f":10.345678,"id":1,"name":null,"ts":"2018-01-01T22:35:35+08:00"},"Old":null,"Pks":null}`, 44 | }, 45 | } 46 | 47 | for _, c := range cases { 48 | b, err := serializeVersion20JsonMsg(&c.msg) 49 | if err != nil { 50 | assert.FailNow(err.Error()) 51 | } 52 | assert.Equal(c.jsonString, string(b)) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pkg/core/filter.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type IFilter interface { 4 | Configure(configData map[string]interface{}) error 5 | Filter(msg *Msg) (continueNext bool, err error) 6 | Close() error 7 | } 8 | 9 | type IFilterFactory interface { 10 | NewFilter() IFilter 11 | } 12 | -------------------------------------------------------------------------------- /pkg/core/input.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "github.com/moiot/gravity/pkg/config" 5 | "github.com/moiot/gravity/pkg/position_cache" 6 | "github.com/moiot/gravity/pkg/position_repos" 7 | ) 8 | 9 | type Input interface { 10 | Start(emitter Emitter, router Router, positionCache position_cache.PositionCacheInterface) error 11 | Close() 12 | Stage() config.InputMode 13 | Done() chan position_repos.Position 14 | SendDeadSignal() error // for test only 15 | Wait() 16 | } 17 | -------------------------------------------------------------------------------- /pkg/core/matcher.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type IMatcher interface { 4 | Configure(arg interface{}) error 5 | Match(msg *Msg) bool 6 | } 7 | 8 | type IMatcherFactory interface { 9 | NewMatcher() IMatcher 10 | } 11 | 12 | type IMatcherGroup []IMatcher 13 | 14 | // Match returns true if all matcher returns true 15 | func (matcherGroup IMatcherGroup) Match(msg *Msg) bool { 16 | for _, m := range matcherGroup { 17 | if !m.Match(msg) { 18 | return false 19 | } 20 | } 21 | return true 22 | } 23 | -------------------------------------------------------------------------------- /pkg/core/output.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type Output interface { 4 | Execute(msgs []*Msg) error 5 | GetRouter() Router 6 | Close() 7 | } 8 | 9 | type SynchronousOutput interface { 10 | Start() error 11 | Output 12 | } 13 | 14 | type AsynchronousOutput interface { 15 | Start(msgAcker MsgAcker) error 16 | Output 17 | } 18 | -------------------------------------------------------------------------------- /pkg/core/position_cache_creator.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import "github.com/moiot/gravity/pkg/position_cache" 4 | 5 | type PositionCacheCreator interface { 6 | NewPositionCache() (position_cache.PositionCacheInterface, error) 7 | } 8 | -------------------------------------------------------------------------------- /pkg/core/router.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type Router interface { 4 | Exists(msg *Msg) bool 5 | } 6 | 7 | type EmptyRouter struct{} 8 | 9 | func (EmptyRouter) Exists(msg *Msg) bool { 10 | return true 11 | } 12 | -------------------------------------------------------------------------------- /pkg/core/scheduler.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import "github.com/moiot/gravity/pkg/sliding_window" 4 | 5 | type Scheduler interface { 6 | MsgSubmitter 7 | MsgAcker 8 | Healthy() bool 9 | Watermarks() map[string]sliding_window.Watermark 10 | Start(output Output) error 11 | Close() 12 | } 13 | -------------------------------------------------------------------------------- /pkg/elasticsearch_test/test.go: -------------------------------------------------------------------------------- 1 | package elasticsearch_test 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | 7 | "github.com/olivere/elastic" 8 | ) 9 | 10 | func TestURLs() []string { 11 | urls, ok := os.LookupEnv("ELASTICSEARCH_URLS") 12 | if !ok { 13 | return []string{"http://127.0.0.1:9200"} 14 | } 15 | return strings.Split(urls, ",") 16 | } 17 | 18 | func CreateTestClient() (*elastic.Client, error) { 19 | return elastic.NewClient(elastic.SetSniff(false), elastic.SetURL(TestURLs()...)) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/env/env.go: -------------------------------------------------------------------------------- 1 | package env 2 | 3 | import "time" 4 | 5 | func init() { 6 | StartTime = time.Now() 7 | } 8 | 9 | var StartTime time.Time 10 | 11 | var PipelineName string 12 | -------------------------------------------------------------------------------- /pkg/filters/accept_filter.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | "github.com/moiot/gravity/pkg/registry" 8 | ) 9 | 10 | const AcceptFilterName = "accept" 11 | 12 | type acceptFilterType struct { 13 | BaseFilter 14 | } 15 | 16 | func (f *acceptFilterType) Configure(configData map[string]interface{}) error { 17 | err := f.ConfigureMatchers(configData) 18 | if err != nil { 19 | return errors.Trace(err) 20 | } 21 | return nil 22 | } 23 | 24 | func (f *acceptFilterType) Filter(msg *core.Msg) (continueNext bool, err error) { 25 | if f.Matchers.Match(msg) { 26 | return true, nil 27 | } 28 | return false, nil 29 | } 30 | 31 | func (f *acceptFilterType) Close() error { 32 | return nil 33 | } 34 | 35 | type acceptFilterFactoryType struct{} 36 | 37 | func (factory *acceptFilterFactoryType) Configure(_ string, _ map[string]interface{}) error { 38 | return nil 39 | } 40 | 41 | func (factory *acceptFilterFactoryType) NewFilter() core.IFilter { 42 | return &acceptFilterType{} 43 | } 44 | 45 | func init() { 46 | registry.RegisterPlugin(registry.FilterPlugin, AcceptFilterName, &acceptFilterFactoryType{}, true) 47 | } 48 | -------------------------------------------------------------------------------- /pkg/filters/accept_filter_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/moiot/gravity/pkg/config" 9 | "github.com/moiot/gravity/pkg/matchers" 10 | 11 | "github.com/moiot/gravity/pkg/core" 12 | ) 13 | 14 | func Test_acceptFilterType_Filter(t *testing.T) { 15 | a := assert.New(t) 16 | 17 | data := []config.GenericPluginConfig{ 18 | { 19 | Type: AcceptFilterName, 20 | Config: map[string]interface{}{ 21 | matchers.SchemaMatcherName: "test_db", 22 | matchers.TableMatcherName: "test_table_1", 23 | }, 24 | }, 25 | } 26 | 27 | filters, err := NewFilters(data) 28 | if err != nil { 29 | a.FailNow(err.Error()) 30 | } 31 | 32 | a.Equal(1, len(filters)) 33 | 34 | f1 := filters[0] 35 | cases := []struct { 36 | name string 37 | msg core.Msg 38 | continueNext bool 39 | }{ 40 | {"test_table_1 accepted", core.Msg{Database: "test_db", Table: "test_table_1"}, true}, 41 | {"other_table not accepted", core.Msg{Database: "test_db", Table: "other_table"}, false}, 42 | } 43 | for _, c := range cases { 44 | continueNext, err := f1.Filter(&c.msg) 45 | if err != nil { 46 | a.FailNow(err.Error()) 47 | } 48 | a.Equalf(c.continueNext, continueNext, c.name) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /pkg/filters/base_filter.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | "github.com/moiot/gravity/pkg/matchers" 8 | ) 9 | 10 | type BaseFilter struct { 11 | Matchers core.IMatcherGroup 12 | } 13 | 14 | func (baseFilter *BaseFilter) ConfigureMatchers(configData map[string]interface{}) error { 15 | // find Matchers based on configData 16 | retMatchers, err := matchers.NewMatchers(configData) 17 | if err != nil { 18 | return errors.Trace(err) 19 | } 20 | baseFilter.Matchers = retMatchers 21 | 22 | if len(baseFilter.Matchers) == 0 { 23 | return errors.Errorf("no matcher configured for this filter. config: %v", configData) 24 | } 25 | return nil 26 | } 27 | 28 | func (baseFilter *BaseFilter) MatchMsg(msg *core.Msg) bool { 29 | return baseFilter.Matchers.Match(msg) 30 | } 31 | -------------------------------------------------------------------------------- /pkg/filters/delete_dml_column_filter.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | "github.com/moiot/gravity/pkg/registry" 8 | "github.com/moiot/gravity/pkg/utils" 9 | ) 10 | 11 | // [[filters]] 12 | // type = "delete-dml-column" 13 | // [[filters.config]] 14 | // match-schema = "test" 15 | // match-table = "test_table" 16 | // columns = ["e", "f"] 17 | 18 | const DeleteDMLColumnFilterName = "delete-dml-column" 19 | 20 | type deleteDmlColumnFilter struct { 21 | BaseFilter 22 | columns []string 23 | } 24 | 25 | func (f *deleteDmlColumnFilter) Configure(data map[string]interface{}) error { 26 | err := f.ConfigureMatchers(data) 27 | if err != nil { 28 | return errors.Trace(err) 29 | } 30 | 31 | columns, ok := data["columns"] 32 | if !ok { 33 | return errors.Errorf("'column' is not configured") 34 | } 35 | 36 | // columns can be any type of slice, for example: 37 | // []interface{}, []string{} 38 | c, ok := utils.CastToSlice(columns) 39 | if !ok { 40 | return errors.Errorf("'column' should be an array") 41 | } 42 | 43 | columnStrings, err := utils.CastSliceInterfaceToSliceString(c) 44 | if err != nil { 45 | return errors.Errorf("'column' should be an array of string") 46 | } 47 | 48 | f.columns = columnStrings 49 | return nil 50 | } 51 | 52 | func (f *deleteDmlColumnFilter) Filter(msg *core.Msg) (continueNext bool, err error) { 53 | if !f.Matchers.Match(msg) { 54 | return true, nil 55 | } 56 | 57 | if msg.DmlMsg == nil { 58 | return true, nil 59 | } 60 | 61 | for _, name := range f.columns { 62 | delete(msg.DmlMsg.Data, name) 63 | 64 | if msg.DmlMsg.Old != nil { 65 | delete(msg.DmlMsg.Old, name) 66 | } 67 | 68 | if msg.DmlMsg.Pks != nil { 69 | delete(msg.DmlMsg.Pks, name) 70 | } 71 | 72 | } 73 | 74 | return true, nil 75 | } 76 | 77 | func (f *deleteDmlColumnFilter) Close() error { 78 | return nil 79 | } 80 | 81 | type deleteDMLColumnFilterFactoryType struct{} 82 | 83 | func (factory *deleteDMLColumnFilterFactoryType) Configure(_ string, _ map[string]interface{}) error { 84 | return nil 85 | } 86 | 87 | func (factory *deleteDMLColumnFilterFactoryType) NewFilter() core.IFilter { 88 | return &deleteDmlColumnFilter{} 89 | } 90 | 91 | func init() { 92 | registry.RegisterPlugin(registry.FilterPlugin, DeleteDMLColumnFilterName, &deleteDMLColumnFilterFactoryType{}, true) 93 | } 94 | -------------------------------------------------------------------------------- /pkg/filters/delete_dml_column_filter_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/juju/errors" 7 | "github.com/moiot/gravity/pkg/core" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func newDeleteDmlColumnFilter(s string) (core.IFilter, error) { 12 | cfg, err := newFilterConfigFromJson(s) 13 | if err != nil { 14 | return nil, errors.Trace(err) 15 | } 16 | 17 | f := &deleteDmlColumnFilter{} 18 | if err := f.Configure(cfg); err != nil { 19 | return nil, errors.Trace(err) 20 | } 21 | 22 | return f, nil 23 | } 24 | 25 | func TestDeleteDmlColumnFilter_Configure(t *testing.T) { 26 | r := require.New(t) 27 | 28 | _, err := newFilterConfigFromJson(` 29 | { 30 | "match-schema": "test", 31 | "match-table": "test_table", 32 | "columns": ["a", "b"] 33 | } 34 | `) 35 | r.NoError(err) 36 | } 37 | 38 | func TestDeleteDmlColumnFilter_Filter(t *testing.T) { 39 | 40 | r := require.New(t) 41 | 42 | f, err := newDeleteDmlColumnFilter(` 43 | { 44 | "match-schema": "test", 45 | "match-table": "test_table", 46 | "columns": ["b", "a", "d"] 47 | } 48 | `) 49 | r.NoError(err) 50 | 51 | msg := core.Msg{ 52 | Type: core.MsgDML, 53 | Database: "test", 54 | Table: "test_table", 55 | DmlMsg: &core.DMLMsg{ 56 | Data: map[string]interface{}{ 57 | "a": 1, 58 | "b": 2, 59 | "c": 3, 60 | "d": 4, 61 | }, 62 | Old: map[string]interface{}{ 63 | "a": 10, 64 | "b": 20, 65 | "c": 30, 66 | "d": 40, 67 | }, 68 | Pks: map[string]interface{}{ 69 | "a": 1, 70 | "b": 2, 71 | "c": 3, 72 | }, 73 | }, 74 | } 75 | 76 | c, err := f.Filter(&msg) 77 | r.True(c) 78 | r.NoError(err) 79 | 80 | // Data 81 | _, ok := msg.DmlMsg.Data["a"] 82 | r.False(ok) 83 | 84 | _, ok = msg.DmlMsg.Data["b"] 85 | r.False(ok) 86 | 87 | _, ok = msg.DmlMsg.Data["c"] 88 | r.True(ok) 89 | 90 | // Old 91 | _, ok = msg.DmlMsg.Old["a"] 92 | r.False(ok) 93 | 94 | _, ok = msg.DmlMsg.Old["b"] 95 | r.False(ok) 96 | 97 | _, ok = msg.DmlMsg.Old["c"] 98 | r.True(ok) 99 | 100 | // Pks 101 | r.Equal(1, len(msg.DmlMsg.Pks)) 102 | 103 | _, ok = msg.DmlMsg.Pks["a"] 104 | r.False(ok) 105 | 106 | _, ok = msg.DmlMsg.Pks["c"] 107 | r.True(ok) 108 | 109 | _, ok = msg.DmlMsg.Pks["d"] 110 | r.False(ok) 111 | 112 | } 113 | -------------------------------------------------------------------------------- /pkg/filters/filters.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "reflect" 5 | 6 | "github.com/juju/errors" 7 | 8 | "github.com/moiot/gravity/pkg/config" 9 | "github.com/moiot/gravity/pkg/core" 10 | "github.com/moiot/gravity/pkg/registry" 11 | ) 12 | 13 | func NewFilters(filterConfigs []config.GenericPluginConfig) ([]core.IFilter, error) { 14 | var retFilters []core.IFilter 15 | for _, c := range filterConfigs { 16 | if c.Type == "go-native-plugin" { 17 | name, p, err := registry.DownloadGoNativePlugin(c.Config) 18 | if err != nil { 19 | return nil, errors.Trace(err) 20 | } 21 | registry.RegisterPlugin(registry.FilterPlugin, name, p, true) 22 | } 23 | 24 | factory, err := registry.GetPlugin(registry.FilterPlugin, c.Type) 25 | if err != nil { 26 | return nil, errors.Trace(err) 27 | } 28 | 29 | filterFactory, ok := factory.(core.IFilterFactory) 30 | if !ok { 31 | return nil, errors.Errorf("wrong type: %v", reflect.TypeOf(factory)) 32 | } 33 | 34 | f := filterFactory.NewFilter() 35 | 36 | if err := f.Configure(c.Config); err != nil { 37 | return nil, errors.Trace(err) 38 | } 39 | 40 | retFilters = append(retFilters, f) 41 | } 42 | return retFilters, nil 43 | } 44 | -------------------------------------------------------------------------------- /pkg/filters/grpc/server.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * // Copyright 2019 , Beijing Mobike Technology Co., Ltd. 4 | * // 5 | * // Licensed under the Apache License, Version 2.0 (the "License"); 6 | * // you may not use this file except in compliance with the License. 7 | * // You may obtain a copy of the License at 8 | * // 9 | * // http://www.apache.org/licenses/LICENSE-2.0 10 | * // 11 | * // Unless required by applicable law or agreed to in writing, software 12 | * // distributed under the License is distributed on an "AS IS" BASIS, 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package grpc 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/juju/errors" 23 | "github.com/moiot/gravity/pkg/core/encoding" 24 | 25 | "github.com/moiot/gravity/pkg/core" 26 | "github.com/moiot/gravity/pkg/protocol/msgpb" 27 | ) 28 | 29 | type GRPCServer struct { 30 | Impl core.IFilter 31 | } 32 | 33 | func (m *GRPCServer) Configure(ctx context.Context, req *msgpb.ConfigureRequest) (*msgpb.ConfigureResponse, error) { 34 | rsp := msgpb.ConfigureResponse{} 35 | 36 | data, err := encoding.PBToDataMap(req.Data) 37 | if err != nil { 38 | return nil, errors.Trace(err) 39 | } 40 | 41 | if err := m.Impl.Configure(data); err != nil { 42 | return nil, errors.Trace(err) 43 | } 44 | 45 | return &rsp, nil 46 | 47 | } 48 | 49 | func (m *GRPCServer) Filter(ctx context.Context, req *msgpb.FilterRequest) (*msgpb.FilterResponse, error) { 50 | rsp := msgpb.FilterResponse{} 51 | 52 | msg, err := encoding.DecodeMsgFromPB(req.Msg) 53 | if err != nil { 54 | return nil, errors.Trace(err) 55 | } 56 | 57 | continueNext, err := m.Impl.Filter(msg) 58 | if err != nil { 59 | return nil, errors.Trace(err) 60 | } 61 | 62 | pbmsg, err := encoding.EncodeMsgToPB(msg) 63 | if err != nil { 64 | return nil, errors.Trace(err) 65 | } 66 | 67 | rsp.ContinueNext = continueNext 68 | rsp.Msg = pbmsg 69 | return &rsp, nil 70 | } 71 | -------------------------------------------------------------------------------- /pkg/filters/grpc/shared.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * // Copyright 2019 , Beijing Mobike Technology Co., Ltd. 4 | * // 5 | * // Licensed under the Apache License, Version 2.0 (the "License"); 6 | * // you may not use this file except in compliance with the License. 7 | * // You may obtain a copy of the License at 8 | * // 9 | * // http://www.apache.org/licenses/LICENSE-2.0 10 | * // 11 | * // Unless required by applicable law or agreed to in writing, software 12 | * // distributed under the License is distributed on an "AS IS" BASIS, 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package grpc 18 | 19 | import ( 20 | "context" 21 | 22 | hplugin "github.com/hashicorp/go-plugin" 23 | 24 | "github.com/moiot/gravity/pkg/core" 25 | "github.com/moiot/gravity/pkg/protocol/msgpb" 26 | "google.golang.org/grpc" 27 | ) 28 | 29 | const PluginName = "filter_grpc" 30 | 31 | var ( 32 | HandshakeConfig = hplugin.HandshakeConfig{ 33 | ProtocolVersion: 1, 34 | MagicCookieKey: "BASIC_PLUGIN", 35 | MagicCookieValue: "hello", 36 | } 37 | 38 | PluginMap = map[string]hplugin.Plugin{ 39 | PluginName: &FilterGRPCPlugin{}, 40 | } 41 | ) 42 | 43 | type FilterGRPCPlugin struct { 44 | hplugin.Plugin 45 | Impl core.IFilter 46 | } 47 | 48 | func (p *FilterGRPCPlugin) GRPCServer(broker *hplugin.GRPCBroker, s *grpc.Server) error { 49 | msgpb.RegisterFilterPluginServer(s, &GRPCServer{Impl: p.Impl}) 50 | return nil 51 | } 52 | 53 | func (p *FilterGRPCPlugin) GRPCClient(ctx context.Context, broker *hplugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { 54 | return &GRPCClient{client: msgpb.NewFilterPluginClient(c)}, nil 55 | } 56 | -------------------------------------------------------------------------------- /pkg/filters/grpc_sidecar_filter_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * // Copyright 2019 , Beijing Mobike Technology Co., Ltd. 4 | * // 5 | * // Licensed under the Apache License, Version 2.0 (the "License"); 6 | * // you may not use this file except in compliance with the License. 7 | * // You may obtain a copy of the License at 8 | * // 9 | * // http://www.apache.org/licenses/LICENSE-2.0 10 | * // 11 | * // Unless required by applicable law or agreed to in writing, software 12 | * // distributed under the License is distributed on an "AS IS" BASIS, 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package filters 18 | 19 | import ( 20 | "runtime" 21 | "testing" 22 | 23 | "github.com/moiot/gravity/pkg/config" 24 | "github.com/moiot/gravity/pkg/core" 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func TestGrpcFilterFactoryType(t *testing.T) { 29 | r := require.New(t) 30 | 31 | var binaryURL string 32 | if runtime.GOOS == "darwin" { 33 | binaryURL = "https://github.com/moiot/gravity-grpc-sidecar-filter-example/releases/download/v0.2/gravity-grpc-sidecar-filter-example.darwin" 34 | } else if runtime.GOOS == "linux" { 35 | binaryURL = "https://github.com/moiot/gravity-grpc-sidecar-filter-example/releases/download/v0.2/gravity-grpc-sidecar-filter-example.linux" 36 | } else { 37 | r.FailNow("runtime not supported") 38 | } 39 | data := []config.GenericPluginConfig{ 40 | { 41 | Type: "grpc-sidecar", 42 | Config: map[string]interface{}{ 43 | "binary-url": binaryURL, 44 | "name": "test-grpc-sidecar", 45 | "match-schema": "test", 46 | }, 47 | }, 48 | } 49 | 50 | filters, err := NewFilters(data) 51 | r.NoError(err) 52 | 53 | defer func() { 54 | for _, f := range filters { 55 | f.Close() 56 | } 57 | }() 58 | 59 | r.Equal(1, len(filters)) 60 | 61 | f1 := filters[0] 62 | 63 | msg := core.Msg{ 64 | Database: "test", 65 | Type: core.MsgDML, 66 | DmlMsg: &core.DMLMsg{ 67 | Data: map[string]interface{}{ 68 | "a": 1, 69 | "b": 2, 70 | "c": 3, 71 | }, 72 | }, 73 | } 74 | 75 | continueNext, err := f1.Filter(&msg) 76 | r.NoError(err) 77 | r.True(continueNext) 78 | 79 | for _, v := range msg.DmlMsg.Data { 80 | r.Equal("hello grpc", v) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /pkg/filters/reject_filter.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | "github.com/moiot/gravity/pkg/registry" 8 | ) 9 | 10 | const RejectFilterName = "reject" 11 | 12 | type rejectFilterType struct { 13 | BaseFilter 14 | } 15 | 16 | func (f *rejectFilterType) Configure(configData map[string]interface{}) error { 17 | err := f.ConfigureMatchers(configData) 18 | if err != nil { 19 | return errors.Trace(err) 20 | } 21 | // we can add more validation for filter's args here. 22 | return nil 23 | } 24 | 25 | func (f *rejectFilterType) Filter(msg *core.Msg) (continueNext bool, err error) { 26 | if f.Matchers.Match(msg) { 27 | return false, nil 28 | } 29 | 30 | return true, nil 31 | } 32 | 33 | func (f *rejectFilterType) Close() error { 34 | return nil 35 | } 36 | 37 | type rejectFilterFactoryType struct{} 38 | 39 | func (factory *rejectFilterFactoryType) Configure(_ string, _ map[string]interface{}) error { 40 | return nil 41 | } 42 | 43 | func (factory *rejectFilterFactoryType) NewFilter() core.IFilter { 44 | return &rejectFilterType{} 45 | } 46 | 47 | func init() { 48 | registry.RegisterPlugin(registry.FilterPlugin, RejectFilterName, &rejectFilterFactoryType{}, true) 49 | } 50 | -------------------------------------------------------------------------------- /pkg/filters/reject_filter_test.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/moiot/gravity/pkg/config" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/moiot/gravity/pkg/core" 11 | ) 12 | 13 | func TestRejectFilter(t *testing.T) { 14 | assert := assert.New(t) 15 | 16 | data := []config.GenericPluginConfig{ 17 | { 18 | Type: "reject", 19 | Config: map[string]interface{}{ 20 | "match-schema": "test_db", 21 | "match-table": "test_table_1", 22 | }, 23 | }, 24 | } 25 | 26 | filters, err := NewFilters(data) 27 | if err != nil { 28 | assert.FailNow(err.Error()) 29 | } 30 | 31 | assert.Equal(1, len(filters)) 32 | 33 | f1 := filters[0] 34 | cases := []struct { 35 | name string 36 | msg core.Msg 37 | continueNext bool 38 | }{ 39 | {"test_table_1 rejected", core.Msg{Database: "test_db", Table: "test_table_1"}, false}, 40 | {"other_table not rejected", core.Msg{Database: "test_db", Table: "other_table"}, true}, 41 | } 42 | for _, c := range cases { 43 | continueNext, err := f1.Filter(&c.msg) 44 | if err != nil { 45 | assert.FailNow(err.Error()) 46 | } 47 | assert.Equalf(c.continueNext, continueNext, c.name) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pkg/filters/utils.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "encoding/json" 5 | ) 6 | 7 | func newFilterConfigFromJson(s string) (map[string]interface{}, error) { 8 | cfg := make(map[string]interface{}) 9 | err := json.Unmarshal([]byte(s), &cfg) 10 | return cfg, err 11 | } 12 | -------------------------------------------------------------------------------- /pkg/inputs/dump_input.go: -------------------------------------------------------------------------------- 1 | package inputs 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | "github.com/mitchellh/mapstructure" 6 | "github.com/moiot/gravity/pkg/position_repos" 7 | 8 | "github.com/moiot/gravity/pkg/config" 9 | "github.com/moiot/gravity/pkg/core" 10 | "github.com/moiot/gravity/pkg/position_cache" 11 | 12 | "github.com/moiot/gravity/pkg/registry" 13 | ) 14 | 15 | type pluginConfig struct { 16 | TestKey1 string `mapstructure:"test-key-1"` 17 | } 18 | 19 | type dumpInput struct { 20 | pipelineName string 21 | cfg *pluginConfig 22 | } 23 | 24 | func (plugin *dumpInput) Configure(pipelineName string, data map[string]interface{}) error { 25 | cfg := pluginConfig{} 26 | if err := mapstructure.Decode(data, &cfg); err != nil { 27 | return errors.Trace(err) 28 | } 29 | return nil 30 | } 31 | 32 | func (plugin *dumpInput) NewPositionStore() (position_cache.PositionCacheInterface, error) { 33 | return nil, nil 34 | } 35 | 36 | func (plugin *dumpInput) Start(emitter core.Emitter, router core.Router, positionCache position_cache.PositionCacheInterface) error { 37 | return nil 38 | } 39 | 40 | func (plugin *dumpInput) Close() { 41 | 42 | } 43 | 44 | func (plugin *dumpInput) Stage() config.InputMode { 45 | return config.Stream 46 | } 47 | 48 | func (plugin *dumpInput) SendDeadSignal() error { 49 | return nil 50 | } 51 | 52 | func (plugin *dumpInput) Done(positionCache position_cache.PositionCacheInterface) chan position_repos.Position { 53 | return make(chan position_repos.Position) 54 | } 55 | 56 | func (plugin *dumpInput) Wait() { 57 | 58 | } 59 | 60 | func init() { 61 | registry.RegisterPlugin(registry.InputPlugin, "dump-input", &dumpInput{}, false) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/inputs/helper/mysql_common.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "fmt" 5 | 6 | jsoniter "github.com/json-iterator/go" 7 | 8 | "github.com/moiot/gravity/pkg/config" 9 | 10 | "github.com/juju/errors" 11 | ) 12 | 13 | var myJson = jsoniter.Config{SortMapKeys: true}.Froze() 14 | 15 | type SourceProbeCfg struct { 16 | SourceMySQL *config.DBConfig `mapstructure:"mysql"json:"mysql"` 17 | Annotation string `mapstructure:"annotation"json:"annotation"` 18 | } 19 | 20 | type BinlogPositionsValue struct { 21 | CurrentPosition *config.MySQLBinlogPosition `json:"current_position"` 22 | StartPosition *config.MySQLBinlogPosition `json:"start_position"` 23 | } 24 | 25 | func BinlogPositionValueEncoder(v interface{}) (string, error) { 26 | return myJson.MarshalToString(v) 27 | } 28 | 29 | func BinlogPositionValueDecoder(s string) (interface{}, error) { 30 | return DeserializeBinlogPositionValue(s) 31 | } 32 | 33 | func SerializeBinlogPositionValue(position BinlogPositionsValue) (string, error) { 34 | return BinlogPositionValueEncoder(position) 35 | } 36 | 37 | func DeserializeBinlogPositionValue(value string) (BinlogPositionsValue, error) { 38 | position := BinlogPositionsValue{} 39 | if err := myJson.UnmarshalFromString(value, &position); err != nil { 40 | return BinlogPositionsValue{}, errors.Trace(err) 41 | } 42 | return position, nil 43 | } 44 | 45 | func GetProbCfg(sourceProbeCfg *SourceProbeCfg, sourceDBCfg *config.DBConfig) (*config.DBConfig, string) { 46 | var probeDBCfg config.DBConfig 47 | var probeAnnotation string 48 | 49 | if sourceProbeCfg != nil { 50 | if sourceProbeCfg.SourceMySQL != nil { 51 | probeDBCfg = *sourceProbeCfg.SourceMySQL 52 | } else { 53 | probeDBCfg = *sourceDBCfg 54 | } 55 | probeAnnotation = sourceProbeCfg.Annotation 56 | } else { 57 | probeDBCfg = *sourceDBCfg 58 | probeDBCfg.MaxIdle = 1 59 | probeDBCfg.MaxOpen = 5 60 | } 61 | 62 | if probeAnnotation != "" { 63 | probeAnnotation = fmt.Sprintf("/*%s*/", probeAnnotation) 64 | } 65 | return &probeDBCfg, probeAnnotation 66 | } 67 | -------------------------------------------------------------------------------- /pkg/inputs/helper/mysql_common_test.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/moiot/gravity/pkg/config" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestSerializeBinlogPositions(t *testing.T) { 13 | r := require.New(t) 14 | gtidSmall := "123edf:1-10" 15 | gtidBig := "23456:1-20" 16 | 17 | position := BinlogPositionsValue{ 18 | CurrentPosition: &config.MySQLBinlogPosition{BinlogGTID: fmt.Sprintf("%s,%s", gtidBig, gtidSmall)}, 19 | } 20 | 21 | v, err := SerializeBinlogPositionValue(position) 22 | r.NoError(err) 23 | 24 | p2, err := DeserializeBinlogPositionValue(v) 25 | r.NoError(err) 26 | r.Equal(fmt.Sprintf("%s,%s", gtidBig, gtidSmall), p2.CurrentPosition.BinlogGTID) 27 | r.Nil(p2.StartPosition) 28 | } 29 | -------------------------------------------------------------------------------- /pkg/inputs/helper/two_stage_input_test.go: -------------------------------------------------------------------------------- 1 | package helper 2 | -------------------------------------------------------------------------------- /pkg/inputs/helper/util.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | "github.com/moiot/gravity/pkg/registry" 8 | ) 9 | 10 | func GetInputFromPlugin(pluginName string, pipelineName string, data map[string]interface{}) (core.Input, error) { 11 | plugin, err := registry.GetPlugin(registry.InputPlugin, pluginName) 12 | if err != nil { 13 | return nil, errors.Trace(err) 14 | } 15 | err = plugin.Configure(pipelineName, data) 16 | if err != nil { 17 | return nil, errors.Trace(err) 18 | } 19 | return plugin.(core.Input), nil 20 | } 21 | -------------------------------------------------------------------------------- /pkg/inputs/mongo/input.go: -------------------------------------------------------------------------------- 1 | package mongo 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | log "github.com/sirupsen/logrus" 6 | 7 | "github.com/moiot/gravity/pkg/config" 8 | "github.com/moiot/gravity/pkg/core" 9 | "github.com/moiot/gravity/pkg/inputs/helper" 10 | "github.com/moiot/gravity/pkg/inputs/mongobatch" 11 | "github.com/moiot/gravity/pkg/inputs/mongostream" 12 | "github.com/moiot/gravity/pkg/position_cache" 13 | "github.com/moiot/gravity/pkg/registry" 14 | ) 15 | 16 | // TODO: remove duplicate with inputs/mysql 17 | const Name = "mongo" 18 | 19 | func init() { 20 | registry.RegisterPlugin(registry.InputPlugin, Name, &input{}, false) 21 | } 22 | 23 | type input struct { 24 | core.Input 25 | } 26 | 27 | func (i *input) Configure(pipelineName string, data map[string]interface{}) error { 28 | mode := data["mode"] 29 | if mode == nil { 30 | return errors.Errorf("mongo input should have mode") 31 | } 32 | 33 | var err error 34 | 35 | switch mode.(config.InputMode) { 36 | case config.Batch: 37 | i.Input, err = helper.GetInputFromPlugin(mongobatch.Name, pipelineName, data) 38 | if err != nil { 39 | return errors.Trace(err) 40 | } 41 | 42 | case config.Stream: 43 | i.Input, err = helper.GetInputFromPlugin(mongostream.Name, pipelineName, data) 44 | if err != nil { 45 | return errors.Trace(err) 46 | } 47 | 48 | case config.Replication: 49 | batch, err := helper.GetInputFromPlugin(mongobatch.Name, pipelineName, data) 50 | if err != nil { 51 | return errors.Trace(err) 52 | } 53 | 54 | stream, err := helper.GetInputFromPlugin(mongostream.Name, pipelineName, data) 55 | if err != nil { 56 | return errors.Trace(err) 57 | } 58 | 59 | i.Input, err = helper.NewTwoStageInputPlugin(batch, stream) 60 | if err != nil { 61 | return errors.Trace(err) 62 | } 63 | 64 | default: 65 | log.Panic("unknown mode ", mode) 66 | } 67 | 68 | return nil 69 | } 70 | 71 | func (i *input) NewPositionCache() (position_cache.PositionCacheInterface, error) { 72 | newer, ok := i.Input.(core.PositionCacheCreator) 73 | if !ok { 74 | return nil, errors.Errorf("not a PositionCacheCreator") 75 | } 76 | 77 | return newer.NewPositionCache() 78 | } 79 | -------------------------------------------------------------------------------- /pkg/inputs/mysql/input.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | log "github.com/sirupsen/logrus" 6 | 7 | "github.com/moiot/gravity/pkg/config" 8 | "github.com/moiot/gravity/pkg/core" 9 | "github.com/moiot/gravity/pkg/inputs/helper" 10 | "github.com/moiot/gravity/pkg/inputs/mysqlbatch" 11 | "github.com/moiot/gravity/pkg/inputs/mysqlstream" 12 | "github.com/moiot/gravity/pkg/position_cache" 13 | "github.com/moiot/gravity/pkg/registry" 14 | ) 15 | 16 | const Name = "mysql" 17 | 18 | func init() { 19 | registry.RegisterPlugin(registry.InputPlugin, Name, &input{}, false) 20 | } 21 | 22 | type input struct { 23 | core.Input 24 | } 25 | 26 | func (i *input) Configure(pipelineName string, data map[string]interface{}) error { 27 | mode := data["mode"] 28 | if mode == nil { 29 | return errors.Errorf("mysql input should have mode %s, %s or %s", config.Batch, config.Stream, config.Replication) 30 | } 31 | 32 | var err error 33 | 34 | switch mode.(config.InputMode) { 35 | case config.Batch: 36 | i.Input, err = helper.GetInputFromPlugin(mysqlbatch.Name, pipelineName, data) 37 | if err != nil { 38 | return errors.Trace(err) 39 | } 40 | 41 | case config.Stream: 42 | i.Input, err = helper.GetInputFromPlugin(mysqlstream.Name, pipelineName, data) 43 | if err != nil { 44 | return errors.Trace(err) 45 | } 46 | 47 | case config.Replication: 48 | scan, err := helper.GetInputFromPlugin(mysqlbatch.Name, pipelineName, data) 49 | if err != nil { 50 | return errors.Trace(err) 51 | } 52 | 53 | binlog, err := helper.GetInputFromPlugin(mysqlstream.Name, pipelineName, data) 54 | if err != nil { 55 | return errors.Trace(err) 56 | } 57 | 58 | i.Input, err = helper.NewTwoStageInputPlugin(scan, binlog) 59 | if err != nil { 60 | return errors.Trace(err) 61 | } 62 | 63 | default: 64 | log.Panic("unknown mode ", mode) 65 | } 66 | 67 | return nil 68 | } 69 | 70 | func (i *input) NewPositionCache() (position_cache.PositionCacheInterface, error) { 71 | newer, ok := i.Input.(core.PositionCacheCreator) 72 | if !ok { 73 | return nil, errors.Errorf("not a PositionCacheCreator") 74 | } 75 | 76 | return newer.NewPositionCache() 77 | } 78 | -------------------------------------------------------------------------------- /pkg/inputs/mysqlbatch/input_test.go: -------------------------------------------------------------------------------- 1 | package mysqlbatch 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/moiot/gravity/pkg/mysql_test" 9 | "github.com/moiot/gravity/pkg/utils" 10 | 11 | "github.com/mitchellh/mapstructure" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestStringOrStringSlice(t *testing.T) { 16 | s := `[{"schema": "t", "table":"123"}, {"schema":"t", "table":["1","2","3"]}]` 17 | var m []map[string]interface{} 18 | err := json.Unmarshal([]byte(s), &m) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | var ret []TableConfig 24 | err = mapstructure.WeakDecode(m, &ret) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | 29 | fmt.Println(ret) 30 | require.Equal(t, []string{"123"}, ret[0].Table) 31 | require.Equal(t, []string{"1", "2", "3"}, ret[1].Table) 32 | } 33 | 34 | func TestDetectScanColumn(t *testing.T) { 35 | r := require.New(t) 36 | t.Run("composite primary", func(tt *testing.T) { 37 | testDBName := utils.TestCaseMd5Name(tt) 38 | db := mysql_test.MustSetupSourceDB(testDBName) 39 | 40 | col, err := DetectScanColumns(db, testDBName, mysql_test.TestScanColumnTableCompositePrimary, 1000, 10000) 41 | r.NoError(err) 42 | r.Equal([]string{"id", "name"}, col) 43 | }) 44 | t.Run("single the primary", func(tt *testing.T) { 45 | testDBName := utils.TestCaseMd5Name(tt) 46 | db := mysql_test.MustSetupSourceDB(testDBName) 47 | col, err := DetectScanColumns(db, testDBName, mysql_test.TestScanColumnTableIdPrimary, 1000, 10000) 48 | r.Nil(err) 49 | r.Equal([]string{"id"}, col) 50 | }) 51 | 52 | t.Run("uniq index", func(tt *testing.T) { 53 | testDBName := utils.TestCaseMd5Name(tt) 54 | db := mysql_test.MustSetupSourceDB(testDBName) 55 | col, err := DetectScanColumns(db, testDBName, mysql_test.TestScanColumnTableUniqueIndexEmailString, 1000, 10000) 56 | r.Nil(err) 57 | r.Equal([]string{"email"}, col) 58 | }) 59 | 60 | t.Run("composite unique key is not supported", func(tt *testing.T) { 61 | testDBName := utils.TestCaseMd5Name(tt) 62 | db := mysql_test.MustSetupSourceDB(testDBName) 63 | _, err := DetectScanColumns(db, testDBName, mysql_test.TestScanColumnTableCompositeUniqueKey, 1000, 900) 64 | r.NotNil(err) 65 | }) 66 | } 67 | -------------------------------------------------------------------------------- /pkg/inputs/mysqlstream/input_test.go: -------------------------------------------------------------------------------- 1 | package mysqlstream 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func Test_mysqlInputPlugin_Configure(t *testing.T) { 11 | r := require.New(t) 12 | raw := `{ 13 | "ignore-bidirectional-data": true, 14 | "source": { 15 | "host": "localhost" 16 | }, 17 | "start-position": { 18 | "binlog-gtid": "gtid" 19 | } 20 | }` 21 | 22 | var m map[string]interface{} 23 | err := json.Unmarshal([]byte(raw), &m) 24 | r.NoError(err) 25 | 26 | p := mysqlStreamInputPlugin{} 27 | err = p.Configure("123", m) 28 | r.NoError(err) 29 | 30 | cfg := p.cfg 31 | r.Equal(true, cfg.IgnoreBiDirectionalData) 32 | r.Equal("localhost", cfg.Source.Host) 33 | r.Equal("gtid", cfg.StartPosition.BinlogGTID) 34 | } 35 | -------------------------------------------------------------------------------- /pkg/inputs/mysqlstream/utils.go: -------------------------------------------------------------------------------- 1 | package mysqlstream 2 | 3 | import ( 4 | "github.com/pingcap/parser" 5 | "github.com/pingcap/parser/ast" 6 | _ "github.com/pingcap/tidb/types/parser_driver" 7 | "github.com/siddontang/go-mysql/replication" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | func IsEventBelongsToMyself(event *replication.RowsEvent, pipelineName string) bool { 12 | if id, ok := event.Rows[0][0].(string); ok { 13 | if id == pipelineName { 14 | return true 15 | } 16 | return false 17 | } 18 | panic("type conversion failed for internal table") 19 | } 20 | 21 | func extractSchemaNameFromDDLQueryEvent(p *parser.Parser, ev *replication.QueryEvent) (db, table []string, node []ast.StmtNode) { 22 | stmt, err := p.ParseOneStmt(string(ev.Query), "", "") 23 | if err != nil { 24 | log.Errorf("sql parser: %s. error: %v", string(ev.Query), err.Error()) 25 | return []string{string(ev.Schema)}, []string{""}, []ast.StmtNode{nil} 26 | } 27 | 28 | switch v := stmt.(type) { 29 | case *ast.CreateDatabaseStmt: 30 | db = append(db, v.Name) 31 | table = append(table, "") 32 | node = append(node, stmt) 33 | case *ast.DropDatabaseStmt: 34 | db = append(db, v.Name) 35 | table = append(table, "") 36 | node = append(node, stmt) 37 | case *ast.CreateTableStmt: 38 | db = append(db, v.Table.Schema.String()) 39 | table = append(table, v.Table.Name.String()) 40 | node = append(node, stmt) 41 | case *ast.DropTableStmt: 42 | for i := range v.Tables { 43 | db = append(db, v.Tables[i].Schema.String()) 44 | table = append(table, v.Tables[i].Name.String()) 45 | dropTableStmt := *v 46 | dropTableStmt.Tables = nil 47 | dropTableStmt.Tables = append(dropTableStmt.Tables, v.Tables[i]) 48 | node = append(node, &dropTableStmt) 49 | } 50 | case *ast.AlterTableStmt: 51 | db = append(db, v.Table.Schema.String()) 52 | table = append(table, v.Table.Name.String()) 53 | node = append(node, stmt) 54 | case *ast.TruncateTableStmt: 55 | db = append(db, v.Table.Schema.String()) 56 | table = append(table, v.Table.Name.String()) 57 | node = append(node, stmt) 58 | case *ast.RenameTableStmt: 59 | db = append(db, v.OldTable.Schema.String()) 60 | table = append(table, v.OldTable.Name.String()) 61 | node = append(node, stmt) 62 | default: 63 | db = append(db, "") 64 | table = append(table, "") 65 | node = append(node, stmt) 66 | } 67 | if len(db) == 1 && db[0] == "" { 68 | db[0] = string(ev.Schema) 69 | } 70 | return 71 | } 72 | -------------------------------------------------------------------------------- /pkg/inputs/plugins.go: -------------------------------------------------------------------------------- 1 | package inputs 2 | 3 | import ( 4 | "github.com/moiot/gravity/pkg/inputs/mongo" 5 | _ "github.com/moiot/gravity/pkg/inputs/mongobatch" 6 | _ "github.com/moiot/gravity/pkg/inputs/mongostream" 7 | "github.com/moiot/gravity/pkg/inputs/mysql" 8 | _ "github.com/moiot/gravity/pkg/inputs/mysqlbatch" 9 | _ "github.com/moiot/gravity/pkg/inputs/mysqlstream" 10 | _ "github.com/moiot/gravity/pkg/inputs/tidb_kafka" 11 | ) 12 | 13 | const ( 14 | Mongo = mongo.Name 15 | Mysql = mysql.Name 16 | ) 17 | -------------------------------------------------------------------------------- /pkg/inputs/tidb_kafka/position_value_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * // Copyright 2019 , Beijing Mobike Technology Co., Ltd. 4 | * // 5 | * // Licensed under the Apache License, Version 2.0 (the "License"); 6 | * // you may not use this file except in compliance with the License. 7 | * // You may obtain a copy of the License at 8 | * // 9 | * // http://www.apache.org/licenses/LICENSE-2.0 10 | * // 11 | * // Unless required by applicable law or agreed to in writing, software 12 | * // distributed under the License is distributed on an "AS IS" BASIS, 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package tidb_kafka 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | func TestKafkaPositionValueEncoder(t *testing.T) { 26 | r := require.New(t) 27 | 28 | v := KafkaPositionValue{} 29 | 30 | s, err := KafkaPositionValueEncoder(v) 31 | r.NoError(err) 32 | 33 | v2, err := KafkaPositionValueDecoder(s) 34 | r.NoError(err) 35 | newV := v2.(KafkaPositionValue) 36 | 37 | r.NotNil(newV.Offsets) 38 | r.Equal(0, len(newV.Offsets)) 39 | } 40 | -------------------------------------------------------------------------------- /pkg/inputs/tidb_kafka/utils.go: -------------------------------------------------------------------------------- 1 | package tidb_kafka 2 | 3 | func ParseTimeStamp(tso uint64) uint64 { 4 | // https://github.com/pingcap/pd/blob/master/tools/pd-ctl/pdctl/command/tso_command.go#L49 5 | // timstamp in seconds format 6 | return (tso >> 18) / 1000 7 | } 8 | -------------------------------------------------------------------------------- /pkg/kafka_test/test.go: -------------------------------------------------------------------------------- 1 | package kafka_test 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | ) 7 | 8 | func TestBroker() []string { 9 | sourceDBHost, ok := os.LookupEnv("KAFKA_BROKER") 10 | if !ok { 11 | return []string{"localhost:9092"} 12 | } 13 | return strings.Split(sourceDBHost, ",") 14 | } 15 | -------------------------------------------------------------------------------- /pkg/matchers/ddl_regex_matcher.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/juju/errors" 7 | 8 | "github.com/moiot/gravity/pkg/core" 9 | "github.com/moiot/gravity/pkg/registry" 10 | ) 11 | 12 | const DDLRegexMatcherName = "match-ddl-regex" 13 | 14 | type DDLRegexMatcher struct { 15 | reg *regexp.Regexp 16 | } 17 | 18 | func (m *DDLRegexMatcher) Configure(data interface{}) error { 19 | if arg, ok := data.(string); !ok { 20 | return errors.Errorf("match-ddl-regex only receives a string value, you provided: %v", data) 21 | } else { 22 | reg, err := regexp.Compile(arg) 23 | if err != nil { 24 | return errors.Annotatef(err, "fail to compile regexp %s", arg) 25 | } 26 | m.reg = reg 27 | } 28 | return nil 29 | } 30 | 31 | func (m *DDLRegexMatcher) Match(msg *core.Msg) bool { 32 | if msg.Type != core.MsgDDL { 33 | return false 34 | } 35 | 36 | return m.reg.Match([]byte(msg.DdlMsg.Statement)) 37 | } 38 | 39 | type ddlRegexMatcherFactoryType struct { 40 | } 41 | 42 | func (f *ddlRegexMatcherFactoryType) Configure(_ string, _ map[string]interface{}) error { 43 | return nil 44 | } 45 | 46 | func (f *ddlRegexMatcherFactoryType) NewMatcher() core.IMatcher { 47 | return &DDLRegexMatcher{} 48 | } 49 | 50 | func init() { 51 | registry.RegisterPlugin(registry.MatcherPlugin, DDLRegexMatcherName, &ddlRegexMatcherFactoryType{}, true) 52 | } 53 | -------------------------------------------------------------------------------- /pkg/matchers/ddl_regex_matcher_test.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/moiot/gravity/pkg/core" 9 | ) 10 | 11 | func TestDDLRegexMatcher(t *testing.T) { 12 | cases := []struct { 13 | name string 14 | config string 15 | msg core.Msg 16 | expected bool 17 | }{ 18 | { 19 | "ignore dml", 20 | ".*", 21 | core.Msg{ 22 | Type: core.MsgDML, 23 | }, 24 | false, 25 | }, 26 | { 27 | "matched ddl", 28 | "(?i)^DROP\\sTABLE", 29 | core.Msg{ 30 | Type: core.MsgDDL, 31 | DdlMsg: &core.DDLMsg{ 32 | Statement: "drop table t", 33 | }, 34 | }, 35 | true, 36 | }, 37 | { 38 | "unmatched ddl", 39 | "(?i)^DROP\\sTABLE", 40 | core.Msg{ 41 | Type: core.MsgDDL, 42 | DdlMsg: &core.DDLMsg{ 43 | Statement: "alter table t add column i int(11)", 44 | }, 45 | }, 46 | false, 47 | }, 48 | } 49 | 50 | for _, c := range cases { 51 | t.Run(c.name, func(t *testing.T) { 52 | m, err := NewMatchers(map[string]interface{}{ 53 | DDLRegexMatcherName: c.config, 54 | }) 55 | require.NoError(t, err) 56 | require.Equal(t, c.expected, m.Match(&c.msg), c.msg) 57 | }) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/matchers/dml_operator_matcher.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | "github.com/moiot/gravity/pkg/registry" 8 | ) 9 | 10 | const dmlOpMatcherName = "match-dml-op" 11 | 12 | type dmlOpMatcher struct { 13 | op map[string]bool 14 | } 15 | 16 | func (m *dmlOpMatcher) Configure(data interface{}) error { 17 | switch d := data.(type) { 18 | case string: 19 | if validateOp(d) { 20 | m.op[d] = true 21 | } else { 22 | return errors.Errorf("match-dml-op invalid operation type %v", data) 23 | } 24 | case []interface{}: 25 | for _, o := range d { 26 | s := o.(string) 27 | if validateOp(s) { 28 | m.op[s] = true 29 | } else { 30 | return errors.Errorf("match-dml-op invalid operation type %v", data) 31 | } 32 | } 33 | case []string: 34 | for _, o := range d { 35 | if validateOp(o) { 36 | m.op[o] = true 37 | } else { 38 | return errors.Errorf("match-dml-op invalid operation type %v", data) 39 | } 40 | } 41 | default: 42 | return errors.Errorf("match-dml-op only accept string or string slice, actual: %v", data) 43 | } 44 | return nil 45 | } 46 | 47 | func validateOp(op string) bool { 48 | if op == string(core.Insert) || op == string(core.Update) || op == string(core.Delete) { 49 | return true 50 | } 51 | 52 | return false 53 | } 54 | 55 | func (m *dmlOpMatcher) Match(msg *core.Msg) bool { 56 | if msg.DmlMsg == nil { 57 | return false 58 | } 59 | 60 | return m.op[string(msg.DmlMsg.Operation)] 61 | } 62 | 63 | type dmlOpMatcherFactoryType struct { 64 | } 65 | 66 | func (f *dmlOpMatcherFactoryType) Configure(_ string, _ map[string]interface{}) error { 67 | return nil 68 | } 69 | 70 | func (f *dmlOpMatcherFactoryType) NewMatcher() core.IMatcher { 71 | return &dmlOpMatcher{ 72 | make(map[string]bool), 73 | } 74 | } 75 | 76 | func init() { 77 | registry.RegisterPlugin(registry.MatcherPlugin, dmlOpMatcherName, &dmlOpMatcherFactoryType{}, true) 78 | } 79 | -------------------------------------------------------------------------------- /pkg/matchers/matchers.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | 7 | "github.com/juju/errors" 8 | 9 | "github.com/moiot/gravity/pkg/core" 10 | "github.com/moiot/gravity/pkg/registry" 11 | ) 12 | 13 | func NewMatchers(configData map[string]interface{}) (core.IMatcherGroup, error) { 14 | var retMatchers []core.IMatcher 15 | // find matchers based on configData 16 | for k, v := range configData { 17 | if strings.HasPrefix(k, "match") { 18 | matcherFactory, err := registry.GetPlugin(registry.MatcherPlugin, k) 19 | if err != nil { 20 | return nil, errors.Trace(err) 21 | } 22 | 23 | factory, ok := matcherFactory.(core.IMatcherFactory) 24 | if !ok { 25 | return nil, errors.Errorf("wrong type: %v", reflect.TypeOf(matcherFactory)) 26 | } 27 | matcher := factory.NewMatcher() 28 | if err := matcher.Configure(v); err != nil { 29 | return nil, errors.Trace(err) 30 | } 31 | retMatchers = append(retMatchers, matcher) 32 | } 33 | } 34 | return retMatchers, nil 35 | } 36 | -------------------------------------------------------------------------------- /pkg/matchers/matchers_test.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/moiot/gravity/pkg/core" 9 | ) 10 | 11 | func TestMatchers(t *testing.T) { 12 | assert := assert.New(t) 13 | 14 | data := map[string]interface{}{ 15 | SchemaMatcherName: "test_db", 16 | TableMatcherName: "test_table", 17 | dmlOpMatcherName: "delete", 18 | } 19 | 20 | matchGroup, err := NewMatchers(data) 21 | if err != nil { 22 | assert.FailNow(err.Error()) 23 | } 24 | 25 | assert.Equal(3, len(matchGroup)) 26 | 27 | cases := []struct { 28 | msg core.Msg 29 | matched bool 30 | }{ 31 | {core.Msg{ 32 | Database: "fake", 33 | }, 34 | false, 35 | }, 36 | { 37 | core.Msg{ 38 | Database: "test_db", 39 | Table: "fake", 40 | }, 41 | false, 42 | }, 43 | { 44 | core.Msg{ 45 | Database: "fake", 46 | Table: "test_table", 47 | }, 48 | false, 49 | }, 50 | { 51 | core.Msg{ 52 | Database: "test_db", 53 | Table: "test_table", 54 | DmlMsg: &core.DMLMsg{ 55 | Operation: core.Insert, 56 | }, 57 | }, 58 | false, 59 | }, 60 | { 61 | core.Msg{ 62 | Database: "test_db", 63 | Table: "test_table", 64 | DmlMsg: &core.DMLMsg{ 65 | Operation: core.Delete, 66 | }, 67 | }, 68 | true, 69 | }, 70 | } 71 | 72 | for _, tt := range cases { 73 | assert.Equal(tt.matched, matchGroup.Match(&tt.msg)) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /pkg/matchers/schema_matcher.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | "github.com/moiot/gravity/pkg/registry" 8 | "github.com/moiot/gravity/pkg/utils" 9 | ) 10 | 11 | const SchemaMatcherName = "match-schema" 12 | 13 | type SchemaMatcher struct { 14 | SchemaGlob string 15 | } 16 | 17 | func (m *SchemaMatcher) Configure(data interface{}) error { 18 | if arg, ok := data.(string); !ok { 19 | return errors.Errorf("match-schema only receives a string value, you provided: %v", data) 20 | } else { 21 | m.SchemaGlob = arg 22 | return nil 23 | } 24 | } 25 | 26 | func (m *SchemaMatcher) Match(msg *core.Msg) bool { 27 | if utils.Glob(m.SchemaGlob, msg.Database) { 28 | return true 29 | } else { 30 | return false 31 | } 32 | } 33 | 34 | type schemaMatcherFactoryType struct { 35 | } 36 | 37 | func (f *schemaMatcherFactoryType) Configure(_ string, _ map[string]interface{}) error { 38 | return nil 39 | } 40 | 41 | func (f *schemaMatcherFactoryType) NewMatcher() core.IMatcher { 42 | return &SchemaMatcher{} 43 | } 44 | 45 | func init() { 46 | registry.RegisterPlugin(registry.MatcherPlugin, SchemaMatcherName, &schemaMatcherFactoryType{}, true) 47 | } 48 | -------------------------------------------------------------------------------- /pkg/matchers/table_matcher.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | "github.com/moiot/gravity/pkg/registry" 8 | "github.com/moiot/gravity/pkg/utils" 9 | ) 10 | 11 | const TableMatcherName = "match-table" 12 | 13 | type TableMatcher struct { 14 | TableGlob []string 15 | } 16 | 17 | func (m *TableMatcher) Configure(data interface{}) error { 18 | switch d := data.(type) { 19 | case string: 20 | m.TableGlob = append(m.TableGlob, d) 21 | case []interface{}: 22 | for _, o := range d { 23 | s := o.(string) 24 | m.TableGlob = append(m.TableGlob, s) 25 | } 26 | case []string: 27 | m.TableGlob = append(m.TableGlob, d...) 28 | default: 29 | return errors.Errorf("match-table only accept string or string slice, actual: %v", data) 30 | } 31 | return nil 32 | } 33 | 34 | func (m *TableMatcher) Match(msg *core.Msg) bool { 35 | for _, g := range m.TableGlob { 36 | if utils.Glob(g, msg.Table) { 37 | return true 38 | } 39 | } 40 | return false 41 | } 42 | 43 | type tableMatcherFactoryType struct { 44 | } 45 | 46 | func (f *tableMatcherFactoryType) Configure(_ string, _ map[string]interface{}) error { 47 | return nil 48 | } 49 | 50 | func (f *tableMatcherFactoryType) NewMatcher() core.IMatcher { 51 | return &TableMatcher{} 52 | } 53 | 54 | func init() { 55 | registry.RegisterPlugin(registry.MatcherPlugin, TableMatcherName, &tableMatcherFactoryType{}, true) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/matchers/table_matcher_test.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/BurntSushi/toml" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/moiot/gravity/pkg/core" 11 | ) 12 | 13 | func TestTableMatcher_Match(t *testing.T) { 14 | tests := []struct { 15 | name string 16 | config string 17 | table string 18 | want bool 19 | }{ 20 | { 21 | "single glob", 22 | `"t*"`, 23 | "tt", 24 | true, 25 | }, 26 | { 27 | "multiple glob match", 28 | `["a*", "t*"]`, 29 | "tt", 30 | true, 31 | }, 32 | { 33 | "multiple glob not match", 34 | `["a*", "t*"]`, 35 | "bb", 36 | false, 37 | }, 38 | } 39 | for _, tt := range tests { 40 | t.Run(tt.name, func(t *testing.T) { 41 | config := make(map[string]interface{}) 42 | _, err := toml.Decode(TableMatcherName+" = "+tt.config, &config) 43 | require.NoError(t, err) 44 | m, err := NewMatchers(config) 45 | require.NoError(t, err) 46 | require.Equal(t, tt.want, m.Match(&core.Msg{ 47 | Table: tt.table, 48 | })) 49 | }) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /pkg/matchers/table_regex_matcher.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/juju/errors" 7 | 8 | "github.com/moiot/gravity/pkg/core" 9 | "github.com/moiot/gravity/pkg/registry" 10 | ) 11 | 12 | const TableRegexMatcherName = "match-table-regex" 13 | 14 | type TableRegexMatcher struct { 15 | Regex []*regexp.Regexp 16 | } 17 | 18 | func (m *TableRegexMatcher) Configure(data interface{}) error { 19 | switch d := data.(type) { 20 | case string: 21 | m.Regex = append(m.Regex, regexp.MustCompile(d)) 22 | case []interface{}: 23 | for _, o := range d { 24 | s := o.(string) 25 | m.Regex = append(m.Regex, regexp.MustCompile(s)) 26 | } 27 | case []string: 28 | for _, s := range d { 29 | m.Regex = append(m.Regex, regexp.MustCompile(s)) 30 | } 31 | default: 32 | return errors.Errorf("match-table-regex only accept string or string slice, actual: %v", data) 33 | } 34 | return nil 35 | } 36 | 37 | func (m *TableRegexMatcher) Match(msg *core.Msg) bool { 38 | for _, r := range m.Regex { 39 | if r.MatchString(msg.Table) { 40 | return true 41 | } 42 | } 43 | return false 44 | } 45 | 46 | type TableRegexMatcherFactoryType struct { 47 | } 48 | 49 | func (f *TableRegexMatcherFactoryType) Configure(_ string, _ map[string]interface{}) error { 50 | return nil 51 | } 52 | 53 | func (f *TableRegexMatcherFactoryType) NewMatcher() core.IMatcher { 54 | return &TableRegexMatcher{} 55 | } 56 | 57 | func init() { 58 | registry.RegisterPlugin(registry.MatcherPlugin, TableRegexMatcherName, &TableRegexMatcherFactoryType{}, true) 59 | } 60 | -------------------------------------------------------------------------------- /pkg/matchers/table_regex_matcher_test.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/BurntSushi/toml" 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/moiot/gravity/pkg/core" 10 | ) 11 | 12 | func TestTableRegexMatcher_Match(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | config string 16 | table string 17 | want bool 18 | }{ 19 | { 20 | "single", 21 | `"^t_\\d+$"`, 22 | "t_01", 23 | true, 24 | }, 25 | { 26 | "multiple", 27 | `["^a.*$", "^t_\\d+$"]`, 28 | "t_01", 29 | true, 30 | }, 31 | { 32 | "multiple not match", 33 | `["^a.*$", "^t_\\d+$"]`, 34 | "t_0a", 35 | false, 36 | }, 37 | } 38 | for _, tt := range tests { 39 | t.Run(tt.name, func(t *testing.T) { 40 | config := make(map[string]interface{}) 41 | _, err := toml.Decode(TableRegexMatcherName+" = "+tt.config, &config) 42 | require.NoError(t, err) 43 | m, err := NewMatchers(config) 44 | require.NoError(t, err) 45 | require.Equal(t, tt.want, m.Match(&core.Msg{ 46 | Table: tt.table, 47 | })) 48 | }) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /pkg/mongo/gtm/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | -------------------------------------------------------------------------------- /pkg/mongo_test/test.go: -------------------------------------------------------------------------------- 1 | package mongo_test 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | 7 | log "github.com/sirupsen/logrus" 8 | mgo "gopkg.in/mgo.v2" 9 | "gopkg.in/mgo.v2/bson" 10 | 11 | "github.com/moiot/gravity/pkg/config" 12 | ) 13 | 14 | func TestConfig() config.MongoConnConfig { 15 | cfg := config.MongoConnConfig{ 16 | Host: "127.0.0.1", 17 | Port: 27017, 18 | Direct: true, 19 | } 20 | 21 | sourceHost, ok := os.LookupEnv("MONGO_HOST") 22 | if ok { 23 | cfg.Host = sourceHost 24 | } 25 | 26 | sourceMongoPort, ok := os.LookupEnv("MONGO_PORT") 27 | if ok { 28 | p, err := strconv.Atoi(sourceMongoPort) 29 | if err != nil { 30 | log.Fatalf("invalid port") 31 | } 32 | cfg.Port = p 33 | } 34 | 35 | sourceMongoUser, ok := os.LookupEnv("MONGO_USER") 36 | if ok { 37 | cfg.Username = sourceMongoUser 38 | } 39 | 40 | sourceMongoPass, ok := os.LookupEnv("MONGO_PASSWORD") 41 | if ok { 42 | cfg.Password = sourceMongoPass 43 | } 44 | 45 | return cfg 46 | } 47 | 48 | // see https://stackoverflow.com/a/44342358 for mgo and mongo replication init 49 | func InitReplica(session *mgo.Session) { 50 | // Session mode should be monotonic as the default session used by mgo is primary which performs all operations on primary. 51 | // Since the replica set has not been initialized yet, there wont be a primary and the operation (in this case, replSetInitiate) will just timeout 52 | session.SetMode(mgo.Monotonic, true) 53 | result := bson.M{} 54 | err := session.Run("replSetInitiate", &result) 55 | if err != nil { 56 | if (result["codeName"] != "AlreadyInitialized") && result["code"] != 23 { 57 | panic(err.Error()) 58 | } 59 | } 60 | log.Info("mongo replSet initialized") 61 | } 62 | -------------------------------------------------------------------------------- /pkg/mysql_test/types/date_time.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | //const zeroTime = "0000-00-00 00:00:00" 9 | const dateTimeLayout = "2006-01-02 15:03:04.000000" 10 | 11 | type datetimeCol struct { 12 | col string 13 | } 14 | 15 | func NewDateTimeCol(col string) *datetimeCol { 16 | return &datetimeCol{col} 17 | } 18 | 19 | func (c *datetimeCol) ColType() string { 20 | return c.col 21 | } 22 | 23 | func (c *datetimeCol) Generate(r *rand.Rand) interface{} { 24 | //if r.Float64() < 0.05 { 25 | // return zeroTime 26 | //} else { 27 | secondsPerYear := r.Int63n(3600 * 24 * 365) 28 | return time.Now().Add(-1 * time.Second * time.Duration(secondsPerYear)).Format(dateTimeLayout) 29 | //} 30 | } 31 | 32 | type timestampCol struct { 33 | col string 34 | } 35 | 36 | func NewTimestampCol(col string) *timestampCol { 37 | return ×tampCol{col} 38 | } 39 | 40 | func (c *timestampCol) ColType() string { 41 | return c.col 42 | } 43 | 44 | func (c *timestampCol) Generate(r *rand.Rand) interface{} { 45 | //if r.Float64() < 0.05 { 46 | // return zeroTime 47 | //} else { 48 | secondsPerYear := r.Int63n(3600 * 24 * 365) 49 | return time.Now().Add(-1 * time.Second * time.Duration(secondsPerYear)) 50 | //} 51 | } 52 | -------------------------------------------------------------------------------- /pkg/mysql_test/types/numeric.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "math/rand" 7 | 8 | "github.com/shopspring/decimal" 9 | ) 10 | 11 | type integerColumn struct { 12 | colType string 13 | 14 | signed bool 15 | 16 | minSigned int64 17 | maxSigned int64 18 | maxUnsigned uint64 19 | } 20 | 21 | func NewIntegerColumn(colType string, signed bool, bytes int) *integerColumn { 22 | bits := uint64(bytes * 8) 23 | return &integerColumn{ 24 | colType: colType, 25 | signed: signed, 26 | minSigned: -(1 << (bits - 1)), 27 | maxSigned: (1 << (bits - 1)) - 1, 28 | maxUnsigned: (1 << bits) - 1, 29 | } 30 | } 31 | 32 | func (c *integerColumn) ColType() string { 33 | return c.colType 34 | } 35 | 36 | func (c *integerColumn) Generate(r *rand.Rand) interface{} { 37 | if c.signed { 38 | return r.Int63n(c.maxSigned-c.minSigned) + c.minSigned 39 | } else { 40 | v := r.Uint64() 41 | if v > c.maxUnsigned { 42 | return v % c.maxUnsigned 43 | } else { 44 | return v 45 | } 46 | } 47 | } 48 | 49 | type decimalCol struct { 50 | colType string 51 | precision int 52 | scale int 53 | } 54 | 55 | func NewDecimalCol(colType string) *decimalCol { 56 | precision, scale := twoParamFromType(colType) 57 | return &decimalCol{colType: colType, precision: precision, scale: scale} 58 | } 59 | 60 | func (c *decimalCol) ColType() string { 61 | return c.colType 62 | } 63 | func (c *decimalCol) Generate(r *rand.Rand) interface{} { 64 | fractional := r.Int63n(int64(math.Pow10(c.scale))) 65 | integer := r.Float64() * math.Pow10(c.precision-c.scale) 66 | d, err := decimal.NewFromString(fmt.Sprintf("%.0f.%d", integer, fractional)) 67 | if err != nil { 68 | panic(err) 69 | } 70 | return d 71 | } 72 | 73 | type floatCol struct { 74 | *decimalCol 75 | } 76 | 77 | func NewFloatCol(colType string) *floatCol { 78 | return &floatCol{NewDecimalCol(colType)} 79 | } 80 | 81 | func (c *floatCol) Generate(r *rand.Rand) interface{} { 82 | f, _ := c.decimalCol.Generate(r).(decimal.Decimal).Float64() 83 | return f 84 | } 85 | 86 | type bitCol struct { 87 | col string 88 | bit uint 89 | } 90 | 91 | func (c *bitCol) ColType() string { 92 | return c.col 93 | } 94 | 95 | func (c *bitCol) Generate(r *rand.Rand) interface{} { 96 | return r.Intn(1 << c.bit) 97 | } 98 | 99 | func NewBitCol(col string) *bitCol { 100 | b := oneParamFromType(col) 101 | return &bitCol{col, uint(b)} 102 | } 103 | -------------------------------------------------------------------------------- /pkg/mysql_test/types/types.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "math/rand" 5 | "regexp" 6 | "strconv" 7 | 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | type ColumnValGenerator interface { 12 | ColType() string 13 | Generate(r *rand.Rand) interface{} 14 | } 15 | 16 | var twoR, _ = regexp.Compile(`.+\((\d+),(\d+)\)`) 17 | var onwR, _ = regexp.Compile(`.+\((\d+)\)`) 18 | 19 | func twoParamFromType(colType string) (int, int) { 20 | matches := twoR.FindStringSubmatch(colType) 21 | if len(matches) != 3 { 22 | log.Fatalf("[twoParamFromType] invalid col type %s", colType) 23 | } 24 | one, err := strconv.Atoi(matches[1]) 25 | if err != nil { 26 | log.Fatalf("[twoParamFromType] invalid col type %s", colType) 27 | } 28 | two, err := strconv.Atoi(matches[2]) 29 | if err != nil { 30 | log.Fatalf("[twoParamFromType] invalid col type %s", colType) 31 | } 32 | 33 | return one, two 34 | } 35 | 36 | func oneParamFromType(colType string) int { 37 | matches := onwR.FindStringSubmatch(colType) 38 | if len(matches) != 2 { 39 | log.Fatalf("[twoParamFromType] invalid col type %s", colType) 40 | } 41 | one, err := strconv.Atoi(matches[1]) 42 | if err != nil { 43 | log.Fatalf("[twoParamFromType] invalid col type %s", colType) 44 | } 45 | 46 | return one 47 | } 48 | -------------------------------------------------------------------------------- /pkg/mysql_test/types/wrapper.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import "math/rand" 4 | 5 | type nullableCol struct { 6 | delegate ColumnValGenerator 7 | } 8 | 9 | func Nullable(c ColumnValGenerator) *nullableCol { 10 | return &nullableCol{c} 11 | } 12 | 13 | func (c *nullableCol) ColType() string { 14 | return c.delegate.ColType() 15 | } 16 | 17 | func (c *nullableCol) Generate(r *rand.Rand) interface{} { 18 | if r.Float32() < 0.2 { 19 | return nil 20 | } else { 21 | return c.delegate.Generate(r) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /pkg/offsets/db.go: -------------------------------------------------------------------------------- 1 | package offsets 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "database/sql" 8 | 9 | "github.com/jinzhu/gorm" 10 | _ "github.com/jinzhu/gorm/dialects/mysql" 11 | 12 | "github.com/juju/errors" 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | func SaveOffsetToDB(db *sql.DB, fullTableName string, request *OffsetCommitRequest) (*OffsetCommitResponse, error) { 17 | ts := time.Now() 18 | tx, err := db.Begin() 19 | if err != nil { 20 | log.Fatal(err) 21 | } 22 | for topic, pbs := range request.blocks { 23 | for partition, b := range pbs { 24 | stmt := fmt.Sprintf("REPLACE INTO %s (consumer_group, topic, kafka_partition, offset, ts, metadata) VALUES(?,?,?,?,?,?)", fullTableName) 25 | tx.Exec(stmt, request.ConsumerGroup, topic, partition, b.Offset, ts, b.Metadata) 26 | } 27 | } 28 | tx.Commit() 29 | 30 | resp := &OffsetCommitResponse{} 31 | for topic, pbs := range request.blocks { 32 | for partition := range pbs { 33 | resp.AddError(topic, partition, nil) 34 | } 35 | } 36 | return resp, nil 37 | } 38 | 39 | func FetchOffsetFromDB(db *sql.DB, request *OffsetFetchRequest) (*OffsetFetchResponse, error) { 40 | godb, err := gorm.Open("mysql", db) 41 | if err != nil { 42 | log.Fatal(err) 43 | } 44 | offsets := make([]ConsumerOffset, 0) 45 | godb = godb.Where("consumer_group = ?", request.ConsumerGroup).Find(&offsets) 46 | if godb.Error != nil { 47 | return nil, errors.Trace(godb.Error) 48 | } 49 | 50 | resp := &OffsetFetchResponse{} 51 | for _, o := range offsets { 52 | resp.AddBlock(o.Topic, o.KafkaPartition, o.Offset, o.Metadata) 53 | } 54 | return resp, nil 55 | } 56 | 57 | type ConsumerOffset struct { 58 | ConsumerGroup string `gorm:"column:consumer_group;primary_key"` 59 | Topic string `gorm:"column:topic;primary_key"` 60 | KafkaPartition int32 `gorm:"column:kafka_partition;primary_key"` 61 | Offset int64 `gorm:"column:offset"` 62 | Ts time.Time `gorm:"column:ts"` 63 | Metadata string `gorm:"column:metadata"` 64 | } 65 | 66 | const OffsetTableName = "consumer_offset" 67 | 68 | func (ConsumerOffset) TableName() string { 69 | return OffsetTableName 70 | } 71 | -------------------------------------------------------------------------------- /pkg/outputs/dump_output.go: -------------------------------------------------------------------------------- 1 | package outputs 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | "github.com/mitchellh/mapstructure" 6 | 7 | "github.com/moiot/gravity/pkg/core" 8 | "github.com/moiot/gravity/pkg/registry" 9 | ) 10 | 11 | type pluginConfig struct { 12 | TestKey1 string `mapstructure:"test-key-1"` 13 | } 14 | 15 | type DumpOutput struct { 16 | pipelineName string 17 | cfg *pluginConfig 18 | } 19 | 20 | func (plugin *DumpOutput) Configure(pipelineName string, data map[string]interface{}) error { 21 | cfg := pluginConfig{} 22 | if err := mapstructure.Decode(data, &cfg); err != nil { 23 | return errors.Trace(err) 24 | } 25 | return nil 26 | } 27 | 28 | func (plugin *DumpOutput) GetRouter() core.Router { 29 | return core.EmptyRouter{} 30 | } 31 | 32 | func (plugin *DumpOutput) Start() error { 33 | return nil 34 | } 35 | 36 | func (plugin *DumpOutput) Close() { 37 | 38 | } 39 | 40 | func (plugin *DumpOutput) Execute(msgs []*core.Msg) error { 41 | return nil 42 | } 43 | 44 | func init() { 45 | registry.RegisterPlugin(registry.OutputPlugin, "dump-output", &DumpOutput{}, false) 46 | } 47 | -------------------------------------------------------------------------------- /pkg/outputs/elasticsearch/helper.go: -------------------------------------------------------------------------------- 1 | package elasticsearch 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/moiot/gravity/pkg/core" 9 | "github.com/olivere/elastic" 10 | ) 11 | 12 | func genDocID(msg *core.Msg) string { 13 | pks := []string{} 14 | for _, v := range msg.DmlMsg.Pks { 15 | pks = append(pks, fmt.Sprint(v)) 16 | } 17 | return strings.Join(pks, "_") 18 | } 19 | 20 | // TODO validate index name 21 | // https://github.com/elastic/elasticsearch/blob/608a61ab85e82f8f6e88002ba7d8458411e7da62 22 | // /core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java#L188-L202 23 | func genIndexName(table string) string { 24 | return strings.ToLower(strings.TrimLeft(table, "_-+")) 25 | } 26 | 27 | func marshalError(err *elastic.ErrorDetails) string { 28 | bytes, _ := json.Marshal(err) 29 | return string(bytes) 30 | } 31 | -------------------------------------------------------------------------------- /pkg/outputs/elasticsearch/helper_test.go: -------------------------------------------------------------------------------- 1 | package elasticsearch 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestGenIndex(t *testing.T) { 10 | r := require.New(t) 11 | 12 | r.Equal("orders", genIndexName("ORDERS")) 13 | r.Equal("user_info", genIndexName("_user_info")) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/outputs/esmodel/helper.go: -------------------------------------------------------------------------------- 1 | package esmodel 2 | 3 | import ( 4 | "fmt" 5 | "github.com/moiot/gravity/pkg/core" 6 | "github.com/moiot/gravity/pkg/utils" 7 | "github.com/olivere/elastic/v7" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | func genDocID(msg *core.Msg, fk string) string { 12 | 13 | if fk != "" { 14 | return fmt.Sprint(msg.DmlMsg.Data[fk]) 15 | } 16 | for _, v := range msg.DmlMsg.Pks { 17 | return fmt.Sprint(v) 18 | } 19 | return "" 20 | } 21 | 22 | func genDocIDBySon(msg *core.Msg, fk string) string { 23 | return fmt.Sprint(msg.DmlMsg.Data[fk]) 24 | } 25 | 26 | func genPrimary(msg *core.Msg) (string, interface{}) { 27 | for k, v := range msg.DmlMsg.Pks { 28 | return k, v 29 | } 30 | return "", "" 31 | } 32 | 33 | func marshalError(err *elastic.ErrorDetails) string { 34 | bytes, _ := json.Marshal(err) 35 | return string(bytes) 36 | } 37 | 38 | func Capitalize(str string) string { 39 | var upperStr string 40 | vv := []rune(str) 41 | for i := 0; i < len(vv); i++ { 42 | if i == 0 { 43 | if vv[i] >= 97 && vv[i] <= 122 { 44 | vv[i] -= 32 45 | upperStr += string(vv[i]) 46 | } else { 47 | fmt.Println("Not begins with lowercase letter,") 48 | return str 49 | } 50 | } else { 51 | upperStr += string(vv[i]) 52 | } 53 | } 54 | return upperStr 55 | } 56 | 57 | func printJsonEncodef(format string, data ...interface{}) { 58 | jsons := make([]interface{}, 0, 1) 59 | for _, v := range data { 60 | switch v.(type) { 61 | case string: 62 | jsons = append(jsons, v) 63 | break 64 | default: 65 | bs, _ := json.Marshal(v) 66 | jsons = append(jsons, string(bs)) 67 | break 68 | } 69 | } 70 | 71 | if utils.Version == "None" { 72 | fmt.Printf(format+"\n", jsons...) 73 | } 74 | log.Infof(format, jsons...) 75 | } 76 | -------------------------------------------------------------------------------- /pkg/outputs/esmodel/helper_test.go: -------------------------------------------------------------------------------- 1 | package esmodel 2 | 3 | import ( 4 | "github.com/stretchr/testify/require" 5 | "testing" 6 | ) 7 | 8 | func TestCapitalize(t *testing.T) { 9 | 10 | r := require.New(t) 11 | 12 | r.Equal("Capitalize", Capitalize("capitalize")) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/outputs/mysql/add_missing_column.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "github.com/moiot/gravity/pkg/core" 5 | "github.com/moiot/gravity/pkg/schema_store" 6 | ) 7 | 8 | // AddMissingColumn add data to msg if target table has extra columns. 9 | // It is considered safe since this casting does not lose information. 10 | func AddMissingColumn(msg *core.Msg, targetTableDef *schema_store.Table) (bool, error) { 11 | if msg.DmlMsg == nil || targetTableDef == nil { 12 | return true, nil 13 | } 14 | 15 | targetColumns := targetTableDef.Columns 16 | for i := range targetColumns { 17 | column := targetColumns[i] 18 | 19 | name := column.Name 20 | _, ok := msg.DmlMsg.Data[name] 21 | if !ok { 22 | if column.DefaultVal.IsNull { 23 | msg.DmlMsg.Data[name] = nil 24 | } else { 25 | msg.DmlMsg.Data[name] = targetColumns[i].DefaultVal.ValueString 26 | } 27 | } 28 | } 29 | return true, nil 30 | } 31 | -------------------------------------------------------------------------------- /pkg/outputs/mysql/add_missing_column_test.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/moiot/gravity/pkg/core" 9 | 10 | "github.com/moiot/gravity/pkg/schema_store" 11 | ) 12 | 13 | func TestAddMissingColumn(t *testing.T) { 14 | r := require.New(t) 15 | 16 | t.Run("when the missing column have default value NOT NULL", func(tt *testing.T) { 17 | msg := &core.Msg{ 18 | DmlMsg: &core.DMLMsg{ 19 | Data: make(map[string]interface{}), 20 | Operation: core.Insert, 21 | }, 22 | } 23 | 24 | t := &schema_store.Table{ 25 | Columns: []schema_store.Column{ 26 | { 27 | Name: "a", 28 | IsNullable: false, 29 | DefaultVal: schema_store.ColumnValueString{ 30 | ValueString: "123", 31 | IsNull: false, 32 | }, 33 | }, 34 | }, 35 | } 36 | b, err := AddMissingColumn(msg, t) 37 | r.True(b) 38 | r.NoError(err) 39 | r.Equal("123", msg.DmlMsg.Data["a"]) 40 | }) 41 | 42 | t.Run("when the missing column have default value of NULL", func(tt *testing.T) { 43 | msg := &core.Msg{ 44 | DmlMsg: &core.DMLMsg{ 45 | Data: make(map[string]interface{}), 46 | Operation: core.Insert, 47 | }, 48 | } 49 | 50 | t := &schema_store.Table{ 51 | Columns: []schema_store.Column{ 52 | { 53 | Name: "a", 54 | IsNullable: true, 55 | DefaultVal: schema_store.ColumnValueString{ 56 | ValueString: "", 57 | IsNull: true, 58 | }, 59 | }, 60 | }, 61 | } 62 | b, err := AddMissingColumn(msg, t) 63 | r.True(b) 64 | r.NoError(err) 65 | v, ok := msg.DmlMsg.Data["a"] 66 | r.True(ok) 67 | r.Nil(v) 68 | }) 69 | } 70 | -------------------------------------------------------------------------------- /pkg/outputs/mysql/cast_default_data.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | // import ( 4 | // "github.com/json-iterator/go" 5 | // "github.com/juju/errors" 6 | // 7 | // "github.com/moiot/gravity/pkg/core" 8 | // 9 | // "github.com/moiot/gravity/schema_store" 10 | // ) 11 | // 12 | // 13 | // 14 | // func CastDefaultData(msg *core.Msg, targetTableDef *schema_store.Table) (bool, error) { 15 | // if msg.DmlMsg == nil || targetTableDef == nil { 16 | // return true, nil 17 | // } 18 | // 19 | // for columnName := range msg.DmlMsg.Data { 20 | // targetColumn, ok := targetTableDef.Column(columnName) 21 | // if !ok { 22 | // return false, errors.Errorf("column %s not found", columnName) 23 | // } 24 | // 25 | // if msg.TableDef == nil { 26 | // s, _ := jsoniter.MarshalToString(msg) 27 | // return false, errors.Errorf("source table def is nil. msg: %s", s) 28 | // } 29 | // 30 | // columnDef := msg.TableDef.MustColumn(columnName) 31 | // value := msg.DmlMsg.Data[columnName] 32 | // if value == nil && columnDef.DefaultVal.IsNull && !targetColumn.DefaultVal.IsNull { 33 | // msg.DmlMsg.Data[columnName] = targetColumn.DefaultVal.ValueString 34 | // } 35 | // } 36 | // return true, nil 37 | // } 38 | -------------------------------------------------------------------------------- /pkg/outputs/plugins.go: -------------------------------------------------------------------------------- 1 | package outputs 2 | 3 | import ( 4 | "github.com/moiot/gravity/pkg/outputs/async_kafka" 5 | "github.com/moiot/gravity/pkg/outputs/elasticsearch" 6 | "github.com/moiot/gravity/pkg/outputs/esmodel" 7 | "github.com/moiot/gravity/pkg/outputs/mysql" 8 | "github.com/moiot/gravity/pkg/outputs/stdout" 9 | ) 10 | 11 | const ( 12 | AsyncKafka = async_kafka.Name 13 | Mysql = mysql.Name 14 | Stdout = stdout.Name 15 | Elasticsearch = elasticsearch.Name 16 | EsModel = esmodel.Name 17 | ) 18 | -------------------------------------------------------------------------------- /pkg/outputs/routers/elasticsearch_router.go: -------------------------------------------------------------------------------- 1 | package routers 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | "github.com/moiot/gravity/pkg/matchers" 8 | ) 9 | 10 | type ElasticsearchRoute struct { 11 | RouteMatchers 12 | TargetIndex string 13 | TargetType string 14 | IgnoreNoPrimaryKey bool 15 | } 16 | 17 | type ElasticsearchRouter []*ElasticsearchRoute 18 | 19 | func (r ElasticsearchRouter) Exists(msg *core.Msg) bool { 20 | _, ok := r.Match(msg) 21 | return ok 22 | } 23 | 24 | func (r ElasticsearchRouter) Match(msg *core.Msg) (*ElasticsearchRoute, bool) { 25 | for _, route := range r { 26 | if route.Match(msg) { 27 | return route, true 28 | } 29 | } 30 | return nil, false 31 | } 32 | 33 | func NewElasticsearchRoutes(configData []map[string]interface{}) ([]*ElasticsearchRoute, error) { 34 | var routes []*ElasticsearchRoute 35 | 36 | for _, routeConfig := range configData { 37 | route := ElasticsearchRoute{} 38 | retMatchers, err := matchers.NewMatchers(routeConfig) 39 | if err != nil { 40 | return nil, errors.Trace(err) 41 | } 42 | 43 | route.AllMatchers = retMatchers 44 | 45 | targetIndex, err := getString(routeConfig, "target-index", "") 46 | if err != nil { 47 | return nil, err 48 | } 49 | route.TargetIndex = targetIndex 50 | 51 | targetType, err := getString(routeConfig, "target-type", "doc") 52 | if err != nil { 53 | return nil, err 54 | } 55 | route.TargetType = targetType 56 | 57 | ignoreNoPrimaryKey, err := getBool(routeConfig, "ignore-no-primary-key", false) 58 | if err != nil { 59 | return nil, err 60 | } 61 | route.IgnoreNoPrimaryKey = ignoreNoPrimaryKey 62 | 63 | routes = append(routes, &route) 64 | } 65 | return routes, nil 66 | } 67 | -------------------------------------------------------------------------------- /pkg/outputs/routers/esmodel_router_test.go: -------------------------------------------------------------------------------- 1 | package routers 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/mitchellh/mapstructure" 7 | "github.com/moiot/gravity/pkg/config" 8 | "testing" 9 | ) 10 | 11 | type ElasticsearchServerAuth struct { 12 | Username string `mapstructure:"username" toml:"username" json:"username"` 13 | Password string `mapstructure:"password" toml:"password" json:"password"` 14 | } 15 | 16 | type ElasticsearchServerConfig struct { 17 | URLs []string `mapstructure:"urls" toml:"urls" json:"urls"` 18 | Sniff bool `mapstructure:"sniff" toml:"sniff" json:"sniff"` 19 | Auth *ElasticsearchServerAuth `mapstructure:"auth" toml:"auth" json:"auth"` 20 | Timeout int `mapstructure:"timeout" toml:"timeout" json:"timeout"` 21 | } 22 | 23 | type EsModelPluginConfig struct { 24 | ServerConfig *ElasticsearchServerConfig `mapstructure:"server" json:"server"` 25 | Routes []map[string]interface{} `mapstructure:"routes" json:"routes"` 26 | IgnoreBadRequest bool `mapstructure:"ignore-bad-request" json:"ignore-bad-request"` 27 | } 28 | 29 | func TestNewEsModelRoutes(t *testing.T) { 30 | 31 | cfg := config.NewConfig() 32 | 33 | if err := cfg.ConfigFromFile("../../../docs/2.0/example-mysql2esmodel.toml"); err != nil { 34 | println(err) 35 | } 36 | 37 | //if bs, err := json.Marshal(cfg.PipelineConfig.OutputPlugin); err != nil { 38 | // println(err) 39 | //}else{ 40 | // fmt.Printf("%s\n", string(bs)) 41 | //} 42 | 43 | pluginConfig := EsModelPluginConfig{} 44 | 45 | err := mapstructure.Decode(cfg.PipelineConfig.OutputPlugin.Config, &pluginConfig) 46 | if err != nil { 47 | fmt.Println(err) 48 | } 49 | 50 | if bs, err := json.Marshal(pluginConfig); err != nil { 51 | println(err) 52 | } else { 53 | fmt.Printf("%s\n", string(bs)) 54 | } 55 | 56 | routers, err := NewEsModelRoutes(pluginConfig.Routes) 57 | 58 | fmt.Println(err) 59 | 60 | if bs, err := json.Marshal(routers); err != nil { 61 | println(err) 62 | } else { 63 | fmt.Printf("%s\n", string(bs)) 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /pkg/outputs/routers/kafka_router.go: -------------------------------------------------------------------------------- 1 | package routers 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | "github.com/moiot/gravity/pkg/matchers" 8 | ) 9 | 10 | type KafkaRoute struct { 11 | RouteMatchers 12 | DMLTargetTopic string 13 | // DDLTargetTopic string 14 | } 15 | 16 | type KafkaRouter []*KafkaRoute 17 | 18 | func (r KafkaRouter) Exists(msg *core.Msg) bool { 19 | for i := range r { 20 | if r[i].Match(msg) { 21 | return true 22 | } 23 | } 24 | return false 25 | } 26 | 27 | func NewKafkaRoutes(configData []map[string]interface{}) ([]*KafkaRoute, error) { 28 | var kafkaRoutes []*KafkaRoute 29 | 30 | for _, routeConfig := range configData { 31 | route := KafkaRoute{} 32 | retMatchers, err := matchers.NewMatchers(routeConfig) 33 | if err != nil { 34 | return nil, errors.Trace(err) 35 | } 36 | 37 | route.AllMatchers = retMatchers 38 | 39 | if dmlTargetTopic, ok := routeConfig["dml-topic"]; !ok { 40 | route.DMLTargetTopic = "" 41 | } else { 42 | dmlTargetTopicString, ok := dmlTargetTopic.(string) 43 | if !ok { 44 | return nil, errors.Errorf("dml-topic invalid type") 45 | } 46 | route.DMLTargetTopic = dmlTargetTopicString 47 | } 48 | // TODO add ddl topic 49 | // if ddlTargetTopic, ok := routeConfig["ddl-target-topic"]; !ok { 50 | // route.DDLTargetTopic = "" 51 | // } else { 52 | // ddlTargetTopicString, ok := ddlTargetTopic.(string) 53 | // if !ok { 54 | // return nil, errors.Errorf("ddl-target-topic invalid type") 55 | // } 56 | // route.DDLTargetTopic = ddlTargetTopicString 57 | // } 58 | kafkaRoutes = append(kafkaRoutes, &route) 59 | } 60 | return kafkaRoutes, nil 61 | } 62 | -------------------------------------------------------------------------------- /pkg/outputs/routers/mysql_router.go: -------------------------------------------------------------------------------- 1 | package routers 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | "github.com/moiot/gravity/pkg/matchers" 8 | ) 9 | 10 | type MySQLRoute struct { 11 | RouteMatchers 12 | TargetSchema string 13 | TargetTable string 14 | } 15 | 16 | func (route *MySQLRoute) GetTarget(msgSchema string, msgTable string) (string, string) { 17 | var targetSchema string 18 | var targetTable string 19 | 20 | if route.TargetSchema == "*" || route.TargetSchema == "" { 21 | targetSchema = msgSchema 22 | } else { 23 | targetSchema = route.TargetSchema 24 | } 25 | 26 | if route.TargetTable == "*" || route.TargetTable == "" { 27 | targetTable = msgTable 28 | } else { 29 | targetTable = route.TargetTable 30 | } 31 | return targetSchema, targetTable 32 | } 33 | 34 | type MySQLRouter []*MySQLRoute 35 | 36 | func (r MySQLRouter) Exists(msg *core.Msg) bool { 37 | for i := range r { 38 | if r[i].Match(msg) { 39 | return true 40 | } 41 | } 42 | return false 43 | } 44 | 45 | func NewMySQLRoutes(configData []map[string]interface{}) ([]*MySQLRoute, error) { 46 | var mysqlRoutes []*MySQLRoute 47 | 48 | // init matchers 49 | if len(configData) == 0 { 50 | // default match all and target to the same as source 51 | return []*MySQLRoute{{}}, nil 52 | } else { 53 | for _, routeConfig := range configData { 54 | route := MySQLRoute{} 55 | retMatchers, err := matchers.NewMatchers(routeConfig) 56 | if err != nil { 57 | return nil, errors.Trace(err) 58 | } 59 | 60 | route.AllMatchers = retMatchers 61 | 62 | if targetSchema, ok := routeConfig["target-schema"]; !ok { 63 | route.TargetSchema = "*" 64 | } else { 65 | if targetSchemaString, ok := targetSchema.(string); !ok { 66 | return nil, errors.Errorf("target-schema invalid type") 67 | } else { 68 | route.TargetSchema = targetSchemaString 69 | } 70 | } 71 | 72 | if targetTable, ok := routeConfig["target-table"]; !ok { 73 | route.TargetTable = "*" 74 | } else { 75 | if targetTableString, ok := targetTable.(string); !ok { 76 | return nil, errors.Errorf("target-table invalid type") 77 | } else { 78 | route.TargetTable = targetTableString 79 | } 80 | } 81 | mysqlRoutes = append(mysqlRoutes, &route) 82 | } 83 | } 84 | return mysqlRoutes, nil 85 | } 86 | -------------------------------------------------------------------------------- /pkg/outputs/routers/mysql_router_test.go: -------------------------------------------------------------------------------- 1 | package routers 2 | 3 | import "testing" 4 | 5 | func TestMySQLRoute_GetTarget(t *testing.T) { 6 | type fields struct { 7 | RouteMatchers RouteMatchers 8 | TargetSchema string 9 | TargetTable string 10 | } 11 | type args struct { 12 | msgSchema string 13 | msgTable string 14 | } 15 | tests := []struct { 16 | name string 17 | fields fields 18 | args args 19 | want string 20 | want1 string 21 | }{ 22 | { 23 | name: "empty route should use source schema and table", 24 | fields: fields{}, 25 | args: args{ 26 | msgSchema: "schema", 27 | msgTable: "table", 28 | }, 29 | want: "schema", 30 | want1: "table", 31 | }, 32 | } 33 | for _, tt := range tests { 34 | t.Run(tt.name, func(t *testing.T) { 35 | route := &MySQLRoute{ 36 | RouteMatchers: tt.fields.RouteMatchers, 37 | TargetSchema: tt.fields.TargetSchema, 38 | TargetTable: tt.fields.TargetTable, 39 | } 40 | got, got1 := route.GetTarget(tt.args.msgSchema, tt.args.msgTable) 41 | if got != tt.want { 42 | t.Errorf("MySQLRoute.GetTarget() got = %v, want %v", got, tt.want) 43 | } 44 | if got1 != tt.want1 { 45 | t.Errorf("MySQLRoute.GetTarget() got1 = %v, want %v", got1, tt.want1) 46 | } 47 | }) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pkg/outputs/routers/routers.go: -------------------------------------------------------------------------------- 1 | package routers 2 | 3 | import ( 4 | "github.com/moiot/gravity/pkg/core" 5 | ) 6 | 7 | type RouteMatchers struct { 8 | AllMatchers []core.IMatcher 9 | } 10 | 11 | func (r RouteMatchers) Match(msg *core.Msg) bool { 12 | matched := true 13 | 14 | for _, m := range r.AllMatchers { 15 | if !m.Match(msg) { 16 | matched = false 17 | break 18 | } 19 | } 20 | return matched 21 | } 22 | -------------------------------------------------------------------------------- /pkg/outputs/routers/utils_test.go: -------------------------------------------------------------------------------- 1 | package routers 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestGetString(t *testing.T) { 10 | r := require.New(t) 11 | data := map[string]interface{}{ 12 | "a": "a", 13 | "b": 1, 14 | } 15 | 16 | val, err := getString(data, "a", "default") 17 | r.NoError(err) 18 | r.Equal("a", val) 19 | 20 | val, err = getString(data, "b", "default") 21 | r.Error(err) 22 | r.Equal("b is invalid", err.Error()) 23 | 24 | val, err = getString(data, "c", "default") 25 | r.NoError(err) 26 | r.Equal("default", val) 27 | } 28 | 29 | func TestGetBool(t *testing.T) { 30 | r := require.New(t) 31 | data := map[string]interface{}{ 32 | "a": true, 33 | "b": 1, 34 | } 35 | 36 | val, err := getBool(data, "a", false) 37 | r.NoError(err) 38 | r.Equal(true, val) 39 | 40 | val, err = getBool(data, "b", false) 41 | r.Error(err) 42 | r.Equal("b is invalid", err.Error()) 43 | 44 | val, err = getBool(data, "c", false) 45 | r.NoError(err) 46 | r.Equal(false, val) 47 | } 48 | -------------------------------------------------------------------------------- /pkg/position_repos/mysql_repo_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * // Copyright 2019 , Beijing Mobike Technology Co., Ltd. 4 | * // 5 | * // Licensed under the Apache License, Version 2.0 (the "License"); 6 | * // you may not use this file except in compliance with the License. 7 | * // You may obtain a copy of the License at 8 | * // 9 | * // http://www.apache.org/licenses/LICENSE-2.0 10 | * // 11 | * // Unless required by applicable law or agreed to in writing, software 12 | * // distributed under the License is distributed on an "AS IS" BASIS, 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package position_repos 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/moiot/gravity/pkg/registry" 23 | 24 | "github.com/moiot/gravity/pkg/config" 25 | "github.com/moiot/gravity/pkg/mysql_test" 26 | "github.com/stretchr/testify/require" 27 | ) 28 | 29 | func TestMysqlPositionRepo_GetPut(t *testing.T) { 30 | r := require.New(t) 31 | 32 | dbCfg := mysql_test.SourceDBConfig() 33 | 34 | repoConfig := NewMySQLRepoConfig("", dbCfg) 35 | plugin, err := registry.GetPlugin(registry.PositionRepo, repoConfig.Type) 36 | r.NoError(err) 37 | 38 | r.NoError(plugin.Configure(t.Name(), repoConfig.Config)) 39 | repo := plugin.(PositionRepo) 40 | r.NoError(repo.Init()) 41 | 42 | // delete it first 43 | r.NoError(repo.Delete(t.Name())) 44 | 45 | _, _, exist, err := repo.Get(t.Name()) 46 | r.NoError(err) 47 | 48 | r.False(exist) 49 | 50 | // put first value 51 | meta := PositionMeta{ 52 | Name: t.Name(), 53 | Stage: config.Stream, 54 | } 55 | r.NoError(repo.Put(t.Name(), meta, "test")) 56 | 57 | meta, v, exist, err := repo.Get(t.Name()) 58 | r.NoError(err) 59 | r.True(exist) 60 | r.Equal("test", v) 61 | r.Equal(config.Stream, meta.Stage) 62 | 63 | // put another value 64 | r.NoError(repo.Put(t.Name(), meta, "test2")) 65 | 66 | meta, v, exist, err = repo.Get(t.Name()) 67 | r.NoError(err) 68 | r.True(exist) 69 | r.Equal("test2", v) 70 | 71 | // put an invalid value 72 | err = repo.Put(t.Name(), meta, "") 73 | r.NotNil(err) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/position_repos/position_repos.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * // Copyright 2019 , Beijing Mobike Technology Co., Ltd. 4 | * // 5 | * // Licensed under the Apache License, Version 2.0 (the "License"); 6 | * // you may not use this file except in compliance with the License. 7 | * // You may obtain a copy of the License at 8 | * // 9 | * // http://www.apache.org/licenses/LICENSE-2.0 10 | * // 11 | * // Unless required by applicable law or agreed to in writing, software 12 | * // distributed under the License is distributed on an "AS IS" BASIS, 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package position_repos 18 | 19 | import ( 20 | "github.com/juju/errors" 21 | "github.com/moiot/gravity/pkg/config" 22 | "time" 23 | ) 24 | 25 | type PositionRepo interface { 26 | Init() error 27 | Get(pipelineName string) (PositionMeta, string, bool, error) 28 | Put(pipelineName string, positionMeta PositionMeta, v string) error 29 | Delete(pipelineName string) error 30 | Close() error 31 | } 32 | 33 | type PositionMeta struct { 34 | // Version is the schema version of position 35 | Version string `bson:"version" json:"version"` 36 | // Name is the unique name of a pipeline 37 | Name string 38 | Stage config.InputMode 39 | UpdateTime time.Time 40 | } 41 | 42 | func (meta PositionMeta) Validate() error { 43 | if meta.Stage != config.Stream && meta.Stage != config.Batch { 44 | return errors.Errorf("invalid position stage: %v", meta.Stage) 45 | } 46 | 47 | if meta.Name == "" { 48 | return errors.Errorf("empty name") 49 | } 50 | 51 | return nil 52 | } 53 | 54 | type Position struct { 55 | PositionMeta 56 | Value interface{} `bson:"-" json:"-"` 57 | } 58 | 59 | func (p Position) Validate() error { 60 | if err := p.PositionMeta.Validate(); err != nil { 61 | return errors.Trace(err) 62 | } 63 | 64 | if p.Value == nil { 65 | return errors.Errorf("empty position value: %v", p.Value) 66 | } 67 | 68 | return nil 69 | } 70 | 71 | type PositionValueEncoder func(v interface{}) (string, error) 72 | type PositionValueDecoder func(s string) (interface{}, error) 73 | 74 | -------------------------------------------------------------------------------- /pkg/protocol/meta.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | type MsgTimestamp int 4 | 5 | type MessageRouter interface { 6 | GetTopic() string 7 | GetPartitions() []int32 8 | } 9 | 10 | type MessageTracer interface { 11 | AddTimestamp(t MsgTimestamp) 12 | AddMetrics() 13 | MetricsString() string 14 | } 15 | -------------------------------------------------------------------------------- /pkg/protocol/protocol_suite_test.go: -------------------------------------------------------------------------------- 1 | package protocol_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | "testing" 7 | ) 8 | 9 | func TestRouterConfig(t *testing.T) { 10 | RegisterFailHandler(Fail) 11 | RunSpecs(t, "protocol Suite") 12 | } 13 | -------------------------------------------------------------------------------- /pkg/registry/go_plugin_getter.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "plugin" 7 | "reflect" 8 | 9 | "github.com/hashicorp/go-getter" 10 | "github.com/juju/errors" 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | const ( 15 | PluginDir = "./go-plugins" 16 | InterfaceName = "Plugin" 17 | ) 18 | 19 | func DownloadGoNativePlugin(goPluginConfig map[string]interface{}) (string, Plugin, error) { 20 | pluginName, ok := goPluginConfig["name"] 21 | if !ok { 22 | return "", nil, errors.Errorf("go-plugin filter should have a unique name") 23 | } 24 | pluginNameString, ok := pluginName.(string) 25 | if !ok { 26 | return "", nil, errors.Errorf("filter name must be a string") 27 | } 28 | 29 | pluginUrl, ok := goPluginConfig["url"] 30 | if !ok { 31 | return "", nil, errors.Errorf("go-plugin filter should have a url to download") 32 | } 33 | pluginUrlString, ok := pluginUrl.(string) 34 | if !ok { 35 | return "", nil, errors.Errorf("filter url must be a string") 36 | } 37 | 38 | if _, err := os.Stat(PluginDir); os.IsNotExist(err) { 39 | if err := os.Mkdir(PluginDir, 755); err != nil { 40 | return "", nil, errors.Trace(err) 41 | } 42 | } 43 | 44 | dstFileName := fmt.Sprintf("%s/%s", PluginDir, pluginNameString) 45 | 46 | pwd, err := os.Getwd() 47 | if err != nil { 48 | return "", nil, errors.Trace(err) 49 | } 50 | 51 | log.Infof("[registry] downloading plugin pwd %s, dstFileName: %s from %s", pwd, dstFileName, pluginUrlString) 52 | 53 | client := getter.Client{ 54 | Src: pluginUrlString, 55 | Dst: dstFileName, 56 | Dir: false, 57 | Mode: getter.ClientModeFile, 58 | Getters: getter.Getters, 59 | Pwd: pwd, 60 | } 61 | if err := client.Get(); err != nil { 62 | return "", nil, errors.Trace(err) 63 | } 64 | 65 | plug, err := plugin.Open(dstFileName) 66 | if err != nil { 67 | return "", nil, errors.Annotatef(err, "failed to open go-plugin: %v", dstFileName) 68 | } 69 | 70 | plugSymbol, err := plug.Lookup(InterfaceName) 71 | if err != nil { 72 | return "", nil, errors.Trace(err) 73 | } 74 | 75 | p, ok := plugSymbol.(Plugin) 76 | if !ok { 77 | return "", nil, errors.Errorf("go-plugin %s is does not have Plugin interface: %v", dstFileName, reflect.TypeOf(plugSymbol)) 78 | } 79 | 80 | return pluginNameString, p, nil 81 | } 82 | -------------------------------------------------------------------------------- /pkg/registry/go_plugin_getter_test.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "os" 5 | "runtime" 6 | "testing" 7 | 8 | "github.com/juju/errors" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestGoPluginGetter(t *testing.T) { 13 | assert := assert.New(t) 14 | 15 | defer func() { 16 | os.Remove(PluginDir) 17 | }() 18 | 19 | var fileUrl string 20 | if runtime.GOOS == "darwin" { 21 | fileUrl = "./test_data/dump_filter_plugin.darwin.so" 22 | } else if runtime.GOOS == "linux" { 23 | fileUrl = "./test_data/dump_filter_plugin.linux.so" 24 | } 25 | 26 | configData := map[string]interface{}{ 27 | "name": "dump_filter_plugin", 28 | "url": fileUrl, 29 | } 30 | 31 | _, _, err := DownloadGoNativePlugin(configData) 32 | if err != nil { 33 | assert.FailNow(errors.ErrorStack(err)) 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /pkg/registry/registry.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "sync" 7 | 8 | "github.com/juju/errors" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | type PluginType string 13 | 14 | const ( 15 | InputPlugin PluginType = "input" 16 | PositionRepo PluginType = "positionRepo" 17 | OutputPlugin PluginType = "output" 18 | FilterPlugin PluginType = "filters" 19 | MatcherPlugin PluginType = "matcher" 20 | SchedulerPlugin PluginType = "scheduler" 21 | SQLExecutionEnginePlugin PluginType = "sqlExecutionEngine" 22 | ) 23 | 24 | type Plugin interface { 25 | Configure(pipelineName string, data map[string]interface{}) error 26 | } 27 | 28 | type PluginFactory func() Plugin 29 | 30 | var registry map[PluginType]map[string]PluginFactory 31 | var mutex sync.Mutex 32 | 33 | func RegisterPluginFactory(pluginType PluginType, name string, v PluginFactory) { 34 | mutex.Lock() 35 | defer mutex.Unlock() 36 | 37 | log.Debugf("[RegisterPlugin] type: %v, name: %v", pluginType, name) 38 | if registry == nil { 39 | registry = make(map[PluginType]map[string]PluginFactory) 40 | } 41 | 42 | _, ok := registry[pluginType] 43 | if !ok { 44 | registry[pluginType] = make(map[string]PluginFactory) 45 | } 46 | 47 | _, ok = registry[pluginType][name] 48 | if ok { 49 | panic(fmt.Sprintf("plugin already exists, type: %v, name: %v", pluginType, name)) 50 | } 51 | registry[pluginType][name] = v 52 | } 53 | 54 | func RegisterPlugin(pluginType PluginType, name string, v Plugin, singleton bool) { 55 | var pf PluginFactory 56 | if singleton { 57 | pf = func() Plugin { 58 | return v 59 | } 60 | } else { 61 | pf = func() Plugin { 62 | return reflect.New(reflect.TypeOf(v).Elem()).Interface().(Plugin) 63 | } 64 | } 65 | RegisterPluginFactory(pluginType, name, pf) 66 | } 67 | 68 | func GetPlugin(pluginType PluginType, name string) (Plugin, error) { 69 | mutex.Lock() 70 | defer mutex.Unlock() 71 | 72 | if registry == nil { 73 | return nil, errors.Errorf("empty registry") 74 | } 75 | 76 | plugins, ok := registry[pluginType] 77 | if !ok { 78 | return nil, errors.Errorf("empty plugin type: %v, name: %v", pluginType, name) 79 | } 80 | 81 | p, ok := plugins[name] 82 | if !ok { 83 | return nil, errors.Errorf("empty plugin, type: %v, name: %v", pluginType, name) 84 | } 85 | return p(), nil 86 | } 87 | -------------------------------------------------------------------------------- /pkg/registry/test_data/Makefile: -------------------------------------------------------------------------------- 1 | PWD = $(shell pwd) 2 | 3 | default: 4 | docker run --rm -v $(PWD):/usr/src/myapp -w /usr/src/myapp -e GOOS=linux -e GOARCH=amd64 golang:1.11.4 go build -buildmode=plugin -v -o dump_filter_plugin.linux.so dump_filter_plugin.go 5 | GOARCH=amd64 GOOS=darwin go build -buildmode=plugin -o dump_filter_plugin.darwin.so dump_filter_plugin.go -------------------------------------------------------------------------------- /pkg/registry/test_data/dump_filter_plugin.darwin.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moiot/gravity/15226ce9f24891b52cfc3955fce6ae6bf7e3d32a/pkg/registry/test_data/dump_filter_plugin.darwin.so -------------------------------------------------------------------------------- /pkg/registry/test_data/dump_filter_plugin.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | type filterPlugin struct { 4 | pipelineName string 5 | pluginName string 6 | url string 7 | } 8 | 9 | // exported variable 10 | var Plugin = filterPlugin{} 11 | 12 | func (p *filterPlugin) Configure(pipelineName string, data map[string]interface{}) error { 13 | p.pipelineName = pipelineName 14 | 15 | p.pluginName = data["name"].(string) 16 | p.url = data["url"].(string) 17 | return nil 18 | } 19 | -------------------------------------------------------------------------------- /pkg/registry/test_data/dump_filter_plugin.linux.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moiot/gravity/15226ce9f24891b52cfc3955fce6ae6bf7e3d32a/pkg/registry/test_data/dump_filter_plugin.linux.so -------------------------------------------------------------------------------- /pkg/sarama_cluster/LICENSE: -------------------------------------------------------------------------------- 1 | (The MIT License) 2 | 3 | Copyright (c) 2017 Black Square Media Ltd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | 'Software'), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /pkg/sarama_cluster/README.md: -------------------------------------------------------------------------------- 1 | Consumer is a modified version of [sarama-cluster](https://github.com/bsm/sarama-cluster) using `database` as offset storage. 2 | `etcd` will be supported in future 3 | 4 | -------------------------------------------------------------------------------- /pkg/sarama_cluster/client.go: -------------------------------------------------------------------------------- 1 | package sarama_cluster 2 | 3 | import ( 4 | "errors" 5 | "sync/atomic" 6 | 7 | "github.com/Shopify/sarama" 8 | ) 9 | 10 | var errClientInUse = errors.New("cluster: client is already used by another consumer") 11 | 12 | // Client is a group client 13 | type Client struct { 14 | sarama.Client 15 | config Config 16 | 17 | inUse uint32 18 | } 19 | 20 | // NewClient creates a new client instance 21 | func NewClient(addrs []string, config *Config) (*Client, error) { 22 | if config == nil { 23 | config = NewConfig() 24 | } 25 | 26 | if err := config.Validate(); err != nil { 27 | return nil, err 28 | } 29 | 30 | client, err := sarama.NewClient(addrs, &config.Config) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | return &Client{Client: client, config: *config}, nil 36 | } 37 | 38 | // ClusterConfig returns the cluster configuration. 39 | func (c *Client) ClusterConfig() *Config { 40 | cfg := c.config 41 | return &cfg 42 | } 43 | 44 | func (c *Client) claim() bool { 45 | return atomic.CompareAndSwapUint32(&c.inUse, 0, 1) 46 | } 47 | 48 | func (c *Client) release() { 49 | atomic.CompareAndSwapUint32(&c.inUse, 1, 0) 50 | } 51 | -------------------------------------------------------------------------------- /pkg/sarama_cluster/cluster.go: -------------------------------------------------------------------------------- 1 | package sarama_cluster 2 | 3 | // Strategy for partition to consumer assignement 4 | type Strategy string 5 | 6 | const ( 7 | // StrategyRange is the default and assigns partition ranges to consumers. 8 | // Example with six partitions and two consumers: 9 | // C1: [0, 1, 2] 10 | // C2: [3, 4, 5] 11 | StrategyRange Strategy = "range" 12 | 13 | // StrategyRoundRobin assigns partitions by alternating over consumers. 14 | // Example with six partitions and two consumers: 15 | // C1: [0, 2, 4] 16 | // C2: [1, 3, 5] 17 | StrategyRoundRobin Strategy = "roundrobin" 18 | ) 19 | 20 | // Error instances are wrappers for internal errors with a context and 21 | // may be returned through the consumer's Errors() channel 22 | type Error struct { 23 | Ctx string 24 | error 25 | } 26 | -------------------------------------------------------------------------------- /pkg/sarama_cluster/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | package kafka provides cluster extensions for Sarama, enabing users 3 | to consume topics across from multiple, balanced nodes. 4 | 5 | It requires Kafka v0.9+ and follows the steps guide, described in: 6 | https://cwiki.apache.org/confluence/display/KAFKA/Kafka+0.9+Consumer+Rewrite+Design 7 | */ 8 | package sarama_cluster 9 | -------------------------------------------------------------------------------- /pkg/sarama_cluster/util.go: -------------------------------------------------------------------------------- 1 | package sarama_cluster 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "sync" 7 | ) 8 | 9 | type none struct{} 10 | 11 | type topicPartition struct { 12 | Topic string 13 | Partition int32 14 | } 15 | 16 | func (tp *topicPartition) String() string { 17 | return fmt.Sprintf("%s-%d", tp.Topic, tp.Partition) 18 | } 19 | 20 | type offsetInfo struct { 21 | Offset int64 22 | Metadata string 23 | } 24 | 25 | func (i offsetInfo) NextOffset(fallback int64) int64 { 26 | if i.Offset > -1 { 27 | return i.Offset 28 | } 29 | return fallback 30 | } 31 | 32 | type int32Slice []int32 33 | 34 | func (p int32Slice) Len() int { return len(p) } 35 | func (p int32Slice) Less(i, j int) bool { return p[i] < p[j] } 36 | func (p int32Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } 37 | 38 | func (p int32Slice) Diff(o int32Slice) (res []int32) { 39 | on := len(o) 40 | for _, x := range p { 41 | n := sort.Search(on, func(i int) bool { return o[i] >= x }) 42 | if n < on && o[n] == x { 43 | continue 44 | } 45 | res = append(res, x) 46 | } 47 | return 48 | } 49 | 50 | // -------------------------------------------------------------------- 51 | 52 | type loopTomb struct { 53 | c chan none 54 | o sync.Once 55 | w sync.WaitGroup 56 | } 57 | 58 | func newLoopTomb() *loopTomb { 59 | return &loopTomb{c: make(chan none)} 60 | } 61 | 62 | func (t *loopTomb) stop() { t.o.Do(func() { close(t.c) }) } 63 | func (t *loopTomb) Close() { t.stop(); t.w.Wait() } 64 | 65 | func (t *loopTomb) Dying() <-chan none { return t.c } 66 | func (t *loopTomb) Go(f func(<-chan none)) { 67 | t.w.Add(1) 68 | 69 | go func() { 70 | defer t.stop() 71 | defer t.w.Done() 72 | 73 | f(t.c) 74 | }() 75 | } 76 | -------------------------------------------------------------------------------- /pkg/schedulers/schedulers.go: -------------------------------------------------------------------------------- 1 | package schedulers 2 | 3 | import ( 4 | _ "github.com/moiot/gravity/pkg/schedulers/batch_table_scheduler" 5 | ) 6 | -------------------------------------------------------------------------------- /pkg/schema_store/schema_store_suite_test.go: -------------------------------------------------------------------------------- 1 | package schema_store_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | "testing" 7 | ) 8 | 9 | func TestGravity(t *testing.T) { 10 | RegisterFailHandler(Fail) 11 | RunSpecs(t, "TargetSchemaStore Suite") 12 | } 13 | -------------------------------------------------------------------------------- /pkg/schema_store/simple_schema_store.go: -------------------------------------------------------------------------------- 1 | package schema_store 2 | 3 | import ( 4 | "database/sql" 5 | "sync" 6 | 7 | "github.com/moiot/gravity/pkg/config" 8 | 9 | "github.com/juju/errors" 10 | 11 | "github.com/moiot/gravity/pkg/utils" 12 | ) 13 | 14 | type SimpleSchemaStore struct { 15 | sync.RWMutex 16 | sourceDB *sql.DB 17 | dbCfg *config.DBConfig 18 | schemas map[string]Schema 19 | } 20 | 21 | func (store *SimpleSchemaStore) IsInCache(dbName string) bool { 22 | if _, ok := store.schemas[dbName]; ok { 23 | return true 24 | } else { 25 | return false 26 | } 27 | } 28 | func (store *SimpleSchemaStore) getFromCache(dbName string, lock bool) (Schema, bool) { 29 | if lock { 30 | store.RLock() 31 | defer store.RUnlock() 32 | } 33 | 34 | cachedSchema, ok := store.schemas[dbName] 35 | if ok { 36 | return cachedSchema, true 37 | } 38 | return nil, false 39 | } 40 | 41 | func (store *SimpleSchemaStore) GetSchema(dbName string) (Schema, error) { 42 | if dbName == "" { 43 | return nil, nil 44 | } 45 | 46 | schema, ok := store.getFromCache(dbName, true) 47 | if ok { 48 | return schema, nil 49 | } 50 | 51 | store.Lock() 52 | defer store.Unlock() 53 | schema, ok = store.getFromCache(dbName, false) 54 | if ok { 55 | return schema, nil 56 | } 57 | schema, err := GetSchemaFromDB(store.sourceDB, dbName) 58 | if err != nil { 59 | return nil, errors.Trace(err) 60 | } 61 | store.schemas[dbName] = schema 62 | return schema, nil 63 | } 64 | 65 | func (store *SimpleSchemaStore) InvalidateSchemaCache(schema string) { 66 | store.Lock() 67 | defer store.Unlock() 68 | 69 | delete(store.schemas, schema) 70 | } 71 | 72 | func (store *SimpleSchemaStore) InvalidateCache() { 73 | // Invalidate Schema cache 74 | store.Lock() 75 | defer store.Unlock() 76 | 77 | // make a new map here 78 | store.schemas = make(map[string]Schema) 79 | } 80 | 81 | func (store *SimpleSchemaStore) Close() { 82 | if store.sourceDB != nil { 83 | store.sourceDB.Close() 84 | } 85 | } 86 | 87 | func NewSimpleSchemaStoreFromDBConn(db *sql.DB) (SchemaStore, error) { 88 | return &SimpleSchemaStore{sourceDB: db, schemas: make(map[string]Schema)}, nil 89 | } 90 | 91 | func NewSimpleSchemaStore(dbCfg *config.DBConfig) (*SimpleSchemaStore, error) { 92 | sourceDB, err := utils.CreateDBConnection(dbCfg) 93 | if err != nil { 94 | return nil, errors.Trace(err) 95 | } 96 | 97 | return &SimpleSchemaStore{dbCfg: dbCfg, schemas: make(map[string]Schema), sourceDB: sourceDB}, nil 98 | } 99 | -------------------------------------------------------------------------------- /pkg/schema_store/simple_schema_store_test.go: -------------------------------------------------------------------------------- 1 | package schema_store_test 2 | 3 | import ( 4 | _ "github.com/go-sql-driver/mysql" 5 | . "github.com/onsi/ginkgo" 6 | . "github.com/onsi/gomega" 7 | 8 | "github.com/moiot/gravity/pkg/mysql_test" 9 | . "github.com/moiot/gravity/pkg/schema_store" 10 | ) 11 | 12 | var _ = Describe("simple_schema_store_test", func() { 13 | var schemaStoreTestDB = "simpleSchemaStoreTest" 14 | It("returns schemas", func() { 15 | db := mysql_test.MustSetupSourceDB(schemaStoreTestDB) 16 | 17 | store, err := NewSimpleSchemaStoreFromDBConn(db) 18 | Expect(err).To(BeNil()) 19 | 20 | s, err := store.GetSchema(schemaStoreTestDB) 21 | 22 | Expect(err).To(BeNil()) 23 | table := s[mysql_test.TestTableName] 24 | Expect(table).NotTo(BeNil()) 25 | Expect(table.ColumnNames()).To(Equal([]string{"id", "name", "email", "ts"})) 26 | 27 | emailColumn, ok := table.Column("email") 28 | Expect(ok).To(BeTrue()) 29 | Expect(emailColumn.IsNullable).To(BeFalse()) 30 | Expect(emailColumn.DefaultVal.IsNull).To(BeFalse()) 31 | Expect(emailColumn.DefaultVal.ValueString).To(Equal("default_email")) 32 | 33 | Expect(store.IsInCache("radom_db")).To(Equal(false)) 34 | }) 35 | }) 36 | -------------------------------------------------------------------------------- /pkg/schema_store/utils_test.go: -------------------------------------------------------------------------------- 1 | package schema_store 2 | 3 | import ( 4 | "database/sql" 5 | 6 | _ "github.com/go-sql-driver/mysql" 7 | 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | 13 | "github.com/moiot/gravity/pkg/mysql_test" 14 | ) 15 | 16 | func TestSchemaStoreUtils(t *testing.T) { 17 | assert := assert.New(t) 18 | 19 | var sourceDB *sql.DB 20 | var testDBName = "schemaStoreUtilsTest" 21 | 22 | sourceDB = mysql_test.MustSetupSourceDB(testDBName) 23 | sourceDB.SetMaxOpenConns(20) 24 | sourceDB.SetMaxIdleConns(0) 25 | 26 | t.Run("GetTableDefFromDB for single table", func(tt *testing.T) { 27 | table, err := GetTableDefFromDB(sourceDB, testDBName, mysql_test.TestTableName) 28 | assert.Nil(err) 29 | assert.Equal(4, len(table.Columns)) 30 | 31 | assert.Equal("id", table.Columns[0].Name) 32 | //assert.Equal(0, table.Columns[0].Idx) 33 | assert.False(table.Columns[0].IsNullable) 34 | assert.False(table.Columns[0].IsUnsigned) 35 | assert.True(table.Columns[0].IsPrimaryKey) 36 | assert.Equal("id", table.PrimaryKeyColumns[0].Name) 37 | assert.Equal("id", table.UniqueKeyColumnMap["PRIMARY"][0]) 38 | assert.Equal("name", table.Columns[1].Name) 39 | assert.False(table.Columns[1].IsPrimaryKey) 40 | }) 41 | 42 | t.Run("GetCreateSchemaStatement", func(tt *testing.T) { 43 | err, sql := GetCreateSchemaStatement(sourceDB, testDBName) 44 | assert.Nil(err) 45 | assert.NotEqual("", sql) 46 | }) 47 | 48 | t.Run("GetSchemaFromDB for single table", func(tt *testing.T) { 49 | schema, err := GetSchemaFromDB(sourceDB, testDBName) 50 | assert.Nil(err) 51 | assert.NotNil(schema[mysql_test.TestTableName]) 52 | }) 53 | 54 | t.Run("GetSchemaFromDB for 1024 tables", func(tt *testing.T) { 55 | 56 | for i := 0; i < 1024; i++ { 57 | tableName := fmt.Sprintf("t_%d", i) 58 | statement := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.%s( 59 | id INT NOT NULL, 60 | PRIMARY KEY (id) 61 | )ENGINE=InnoDB DEFAULT CHARSET=utf8`, testDBName, tableName) 62 | _, err := sourceDB.Exec(statement) 63 | assert.Nil(err) 64 | } 65 | 66 | schema, err := GetSchemaFromDB(sourceDB, testDBName) 67 | assert.Nil(err) 68 | assert.NotNil(schema["t_0"]) 69 | assert.NotNil(schema["t_1023"]) 70 | }) 71 | } 72 | -------------------------------------------------------------------------------- /pkg/sliding_window/heap.go: -------------------------------------------------------------------------------- 1 | package sliding_window 2 | 3 | import "github.com/juju/errors" 4 | 5 | type windowItemHeap []int64 6 | 7 | func (h windowItemHeap) Len() int { return len(h) } 8 | func (h windowItemHeap) Less(i, j int) bool { return h[i] < h[j] } 9 | func (h windowItemHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } 10 | 11 | func (h *windowItemHeap) Push(x interface{}) { 12 | // Push and Pop use pointer receivers because they modify the slice's length, 13 | // not just its contents. 14 | *h = append(*h, x.(int64)) 15 | } 16 | 17 | func (h *windowItemHeap) Pop() interface{} { 18 | old := *h 19 | n := len(old) 20 | x := old[n-1] 21 | *h = old[0 : n-1] 22 | return x 23 | } 24 | 25 | func (h *windowItemHeap) SmallestItem() (int64, error) { 26 | if len(*h) > 0 { 27 | return (*h)[0], nil 28 | } 29 | 30 | return 0, errors.Errorf("empty heap") 31 | } 32 | -------------------------------------------------------------------------------- /pkg/sliding_window/sequence.go: -------------------------------------------------------------------------------- 1 | package sliding_window 2 | 3 | const FirstSequenceNumber = 1 4 | 5 | type SequenceGenerator struct { 6 | sequenceNumber int64 7 | } 8 | 9 | func (generator *SequenceGenerator) Next() int64 { 10 | generator.sequenceNumber++ 11 | return generator.sequenceNumber 12 | } 13 | 14 | func NewSequenceGenerator() *SequenceGenerator { 15 | g := SequenceGenerator{sequenceNumber: FirstSequenceNumber - 1} 16 | return &g 17 | } 18 | -------------------------------------------------------------------------------- /pkg/sliding_window/sliding_window.go: -------------------------------------------------------------------------------- 1 | package sliding_window 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | var HealthyThreshold float64 = 60 8 | 9 | type WindowItem interface { 10 | SequenceNumber() int64 11 | BeforeWindowMoveForward() 12 | EventTime() time.Time 13 | ProcessTime() time.Time 14 | } 15 | 16 | // (Output) Watermark is defined as the minimum process time of input(which may be blocked on enqueue) and active items in window. 17 | type Watermark struct { 18 | ProcessTime time.Time 19 | EventTime time.Time 20 | } 21 | 22 | func (w Watermark) Healthy() bool { 23 | return time.Since(w.ProcessTime).Seconds() < HealthyThreshold 24 | } 25 | 26 | type Window interface { 27 | AddWindowItem(item WindowItem) 28 | AckWindowItem(sequence int64) 29 | Size() int 30 | WaitingQueueLen() int 31 | Close() 32 | Watermark() Watermark 33 | } 34 | -------------------------------------------------------------------------------- /pkg/sliding_window/sliding_window_suite_test.go: -------------------------------------------------------------------------------- 1 | package sliding_window_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestSlidingWindow(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Sliding Window") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/sql_execution_engine/internal_txn_tagger.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * // Copyright 2019 , Beijing Mobike Technology Co., Ltd. 4 | * // 5 | * // Licensed under the Apache License, Version 2.0 (the "License"); 6 | * // you may not use this file except in compliance with the License. 7 | * // You may obtain a copy of the License at 8 | * // 9 | * // http://www.apache.org/licenses/LICENSE-2.0 10 | * // 11 | * // Unless required by applicable law or agreed to in writing, software 12 | * // distributed under the License is distributed on an "AS IS" BASIS, 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package sql_execution_engine 18 | 19 | import ( 20 | "database/sql" 21 | 22 | "github.com/juju/errors" 23 | log "github.com/sirupsen/logrus" 24 | 25 | "github.com/moiot/gravity/pkg/utils" 26 | ) 27 | 28 | type InternalTxnTaggerCfg struct { 29 | TagInternalTxn bool `mapstructure:"tag-internal-txn" json:"tag-internal-txn"` 30 | SQLAnnotation string `mapstructure:"sql-annotation" json:"sql-annotation"` 31 | } 32 | 33 | var DefaultInternalTxnTaggerCfg = map[string]interface{}{ 34 | "tag-internal-txn": false, 35 | "sql-annotation": "", 36 | } 37 | 38 | func ExecWithInternalTxnTag( 39 | pipelineName string, 40 | internalTxnTaggerCfg *InternalTxnTaggerCfg, 41 | db *sql.DB, 42 | query string, 43 | args []interface{}) error { 44 | 45 | newQuery := query 46 | if internalTxnTaggerCfg.SQLAnnotation != "" { 47 | newQuery = SQLWithAnnotation(query, internalTxnTaggerCfg.SQLAnnotation) 48 | } 49 | 50 | if !internalTxnTaggerCfg.TagInternalTxn { 51 | if log.IsLevelEnabled(log.DebugLevel) { 52 | log.Debugf("query: %v, args: %v", newQuery, args) 53 | } 54 | _, err := db.Exec(newQuery, args...) 55 | return errors.Annotatef(err, "query: %v, args: %v", query, args) 56 | } 57 | 58 | // 59 | // TagInternalTxn is ON 60 | // 61 | txn, err := db.Begin() 62 | if err != nil { 63 | return errors.Trace(err) 64 | } 65 | 66 | _, err = txn.Exec(utils.GenerateTxnTagSQL(pipelineName)) 67 | if err != nil { 68 | return errors.Trace(err) 69 | } 70 | 71 | if log.IsLevelEnabled(log.DebugLevel) { 72 | log.Debugf("query: %v, args: %v", newQuery, args) 73 | } 74 | _, err = txn.Exec(query, args...) 75 | if err != nil { 76 | txn.Rollback() 77 | return errors.Annotatef(err, "query: %v, args: %+v", query, args) 78 | } 79 | 80 | return errors.Trace(txn.Commit()) 81 | } 82 | -------------------------------------------------------------------------------- /pkg/sql_execution_engine/mysql_insert_ignore_engine.go: -------------------------------------------------------------------------------- 1 | package sql_execution_engine 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/mitchellh/mapstructure" 7 | "github.com/moiot/gravity/pkg/utils" 8 | 9 | "github.com/juju/errors" 10 | 11 | "github.com/moiot/gravity/pkg/core" 12 | "github.com/moiot/gravity/pkg/registry" 13 | "github.com/moiot/gravity/pkg/schema_store" 14 | ) 15 | 16 | const MySQLInsertIgnore = "mysql-insert-ignore" 17 | 18 | type mysqlInsertIgnoreEngineConfig struct { 19 | InternalTxnTaggerCfg `mapstructure:",squash"` 20 | } 21 | 22 | type mysqlInsertIgnoreEngine struct { 23 | pipelineName string 24 | cfg *mysqlInsertIgnoreEngineConfig 25 | db *sql.DB 26 | } 27 | 28 | func init() { 29 | registry.RegisterPlugin(registry.SQLExecutionEnginePlugin, MySQLInsertIgnore, &mysqlInsertIgnoreEngine{}, false) 30 | } 31 | 32 | func (engine *mysqlInsertIgnoreEngine) Configure(pipelineName string, data map[string]interface{}) error { 33 | cfg := mysqlInsertIgnoreEngineConfig{} 34 | if err := mapstructure.Decode(data, &cfg); err != nil { 35 | return errors.Trace(err) 36 | } 37 | engine.cfg = &cfg 38 | engine.pipelineName = pipelineName 39 | return nil 40 | } 41 | 42 | func (engine *mysqlInsertIgnoreEngine) Init(db *sql.DB) error { 43 | engine.db = db 44 | 45 | if engine.cfg.TagInternalTxn { 46 | return errors.Trace(utils.InitInternalTxnTags(db)) 47 | } 48 | 49 | return nil 50 | } 51 | 52 | func (engine *mysqlInsertIgnoreEngine) Execute(msgBatch []*core.Msg, targetTableDef *schema_store.Table) error { 53 | if len(msgBatch) == 0 { 54 | return nil 55 | } 56 | 57 | var query string 58 | var args []interface{} 59 | var err error 60 | 61 | if msgBatch[0].DmlMsg.Operation == core.Delete { 62 | if len(msgBatch) > 1 { 63 | return errors.Errorf("batch size > 1 for delete") 64 | } 65 | query, args, err = GenerateSingleDeleteSQL(msgBatch[0], targetTableDef) 66 | if err != nil { 67 | return errors.Trace(err) 68 | } 69 | } else { 70 | query, args, err = GenerateInsertIgnoreSQL(msgBatch, targetTableDef) 71 | if err != nil { 72 | return errors.Trace(err) 73 | } 74 | } 75 | 76 | return errors.Trace(ExecWithInternalTxnTag(engine.pipelineName, &engine.cfg.InternalTxnTaggerCfg, engine.db, query, args)) 77 | } 78 | -------------------------------------------------------------------------------- /pkg/sql_execution_engine/mysql_insert_ignore_engine_test.go: -------------------------------------------------------------------------------- 1 | package sql_execution_engine 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/moiot/gravity/pkg/core" 11 | "github.com/moiot/gravity/pkg/mysql_test" 12 | "github.com/moiot/gravity/pkg/schema_store" 13 | ) 14 | 15 | func TestMysqlInsertIgnoreEngine_Execute(t *testing.T) { 16 | r := require.New(t) 17 | 18 | testSchemaName := strings.ToLower(t.Name()) 19 | 20 | // init test table 21 | db := mysql_test.MustSetupTargetDB(testSchemaName) 22 | ddl := fmt.Sprintf(` 23 | CREATE TABLE IF NOT EXISTS %s.%s ( 24 | id int(11) unsigned NOT NULL, 25 | v varchar(256) DEFAULT NULL, 26 | PRIMARY KEY (id) 27 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 28 | `, testSchemaName, "t") 29 | 30 | _, err := db.Exec(ddl) 31 | r.NoError(err) 32 | 33 | // tableDef and msgBatch 34 | tbl := &schema_store.Table{ 35 | Schema: testSchemaName, 36 | Name: "t", 37 | Columns: []schema_store.Column{ 38 | {Name: "id"}, //Idx: 0, 39 | 40 | {Name: "v"}, //Idx: 1, 41 | 42 | }, 43 | } 44 | 45 | msg := &core.Msg{} 46 | msg.DmlMsg = &core.DMLMsg{ 47 | Operation: core.Insert, 48 | Data: map[string]interface{}{"id": 1, "v": 1}, 49 | Pks: map[string]interface{}{"id": 1}, 50 | } 51 | msgBatch := []*core.Msg{msg} 52 | 53 | executor := NewEngineExecutor("test", MySQLInsertIgnore, db, nil) 54 | 55 | // no data initially 56 | n, err := mysql_test.CountTestTable(db, testSchemaName, "t") 57 | r.NoError(err) 58 | r.Equal(0, n) 59 | 60 | r.NoError(executor.Execute(msgBatch, tbl)) 61 | 62 | // one row 63 | n, err = mysql_test.CountTestTable(db, testSchemaName, "t") 64 | r.NoError(err) 65 | r.Equal(1, n) 66 | 67 | // execute with the same row again, expect no exception, and still one row 68 | r.NoError(executor.Execute(msgBatch, tbl)) 69 | n, err = mysql_test.CountTestTable(db, testSchemaName, "t") 70 | r.NoError(err) 71 | r.Equal(1, n) 72 | 73 | // execute another row, n == 2 74 | msg.DmlMsg.Data["id"] = 2 75 | r.NoError(executor.Execute(msgBatch, tbl)) 76 | n, err = mysql_test.CountTestTable(db, testSchemaName, "t") 77 | r.NoError(err) 78 | r.Equal(2, n) 79 | 80 | } 81 | -------------------------------------------------------------------------------- /pkg/sql_execution_engine/mysql_replace_engine.go: -------------------------------------------------------------------------------- 1 | package sql_execution_engine 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/mitchellh/mapstructure" 7 | 8 | "github.com/moiot/gravity/pkg/registry" 9 | "github.com/moiot/gravity/pkg/utils" 10 | 11 | "github.com/juju/errors" 12 | 13 | "github.com/moiot/gravity/pkg/core" 14 | 15 | "github.com/moiot/gravity/pkg/schema_store" 16 | ) 17 | 18 | type MysqlReplaceEngineConfig struct { 19 | InternalTxnTaggerCfg `mapstructure:",squash"` 20 | } 21 | 22 | type mysqlReplaceEngine struct { 23 | pipelineName string 24 | cfg *MysqlReplaceEngineConfig 25 | db *sql.DB 26 | } 27 | 28 | const MySQLReplaceEngine = "mysql-replace-engine" 29 | 30 | var DefaultMySQLReplaceEngineConfig = map[string]interface{}{ 31 | MySQLReplaceEngine: DefaultInternalTxnTaggerCfg, 32 | } 33 | 34 | func init() { 35 | registry.RegisterPlugin(registry.SQLExecutionEnginePlugin, MySQLReplaceEngine, &mysqlReplaceEngine{}, false) 36 | } 37 | 38 | func (engine *mysqlReplaceEngine) Configure(pipelineName string, data map[string]interface{}) error { 39 | cfg := MysqlReplaceEngineConfig{} 40 | if err := mapstructure.Decode(data, &cfg); err != nil { 41 | return errors.Trace(err) 42 | } 43 | 44 | engine.cfg = &cfg 45 | engine.pipelineName = pipelineName 46 | return nil 47 | } 48 | 49 | func (engine *mysqlReplaceEngine) Init(db *sql.DB) error { 50 | engine.db = db 51 | 52 | if engine.cfg.TagInternalTxn { 53 | return errors.Trace(utils.InitInternalTxnTags(db)) 54 | } 55 | 56 | return nil 57 | } 58 | 59 | func (engine *mysqlReplaceEngine) Execute(msgBatch []*core.Msg, targetTableDef *schema_store.Table) error { 60 | currentBatchSize := len(msgBatch) 61 | 62 | if currentBatchSize == 0 { 63 | return nil 64 | } 65 | 66 | if currentBatchSize > 1 && msgBatch[0].DmlMsg.Operation == core.Delete { 67 | return errors.Errorf("[mysql_replace_engine] only support single delete") 68 | } 69 | 70 | var query string 71 | var args []interface{} 72 | var err error 73 | 74 | if msgBatch[0].DmlMsg.Operation == core.Delete { 75 | query, args, err = GenerateSingleDeleteSQL(msgBatch[0], targetTableDef) 76 | if err != nil { 77 | return errors.Trace(err) 78 | } 79 | } else { 80 | query, args, err = GenerateReplaceSQLWithMultipleValues(msgBatch, targetTableDef) 81 | if err != nil { 82 | data, pks := DebugDmlMsg(msgBatch) 83 | return errors.Annotatef(err, "query: %v, pb data: %v, pks: %v", query, data, pks) 84 | } 85 | } 86 | 87 | return errors.Trace(ExecWithInternalTxnTag(engine.pipelineName, &engine.cfg.InternalTxnTaggerCfg, engine.db, query, args)) 88 | } 89 | -------------------------------------------------------------------------------- /pkg/sql_execution_engine/sql_execution_engine.go: -------------------------------------------------------------------------------- 1 | package sql_execution_engine 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/moiot/gravity/pkg/core" 7 | 8 | "github.com/moiot/gravity/pkg/schema_store" 9 | ) 10 | 11 | type EngineExecutor interface { 12 | Execute(msgBatch []*core.Msg, tableDef *schema_store.Table) error 13 | } 14 | 15 | type EngineInitializer interface { 16 | Init(db *sql.DB) error 17 | } 18 | 19 | // func SelectEngine(DetectConflict bool, UserBidirection bool, UseShadingProxy bool) (string, error) { 20 | // var engine string 21 | // if !DetectConflict && !UserBidirection && !UseShadingProxy { 22 | // engine = MySQLReplaceEngine 23 | // } else if UserBidirection && !UseShadingProxy { 24 | // engine = BiDirectionEngine 25 | // } else if UserBidirection && UseShadingProxy { 26 | // engine = BiDirectionShadingEngine 27 | // } else if DetectConflict && !UserBidirection && !UseShadingProxy { 28 | // engine = ConflictEngine 29 | // } else { 30 | // return "", errors.BadRequestf("No match sql execution engine found for detect conflict[%t], bidirection[%t], shading proxy[%t]", DetectConflict, UserBidirection, UseShadingProxy) 31 | // } 32 | // log.Infof("SelectEngine: %v", engine) 33 | // return engine, nil 34 | // } 35 | // 36 | // func NewSQLExecutionEngine(db *sql.DB, engineConfig MySQLExecutionEngineConfig) EngineExecutor { 37 | // var engine EngineExecutor 38 | // name := engineConfig.EngineType 39 | // switch name { 40 | // case MySQLReplaceEngine: 41 | // engine = NewMySQLReplaceEngine(db) 42 | // case BiDirectionEngine: 43 | // engine = NewBidirectionEngine(db, utils.Stmt) 44 | // case BiDirectionShadingEngine: 45 | // engine = NewBidirectionEngine(db, utils.Annotation) 46 | // case ConflictEngine: 47 | // engine = NewConflictEngine(db, engineConfig.OverrideConflict, engineConfig.MaxConflictRetry, 1*time.Second, false) 48 | // case ManualEngine: 49 | // engine = NewManualSQLEngine(db, engineConfig) 50 | // default: 51 | // log.Fatal("unknown sql execution engine ", name) 52 | // return nil 53 | // } 54 | // 55 | // return engine 56 | // } 57 | -------------------------------------------------------------------------------- /pkg/utils/etcd.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/coreos/etcd/clientv3" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | const ( 12 | dialTimeout = 5 * time.Second 13 | ) 14 | 15 | func MustCreateEtcdClient(etcdEndpoints string) *clientv3.Client { 16 | etcdClient, err := CreateEtcdClient(etcdEndpoints) 17 | if err != nil { 18 | log.Fatalf("[gravity] failed to contact etcd: %v", err) 19 | } 20 | return etcdClient 21 | } 22 | 23 | func CreateEtcdClient(etcdEndpoints string) (*clientv3.Client, error) { 24 | endpoints := strings.Split(etcdEndpoints, ",") 25 | return clientv3.New(clientv3.Config{ 26 | Endpoints: endpoints, 27 | // Etcd client instance needs to specify a value for DialTimeout to avoid permanent block 28 | // when it encountered an abnormality 29 | DialTimeout: dialTimeout, 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/utils/getter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * // Copyright 2019 , Beijing Mobike Technology Co., Ltd. 4 | * // 5 | * // Licensed under the Apache License, Version 2.0 (the "License"); 6 | * // you may not use this file except in compliance with the License. 7 | * // You may obtain a copy of the License at 8 | * // 9 | * // http://www.apache.org/licenses/LICENSE-2.0 10 | * // 11 | * // Unless required by applicable law or agreed to in writing, software 12 | * // distributed under the License is distributed on an "AS IS" BASIS, 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package utils 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | 23 | "github.com/hashicorp/go-getter" 24 | "github.com/juju/errors" 25 | log "github.com/sirupsen/logrus" 26 | ) 27 | 28 | func GetExecutable(url string, dir string, name string) (string, error) { 29 | if _, err := os.Stat(dir); os.IsNotExist(err) { 30 | if err := os.Mkdir(dir, 0700); err != nil { 31 | return "", errors.Trace(err) 32 | } 33 | } 34 | 35 | dstFileName := fmt.Sprintf("%s/%s", dir, name) 36 | 37 | pwd, err := os.Getwd() 38 | if err != nil { 39 | return "", errors.Trace(err) 40 | } 41 | 42 | log.Infof("[registry] downloading plugin pwd %s, dstFileName: %s from %s", pwd, dstFileName, url) 43 | 44 | client := getter.Client{ 45 | Src: url, 46 | Dst: dstFileName, 47 | Dir: false, 48 | Mode: getter.ClientModeFile, 49 | Getters: getter.Getters, 50 | Pwd: pwd, 51 | } 52 | if err := client.Get(); err != nil { 53 | return "", errors.Trace(err) 54 | } 55 | 56 | if err := os.Chmod(dstFileName, 0700); err != nil { 57 | return "", errors.Trace(err) 58 | } 59 | 60 | return dstFileName, nil 61 | } 62 | -------------------------------------------------------------------------------- /pkg/utils/glob.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "strings" 4 | 5 | // The character which is treated like a glob 6 | const GLOB = "*" 7 | 8 | // Glob will test a string pattern, potentially containing globs, against a 9 | // subject string. The result is a simple true/false, determining whether or 10 | // not the glob pattern matched the subject text. 11 | func Glob(pattern, subj string) bool { 12 | // Empty pattern can only match empty subject 13 | if pattern == "" { 14 | return subj == pattern 15 | } 16 | 17 | // If the pattern _is_ a glob, it matches everything 18 | if pattern == GLOB { 19 | return true 20 | } 21 | 22 | parts := strings.Split(pattern, GLOB) 23 | 24 | if len(parts) == 1 { 25 | // No globs in pattern, so test for equality 26 | return subj == pattern 27 | } 28 | 29 | leadingGlob := strings.HasPrefix(pattern, GLOB) 30 | trailingGlob := strings.HasSuffix(pattern, GLOB) 31 | end := len(parts) - 1 32 | 33 | // Go over the leading parts and ensure they match. 34 | for i := 0; i < end; i++ { 35 | idx := strings.Index(subj, parts[i]) 36 | 37 | switch i { 38 | case 0: 39 | // Check the first section. Requires special handling. 40 | if !leadingGlob && idx != 0 { 41 | return false 42 | } 43 | default: 44 | // Check that the middle parts match. 45 | if idx < 0 { 46 | return false 47 | } 48 | } 49 | 50 | // Trim evaluated text from subj as we loop over the pattern. 51 | subj = subj[idx+len(parts[i]):] 52 | } 53 | 54 | // Reached the last section. Requires special handling. 55 | return trailingGlob || strings.HasSuffix(subj, parts[end]) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/utils/hash.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "hash/crc32" 4 | 5 | // GenHashKey generates key with crc32 algorithm 6 | func GenHashKey(key string) uint32 { 7 | return crc32.ChecksumIEEE([]byte(key)) 8 | } 9 | -------------------------------------------------------------------------------- /pkg/utils/labels.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | 7 | "github.com/juju/errors" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | func GetLabelEnvString() string { 12 | return os.Getenv("DRC_LABELS") 13 | } 14 | 15 | func GetLabelsFromEnv(envString string) (map[string]string, error) { 16 | labels := make(map[string]string) 17 | 18 | if envString == "" { 19 | return labels, nil 20 | } 21 | 22 | labelPairs := strings.Split(envString, ",") 23 | 24 | for _, pair := range labelPairs { 25 | pairSlice := strings.Split(pair, "=") 26 | if len(pairSlice) != 2 { 27 | return labels, errors.Errorf("invalid label env: %v, pairSlice: %v", envString, pairSlice) 28 | } 29 | 30 | k := pairSlice[0] 31 | v := pairSlice[1] 32 | labels[k] = v 33 | } 34 | return labels, nil 35 | } 36 | 37 | func MustGetLabelsFromEnv() map[string]string { 38 | labels, err := GetLabelsFromEnv(GetLabelEnvString()) 39 | if err != nil { 40 | log.Fatalf("failed to get label: %v", err) 41 | } 42 | h, err := os.Hostname() 43 | if err == nil { 44 | labels["hostname"] = h 45 | } 46 | return labels 47 | } 48 | -------------------------------------------------------------------------------- /pkg/utils/labels_test.go: -------------------------------------------------------------------------------- 1 | package utils_test 2 | 3 | import ( 4 | . "github.com/moiot/gravity/pkg/utils" 5 | . "github.com/onsi/ginkgo" 6 | . "github.com/onsi/gomega" 7 | ) 8 | 9 | var _ = Describe("test labels", func() { 10 | It("returns the right labels", func() { 11 | envString := "a=v1,b=v2" 12 | labels, err := GetLabelsFromEnv(envString) 13 | Expect(err).To(BeNil()) 14 | Expect(labels["a"]).To(Equal("v1")) 15 | Expect(labels["b"]).To(Equal("v2")) 16 | 17 | envString = "" 18 | labels, err = GetLabelsFromEnv(envString) 19 | Expect(err).To(BeNil()) 20 | 21 | envString = "a=v1" 22 | labels, err = GetLabelsFromEnv(envString) 23 | Expect(err).To(BeNil()) 24 | Expect(labels["a"]).To(Equal("v1")) 25 | }) 26 | }) 27 | -------------------------------------------------------------------------------- /pkg/utils/life_cycle.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | type Closer interface { 4 | Close() 5 | } 6 | -------------------------------------------------------------------------------- /pkg/utils/mongo_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * // Copyright 2019 , Beijing Mobike Technology Co., Ltd. 4 | * // 5 | * // Licensed under the Apache License, Version 2.0 (the "License"); 6 | * // you may not use this file except in compliance with the License. 7 | * // You may obtain a copy of the License at 8 | * // 9 | * // http://www.apache.org/licenses/LICENSE-2.0 10 | * // 11 | * // Unless required by applicable law or agreed to in writing, software 12 | * // distributed under the License is distributed on an "AS IS" BASIS, 13 | * // See the License for the specific language governing permissions and 14 | * // limitations under the License. 15 | */ 16 | 17 | package utils 18 | 19 | import ( 20 | "os" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/require" 24 | mgo "gopkg.in/mgo.v2" 25 | "gopkg.in/mgo.v2/bson" 26 | 27 | "github.com/moiot/gravity/pkg/mongo_test" 28 | ) 29 | 30 | var session *mgo.Session 31 | var db *mgo.Database 32 | 33 | func TestMain(m *testing.M) { 34 | mongoCfg := mongo_test.TestConfig() 35 | s, err := CreateMongoSession(&mongoCfg) 36 | if err != nil { 37 | panic(err) 38 | } 39 | session = s 40 | mongo_test.InitReplica(session) 41 | 42 | db = s.DB("test_connection") 43 | db.DropDatabase() 44 | 45 | ret := m.Run() 46 | 47 | session.Close() 48 | os.Exit(ret) 49 | } 50 | 51 | func TestCount(t *testing.T) { 52 | r := require.New(t) 53 | coll := db.C(t.Name()) 54 | 55 | cnt := 10 56 | for i := 0; i < cnt; i++ { 57 | r.NoError(coll.Insert(bson.M{"foo": i})) 58 | } 59 | 60 | r.EqualValues(cnt, Count(session, db.Name, t.Name())) 61 | r.NoError(coll.DropCollection()) 62 | r.EqualValues(0, Count(session, db.Name, t.Name())) 63 | } 64 | 65 | func TestBucketAuto(t *testing.T) { 66 | r := require.New(t) 67 | 68 | coll := db.C(t.Name()) 69 | cnt := 100 70 | for i := 0; i < cnt; i++ { 71 | r.NoError(coll.Insert(bson.M{"foo": i})) 72 | } 73 | 74 | BucketAuto(session, db.Name, t.Name(), 10, 2) 75 | } 76 | 77 | func TestGetMinMax(t *testing.T) { 78 | r := require.New(t) 79 | coll := db.C(t.Name()) 80 | 81 | cnt := 100 82 | 83 | for i := 0; i < cnt; i++ { 84 | r.NoError(coll.Insert(bson.M{"_id": i})) 85 | } 86 | 87 | mm, err := GetMinMax(session, db.Name, coll.Name) 88 | r.NoError(err) 89 | r.EqualValues(0, mm.Min) 90 | r.EqualValues(99, mm.Max) 91 | } 92 | -------------------------------------------------------------------------------- /pkg/utils/printer.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | 6 | log "github.com/sirupsen/logrus" 7 | ) 8 | 9 | // Version information. 10 | var ( 11 | Version = "None" 12 | BuildTS = "None" 13 | GitHash = "None" 14 | GitBranch = "None" 15 | ) 16 | 17 | // PrintRawInfo prints the version information without log info. 18 | func PrintRawInfo(app string) { 19 | fmt.Printf("Release Version (%s): %s\n", app, Version) 20 | fmt.Printf("Git Commit Hash: %s\n", GitHash) 21 | fmt.Printf("Git Branch: %s\n", GitBranch) 22 | fmt.Printf("UTC Build Time: %s\n", BuildTS) 23 | } 24 | 25 | // LogRotate prints the version information. 26 | func LogRawInfo(app string) { 27 | log.Infof("Welcome to %s.", app) 28 | log.Infof("Release Version: %s", Version) 29 | log.Infof("Git Commit Hash: %s", GitHash) 30 | log.Infof("Git Branch: %s", GitBranch) 31 | log.Infof("UTC Build Time: %s", BuildTS) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/utils/ptrs.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func NewBoolPtr(b bool) *bool { 4 | return &b 5 | } 6 | 7 | func NewStringPtr(s string) *string { 8 | return &s 9 | } 10 | -------------------------------------------------------------------------------- /pkg/utils/rdb_internal_txn_tag.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "math/rand" 7 | 8 | "github.com/moiot/gravity/pkg/consts" 9 | 10 | "github.com/juju/errors" 11 | ) 12 | 13 | // 14 | // /*drc:bidirectional*/ 15 | 16 | const ( 17 | dbNameV1 = "drc" 18 | tableNameV1 = "_drc_bidirection" 19 | 20 | tableNameV2 = consts.TxnTagTableName 21 | ) 22 | 23 | var dbNameV2 = consts.GravityDBName 24 | 25 | var tableDDLV2 = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.%s ( 26 | id INT(11) UNSIGNED NOT NULL, 27 | ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 28 | pipeline_name VARCHAR(255) NOT NULL, 29 | v BIGINT UNSIGNED NOT NULL DEFAULT 0, 30 | PRIMARY KEY (id) 31 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`, dbNameV2, tableNameV2) 32 | 33 | // Only for test purpose 34 | var TxnTagSQLFormat = fmt.Sprintf("insert into `%s`.`%s`", dbNameV2, tableNameV2) 35 | 36 | func IsCircularTrafficTag(db string, tbl string) bool { 37 | return (db == dbNameV1 && tbl == tableNameV1) || (db == dbNameV2 && tbl == tableNameV2) 38 | } 39 | 40 | func InitInternalTxnTags(db *sql.DB) error { 41 | _, err := db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", dbNameV2)) 42 | if err != nil { 43 | return errors.Trace(err) 44 | } 45 | 46 | _, err = db.Exec(tableDDLV2) 47 | if err != nil { 48 | return errors.Trace(err) 49 | } 50 | return nil 51 | } 52 | 53 | func GenerateTxnTagSQL(pipelineName string) string { 54 | id := rand.Int31n(999) + 1 55 | return fmt.Sprintf("insert into `%s`.`%s` (id,pipeline_name) values (%d,'%s') on duplicate key update v = v + 1, pipeline_name = '%s'", dbNameV2, tableNameV2, id, pipelineName, pipelineName) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/utils/retry/retry.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "math/rand" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | var ( 10 | DefaultSleep = 1 * time.Second 11 | ReadableDefaultSleep = "1s" 12 | ) 13 | 14 | // Func is the function to be executed and eventually retried. 15 | type Func func() error 16 | 17 | type Condition bool 18 | 19 | const ( 20 | Break Condition = true 21 | Continue Condition = false 22 | ) 23 | 24 | // ConditionFunc returns additional flag determine whether to break retry or not. 25 | type ConditionFunc func() (Condition, error) 26 | 27 | // HTTPFunc is the function to be executed and eventually retried. 28 | // The only difference from Func is that it expects an *http.Response on the first returning argument. 29 | type HTTPFunc func() (*http.Response, error) 30 | 31 | // Do runs the passed function until the number of retries is reached. 32 | // Whenever Func returns err it will sleep and Func will be executed again in a recursive fashion. 33 | // The sleep value is slightly modified on every retry (exponential backoff) to prevent the thundering herd problem (https://en.wikipedia.org/wiki/Thundering_herd_problem). 34 | // If no value is given to sleep it will defaults to 500ms. 35 | func Do(fn Func, retries int, sleep time.Duration) error { 36 | if sleep == 0 { 37 | sleep = DefaultSleep 38 | } 39 | 40 | if err := fn(); err != nil { 41 | retries-- 42 | if retries <= 0 { 43 | return err 44 | } 45 | 46 | // preventing thundering herd problem (https://en.wikipedia.org/wiki/Thundering_herd_problem) 47 | sleep += (time.Duration(rand.Int63n(int64(sleep)))) / 2 48 | time.Sleep(sleep) 49 | 50 | return Do(fn, retries, 2*sleep) 51 | } 52 | 53 | return nil 54 | } 55 | 56 | func DoCondition(fn ConditionFunc, retries int, sleep time.Duration) error { 57 | if sleep == 0 { 58 | sleep = DefaultSleep 59 | } 60 | 61 | var cond Condition 62 | var err error 63 | for i := 0; i < retries; i++ { 64 | cond, err = fn() 65 | 66 | if cond == Break { 67 | return err 68 | } 69 | 70 | // preventing thundering herd problem (https://en.wikipedia.org/wiki/Thundering_herd_problem) 71 | sleep += (time.Duration(rand.Int63n(int64(sleep)))) / 2 72 | time.Sleep(sleep) 73 | } 74 | 75 | return err 76 | } 77 | -------------------------------------------------------------------------------- /pkg/utils/struct.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "encoding/json" 4 | 5 | func MustAny2Map(i interface{}) (ret map[string]interface{}) { 6 | t, err := json.Marshal(i) 7 | if err != nil { 8 | panic(err) 9 | } 10 | err = json.Unmarshal(t, &ret) 11 | if err != nil { 12 | panic(err) 13 | } 14 | return 15 | } 16 | -------------------------------------------------------------------------------- /pkg/utils/testing.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | "io" 7 | "testing" 8 | ) 9 | 10 | func TestCaseMd5Name(t *testing.T) string { 11 | h := md5.New() 12 | io.WriteString(h, t.Name()) 13 | return fmt.Sprintf("%x", h.Sum(nil)) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/utils/type_cast.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "reflect" 5 | 6 | "github.com/juju/errors" 7 | ) 8 | 9 | // arg can be slice of any type 10 | func CastToSlice(arg interface{}) (out []interface{}, ok bool) { 11 | slice, success := TakeArg(arg, reflect.Slice) 12 | if !success { 13 | ok = false 14 | return 15 | } 16 | c := slice.Len() 17 | out = make([]interface{}, c) 18 | for i := 0; i < c; i++ { 19 | out[i] = slice.Index(i).Interface() 20 | } 21 | return out, true 22 | } 23 | 24 | func TakeArg(arg interface{}, kind reflect.Kind) (val reflect.Value, ok bool) { 25 | val = reflect.ValueOf(arg) 26 | if val.Kind() == kind { 27 | ok = true 28 | } 29 | return 30 | } 31 | 32 | func CastSliceInterfaceToSliceString(a []interface{}) ([]string, error) { 33 | aStrings := make([]string, len(a)) 34 | for i, c := range a { 35 | name, ok := c.(string) 36 | if !ok { 37 | return nil, errors.Errorf("should be an array of string") 38 | } 39 | aStrings[i] = name 40 | } 41 | return aStrings, nil 42 | } 43 | -------------------------------------------------------------------------------- /pkg/utils/utils_suite_test.go: -------------------------------------------------------------------------------- 1 | package utils_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | log "github.com/sirupsen/logrus" 9 | 10 | "github.com/moiot/gravity/pkg/logutil" 11 | ) 12 | 13 | func TestUtils(t *testing.T) { 14 | logutil.MustInitLogger(&logutil.LogConfig{Format: "text"}) 15 | log.SetOutput(GinkgoWriter) 16 | 17 | RegisterFailHandler(Fail) 18 | RunSpecs(t, "Utils Suite") 19 | } 20 | -------------------------------------------------------------------------------- /protocol/dcp/message.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package dcp; 3 | 4 | message Message { 5 | string id = 1; 6 | 7 | string tag = 2; 8 | 9 | oneof body { 10 | uint64 barrier = 3; 11 | Payload payload = 4; 12 | } 13 | 14 | uint64 timestamp = 14; 15 | string checksum = 15; 16 | } 17 | 18 | message Payload { 19 | string id = 1; 20 | string content = 2; 21 | } 22 | 23 | message Response { 24 | string id = 1; 25 | int32 code = 2; 26 | string msg = 3; 27 | } 28 | 29 | service DCPService { 30 | rpc Process (stream Message) returns (stream Response) {} 31 | } -------------------------------------------------------------------------------- /protocol/msgpb/message.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package msgpb; 3 | 4 | import "google/protobuf/timestamp.proto"; 5 | import "google/protobuf/wrappers.proto"; 6 | import "google/protobuf/any.proto"; 7 | 8 | 9 | enum DataSourceType { 10 | UNKNOWN_DATA_SOURCE = 0; 11 | MYSQL = 1; 12 | MONGODB = 2; 13 | TIDB = 3; 14 | REDIS = 4; 15 | CODIS = 5; 16 | 17 | } 18 | 19 | message Msg { 20 | // Version is the Msg definition version 21 | string version = 1; 22 | // Database is the database/schema name for MySQL 23 | // is the database for Mongo 24 | string database = 2; 25 | // Table is the table name for MySQL 26 | // is the collection name for Mongo 27 | string table = 3; 28 | // MsgType is the message type 29 | string msgType = 4; 30 | 31 | // Timestamp is the binlog event header timestamp for MySQL 32 | google.protobuf.Timestamp timestamp = 5; 33 | 34 | DMLMsg dmlMsg = 6; 35 | DDLMsg ddlMsg = 7; 36 | } 37 | 38 | // DDLMsg is not available for Mongo? 39 | message DDLMsg { 40 | // The DDL SQL 41 | string SQL = 2; 42 | } 43 | 44 | 45 | message DMLMsg { 46 | string Op = 1; 47 | // Data is the changed data 48 | map data = 2; 49 | // Old is the original data (if it is not empty) 50 | map old = 3; 51 | // Pks is the pkColumnName -> pkColumnValue mapping, 52 | map pks = 4; 53 | 54 | } 55 | 56 | message ConfigureRequest { 57 | map data = 1; 58 | } 59 | 60 | message ConfigureResponse { 61 | google.protobuf.StringValue error = 1; 62 | } 63 | 64 | message FilterRequest { 65 | Msg msg = 1; 66 | } 67 | 68 | message FilterResponse { 69 | Msg msg = 1; 70 | bool continueNext = 2; 71 | google.protobuf.StringValue error = 3; 72 | } 73 | 74 | service FilterPlugin { 75 | rpc Configure(ConfigureRequest) returns (ConfigureResponse); 76 | rpc Filter(FilterRequest) returns (FilterResponse); 77 | } --------------------------------------------------------------------------------