├── img ├── dataflow.png └── transaction_rate.png └── src ├── main ├── assembly │ ├── restart.sh │ ├── server.sh │ ├── run.sh │ ├── start.sh │ ├── stop.sh │ └── descriptor.xml ├── scala │ └── com │ │ └── neighborhood │ │ └── aka │ │ └── laplace │ │ └── estuary │ │ ├── mysql │ │ ├── schema │ │ │ ├── defs │ │ │ │ ├── ddl │ │ │ │ │ ├── .ruby-version │ │ │ │ │ ├── RemoveColumnMod.java │ │ │ │ │ ├── ColumnMod.java │ │ │ │ │ ├── InvalidSchemaError.java │ │ │ │ │ ├── DatabaseAlter.java │ │ │ │ │ ├── MaxwellSQLSyntaxError.java │ │ │ │ │ ├── TableTruncate.java │ │ │ │ │ ├── DatabaseDrop.java │ │ │ │ │ ├── ReparseSQLException.java │ │ │ │ │ ├── TableDrop.java │ │ │ │ │ ├── DatabaseCreate.java │ │ │ │ │ ├── AddColumnMod.java │ │ │ │ │ ├── ChangeColumnMod.java │ │ │ │ │ ├── TableAlter.java │ │ │ │ │ ├── TableCreate.java │ │ │ │ │ └── ColumnPosition.java │ │ │ │ └── columndef │ │ │ │ │ ├── FloatColumnDef.java │ │ │ │ │ ├── DecimalColumnDef.java │ │ │ │ │ ├── EnumeratedColumnDef.java │ │ │ │ │ ├── DateColumnDef.java │ │ │ │ │ ├── YearColumnDef.java │ │ │ │ │ ├── EnumColumnDef.java │ │ │ │ │ ├── JsonColumnDef.java │ │ │ │ │ ├── TimeColumnDef.java │ │ │ │ │ ├── DateTimeColumnDef.java │ │ │ │ │ ├── SetColumnDef.java │ │ │ │ │ ├── BigIntColumnDef.java │ │ │ │ │ ├── GeometryColumnDef.java │ │ │ │ │ ├── BitColumnDef.java │ │ │ │ │ └── IntColumnDef.java │ │ │ ├── CaseSensitivity.java │ │ │ ├── row │ │ │ │ └── RawJSONString.java │ │ │ ├── event │ │ │ │ ├── EventData.java │ │ │ │ ├── EventHeader.java │ │ │ │ ├── deserialization │ │ │ │ │ ├── MissingTableMapEventException.java │ │ │ │ │ ├── EventDataDeserializer.java │ │ │ │ │ ├── EventHeaderDeserializer.java │ │ │ │ │ ├── NullEventDataDeserializer.java │ │ │ │ │ ├── ChecksumType.java │ │ │ │ │ ├── XidEventDataDeserializer.java │ │ │ │ │ ├── EventDataDeserializationException.java │ │ │ │ │ ├── ByteArrayEventDataDeserializer.java │ │ │ │ │ ├── IntVarEventDataDeserializer.java │ │ │ │ │ ├── RowsQueryEventDataDeserializer.java │ │ │ │ │ ├── RotateEventDataDeserializer.java │ │ │ │ │ ├── FormatDescriptionEventDataDeserializer.java │ │ │ │ │ └── QueryEventDataDeserializer.java │ │ │ │ ├── PreviousGtidSetEventData.java │ │ │ │ ├── XidEventData.java │ │ │ │ ├── RowsQueryEventData.java │ │ │ │ ├── ByteArrayEventData.java │ │ │ │ ├── GtidEventData.java │ │ │ │ ├── Event.java │ │ │ │ ├── RotateEventData.java │ │ │ │ └── IntVarEventData.java │ │ │ ├── SdaSchemaMappingRule.scala │ │ │ └── tablemeta │ │ │ │ └── package.scala │ │ ├── task │ │ │ ├── SdaBean.scala │ │ │ └── Mysql2MysqlTaskInfoBean.scala │ │ ├── lifecycle │ │ │ └── reborn │ │ │ │ ├── sink │ │ │ │ ├── package.scala │ │ │ │ ├── MysqlInOrderSinkerEvent.scala │ │ │ │ ├── MysqlBinlogInOrderSimpleSinker.scala │ │ │ │ ├── MysqlBinlogInOrderSinkerCommand.scala │ │ │ │ └── MysqlBinlogInOrderMysqlSinkerManager.scala │ │ │ │ ├── batch │ │ │ │ ├── MysqlBinlogInOrderBatcherCommand.scala │ │ │ │ ├── MysqlBinlogInOrderBatcherEvent.scala │ │ │ │ ├── mappings │ │ │ │ │ ├── CanalEntryMappingFormat.scala │ │ │ │ │ └── DefaultCanalEntry2RowDataInfoMappingFormat.scala │ │ │ │ └── imp │ │ │ │ │ └── MysqlBinlogInOrderMysqlBatcher.scala │ │ │ │ ├── record │ │ │ │ ├── MysqlBinlogInOrderRecorderCommand.scala │ │ │ │ └── MysqlBinlogInOrderListenerEvent.scala │ │ │ │ ├── count │ │ │ │ ├── MysqlBinlogInOrderProcessingCounterCommand.scala │ │ │ │ └── MysqlInOrderProcessingCounter.scala │ │ │ │ ├── listen │ │ │ │ └── MysqlBinlogInOrderListenerCommand.scala │ │ │ │ ├── adapt │ │ │ │ ├── DefaultMysqlBinlogInOrderPowerAdapter.scala │ │ │ │ ├── MysqlBinlogInOrderPowerAdapterEvent.scala │ │ │ │ └── MysqlBinlogInOrderPowerAdapterCommand.scala │ │ │ │ ├── fetch │ │ │ │ ├── MysqlBinlogInOrderFetcherEvent.scala │ │ │ │ ├── MysqlBinlogInOrderFetcherCommand.scala │ │ │ │ └── DefaultMysqlBinlogInOrderDirectFetcher.scala │ │ │ │ └── control │ │ │ │ └── MysqlBinlogInOrderControllerCommand.scala │ │ ├── SettingConstant.scala │ │ ├── sink │ │ │ ├── MysqlSinkManagerImp.scala │ │ │ └── MysqlSinkBeanImp.scala │ │ ├── snapshot │ │ │ └── MysqlSnapshotCommand.scala │ │ ├── akkaUtil │ │ │ ├── DivideDDLRoundRobinRoutingGroup.scala │ │ │ └── DivideDDLRoundRobinRoutingLogic.scala │ │ ├── utils │ │ │ └── MysqlBinlogParser.scala │ │ └── source │ │ │ └── MysqlSourceBeanImp.scala │ │ ├── core │ │ ├── plugin │ │ │ └── Plugin.scala │ │ ├── eventsource │ │ │ ├── EstuaryEvent.scala │ │ │ └── EstuaryCommand.scala │ │ ├── lifecycle │ │ │ ├── worker │ │ │ │ ├── PositionRecorder.scala │ │ │ │ ├── SourceDataBatcher.scala │ │ │ │ ├── SourceDataFetcher.scala │ │ │ │ ├── SourceDataSinker.scala │ │ │ │ ├── SyncController.scala │ │ │ │ ├── HeartBeatListener.scala │ │ │ │ ├── WorkerType.scala │ │ │ │ ├── Status.scala │ │ │ │ └── worker.scala │ │ │ ├── prototype │ │ │ │ ├── ProcessingCountPrototype.scala │ │ │ │ ├── ActorPrototype.scala │ │ │ │ ├── PowerAdapterPrototype.scala │ │ │ │ ├── ProcessingCounterPrototype.scala │ │ │ │ ├── SourceDataSpecialBatcherPrototype.scala │ │ │ │ ├── SourceDataBatcherPrototype.scala │ │ │ │ ├── DataSourceFetcherPrototype.scala │ │ │ │ └── SourceDataBatcherManagerPrototype.scala │ │ │ └── package.scala │ │ ├── akkaUtil │ │ │ ├── theActorSystem.scala │ │ │ ├── EstuaryEventCollector.scala │ │ │ ├── SyncDaemonCommand.scala │ │ │ └── EstuaryEventListener.scala │ │ ├── snapshot │ │ │ └── SnapshotStatus.scala │ │ ├── offset │ │ │ ├── LogPositionManager.scala │ │ │ ├── ComparableOffset.scala │ │ │ └── ZooKeeperLogPositionManager.scala │ │ ├── util │ │ │ ├── zookeeper │ │ │ │ └── EsutaryZkClient.scala │ │ │ ├── JavaCommonUtil.java │ │ │ ├── spring │ │ │ │ └── SpringHttpRequestUtil.scala │ │ │ ├── message │ │ │ │ ├── MessageSender.scala │ │ │ │ └── MessageBody.java │ │ │ ├── SimpleEstuaryRingBuffer.scala │ │ │ └── SupportUtil.scala │ │ ├── task │ │ │ ├── PositionHandler.scala │ │ │ ├── package.scala │ │ │ ├── SinkManager.scala │ │ │ └── SourceManager.scala │ │ ├── source │ │ │ └── DataSourceConnection.scala │ │ ├── trans │ │ │ ├── MappingFormat.scala │ │ │ └── package.scala │ │ ├── implicit │ │ │ └── temp.scala │ │ ├── sink │ │ │ ├── SinkFunc.scala │ │ │ └── mysql │ │ │ │ └── MysqlSinkFunc.scala │ │ └── schema │ │ │ └── EventualSinkSchemaHandler.scala │ │ ├── bean │ │ ├── datasink │ │ │ ├── HdfsBean.scala │ │ │ ├── DataSinkBean.scala │ │ │ ├── SinkDataType.java │ │ │ ├── HBaseBean.scala │ │ │ └── MysqlSinkBean.scala │ │ ├── credential │ │ │ ├── DataSourceCredentialBean.scala │ │ │ ├── MongoCredentialBean.scala │ │ │ └── MysqlCredentialBean.scala │ │ ├── identity │ │ │ ├── DataSyncType.scala │ │ │ ├── SyncDataType.java │ │ │ ├── BaseExtractBean.scala │ │ │ └── BaseBean.scala │ │ ├── resource │ │ │ ├── SourceDataType.java │ │ │ ├── DataSourceBase.scala │ │ │ └── MongoSourceBean.scala │ │ ├── exception │ │ │ ├── power │ │ │ │ ├── GapTooLargeException.scala │ │ │ │ └── PowerControlException.scala │ │ │ ├── fetch │ │ │ │ ├── EmptyEntryException.scala │ │ │ │ ├── CannotFindOffsetException.scala │ │ │ │ ├── OutOfFetchRetryThersholdException.scala │ │ │ │ ├── NullOfDataSourceConnectionException.scala │ │ │ │ ├── FetchDataException.scala │ │ │ │ ├── UnexpectedEndStream.scala │ │ │ │ └── ConcernedDatabaseCannotFoundException.scala │ │ │ ├── schema │ │ │ │ ├── SchemaMatchException.scala │ │ │ │ ├── InvalidDdlException.scala │ │ │ │ ├── UpsertHBaseTableException.scala │ │ │ │ ├── UpsertHiveTableException.scala │ │ │ │ ├── SchemaInconsistentException.scala │ │ │ │ ├── SchemaIsNotInitializedException.scala │ │ │ │ ├── UnsupportedDdlTypeException.scala │ │ │ │ ├── SourceSchemaNoFoundException.scala │ │ │ │ ├── NoCorrespondingTableIdException.scala │ │ │ │ ├── RetrieveSchemaFailureException.scala │ │ │ │ ├── CannotFindTableCommentException.scala │ │ │ │ ├── DeplicateInitializationException.scala │ │ │ │ ├── NoCorrespondingSchemaException.scala │ │ │ │ ├── SchemaException.scala │ │ │ │ └── UpsertSchemaIntoDatabaseFailureException.scala │ │ │ ├── snapshot │ │ │ │ ├── TimeOverdueException.scala │ │ │ │ ├── UnmatchedTaskInfoException.scala │ │ │ │ ├── UnknownSnapshotStatusException.scala │ │ │ │ └── SnapshotException.scala │ │ │ ├── sink │ │ │ │ ├── SinkerAbnormalException.scala │ │ │ │ ├── KafkaSinkSendFailureException.scala │ │ │ │ └── SinkDataException.scala │ │ │ ├── control │ │ │ │ ├── RestartCommandException.scala │ │ │ │ ├── WorkerCannotFindException.scala │ │ │ │ ├── PartBuildFailureException.scala │ │ │ │ └── SyncControlException.scala │ │ │ ├── batch │ │ │ │ ├── UnsupportedEntryException.scala │ │ │ │ ├── UnsupportedEventTypeException.scala │ │ │ │ ├── StoreValueParseFailureException.scala │ │ │ │ └── BatchException.scala │ │ │ ├── other │ │ │ │ ├── TimeoutException.scala │ │ │ │ └── WorkerInitialFailureException.scala │ │ │ └── EstuaryException.scala │ │ ├── key │ │ │ ├── PartitionValueJsonKeyPartitioner.scala │ │ │ ├── JsonKeySerializer.java │ │ │ ├── JsonKeyPartitioner.java │ │ │ ├── PartitionStrategy.java │ │ │ ├── MultipleJsonKeyPartitionerJava.java │ │ │ └── MultipleJsonKeyPartitioner.scala │ │ └── support │ │ │ └── KafkaMessage.scala │ │ └── web │ │ ├── bean │ │ ├── TaskRequestBean.java │ │ ├── SdaRequestBean.java │ │ ├── SnapshotRequestBean.java │ │ ├── MysqlSinkBean.java │ │ ├── TableNameMappingBean.java │ │ ├── Mysql2MysqlRequestBean.java │ │ └── MysqlCredentialRequestBean.java │ │ ├── akkaUtil │ │ └── ActorRefHolder.scala │ │ ├── utils │ │ └── ValidationUtils.java │ │ ├── config │ │ ├── RestTemplateConfig.java │ │ ├── SwaggerConfig.java │ │ └── ConfigDataSourceConfig.java │ │ └── App.scala ├── antlr4 │ ├── imports │ │ ├── mysql_rename.g4 │ │ ├── mysql_truncate.g4 │ │ ├── mysql_create_database.g4 │ │ ├── mysql_alter_database.g4 │ │ ├── mysql_drop.g4 │ │ └── mysql_view.g4 │ └── com │ │ └── neighborhood.aka.laplace │ │ └── estuary │ │ └── schema │ │ └── ddl │ │ └── mysql.g4 └── resources │ ├── application.properties.templete │ └── application.conf.templete └── test └── scala └── com └── neighborhood └── aka └── laplace └── estuary ├── UnitSpec.scala └── mysql └── lifecycle └── ddl └── Antlr4DdlParserTest.scala /img/dataflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shouweikun/estuary/HEAD/img/dataflow.png -------------------------------------------------------------------------------- /src/main/assembly/restart.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd `dirname $0` 3 | stop.sh 4 | start.sh -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/.ruby-version: -------------------------------------------------------------------------------- 1 | 2.1.5 2 | -------------------------------------------------------------------------------- /img/transaction_rate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shouweikun/estuary/HEAD/img/transaction_rate.png -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/plugin/Plugin.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.plugin 2 | 3 | /** 4 | * Created by john_liu on 2018/5/26. 5 | */ 6 | trait Plugin { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/eventsource/EstuaryEvent.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.eventsource 2 | 3 | /** 4 | * Created by john_liu on 2019/1/14. 5 | */ 6 | trait EstuaryEvent 7 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/CaseSensitivity.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema; 2 | 3 | public enum CaseSensitivity { CASE_SENSITIVE, CONVERT_TO_LOWER, CONVERT_ON_COMPARE }; 4 | -------------------------------------------------------------------------------- /src/main/antlr4/imports/mysql_rename.g4: -------------------------------------------------------------------------------- 1 | grammar mysql_rename; 2 | import mysql_literal_tokens, mysql_idents; 3 | 4 | rename_table: RENAME TABLE rename_table_spec (',' rename_table_spec)*; 5 | rename_table_spec: table_name TO table_name; 6 | 7 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/datasink/HdfsBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.datasink 2 | 3 | /** 4 | * Created by john_liu on 2018/5/29. 5 | */ 6 | trait HdfsBean { 7 | 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/eventsource/EstuaryCommand.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.eventsource 2 | 3 | /** 4 | * Created by john_liu on 2019/1/14. 5 | */ 6 | trait EstuaryCommand { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/bean/TaskRequestBean.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.bean; 2 | 3 | /** 4 | * Created by john_liu on 2018/4/18. 5 | */ 6 | abstract public class TaskRequestBean { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /src/main/antlr4/imports/mysql_truncate.g4: -------------------------------------------------------------------------------- 1 | grammar mysql_truncate; 2 | import mysql_literal_tokens, mysql_idents; 3 | 4 | // https://dev.mysql.com/doc/refman/8.0/en/truncate-table.html 5 | // TRUNCATE [TABLE] tbl_name 6 | truncate_table: TRUNCATE table_name; 7 | -------------------------------------------------------------------------------- /src/main/antlr4/imports/mysql_create_database.g4: -------------------------------------------------------------------------------- 1 | grammar mysql_create_database; 2 | 3 | import mysql_literal_tokens, mysql_idents; 4 | 5 | create_database: 6 | CREATE (DATABASE | SCHEMA) if_not_exists? name (default_character_set | default_collation)*; 7 | 8 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/task/SdaBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.task 2 | 3 | /** 4 | * Created by john_liu on 2019/1/21. 5 | */ 6 | final case class SdaBean(tableMappingRule: Map[String, String]) 7 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/credential/DataSourceCredentialBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.credential 2 | 3 | /** 4 | * Created by john_liu on 2018/2/7. 5 | */ 6 | trait DataSourceCredentialBean { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/worker/PositionRecorder.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.worker 2 | 3 | /** 4 | * Created by john_liu on 2018/2/27. 5 | */ 6 | trait PositionRecorder extends worker{ 7 | 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/RemoveColumnMod.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | public class RemoveColumnMod extends ColumnMod { 4 | public RemoveColumnMod(String name) { 5 | super(name); 6 | } 7 | 8 | } 9 | -------------------------------------------------------------------------------- /src/test/scala/com/neighborhood/aka/laplace/estuary/UnitSpec.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary 2 | 3 | import org.scalamock.scalatest.MockFactory 4 | import org.scalatest._ 5 | 6 | abstract class UnitSpec extends FlatSpec with MockFactory with Matchers with 7 | OptionValues with Inside with Inspectors -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/worker/SourceDataBatcher.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.worker 2 | 3 | /** 4 | * Created by john_liu on 2018/2/6. 5 | */ 6 | trait SourceDataBatcher extends worker{ 7 | implicit val workerType = WorkerType.Batcher 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/worker/SourceDataFetcher.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.worker 2 | 3 | /** 4 | * Created by john_liu on 2018/2/6. 5 | */ 6 | trait SourceDataFetcher extends worker{ 7 | implicit val workerType = WorkerType.Fetcher 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/worker/SourceDataSinker.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.worker 2 | 3 | /** 4 | * Created by john_liu on 2018/2/6. 5 | */ 6 | trait SourceDataSinker extends worker{ 7 | implicit val workerType = WorkerType.Sinker 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/worker/SyncController.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.worker 2 | 3 | /** 4 | * Created by john_liu on 2018/2/6. 5 | */ 6 | trait SyncController extends worker{ 7 | implicit val workerType = WorkerType.SyncController 8 | } 9 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/ColumnMod.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | public abstract class ColumnMod { 4 | public String name; 5 | 6 | public ColumnMod(String name) { 7 | this.name = name; 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/akkaUtil/theActorSystem.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.akkaUtil 2 | 3 | import akka.actor.ActorSystem 4 | 5 | /** 6 | * Created by john_liu on 2018/2/1. 7 | */ 8 | trait theActorSystem { 9 | val system = ActorSystem("Estuary") 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/antlr4/imports/mysql_alter_database.g4: -------------------------------------------------------------------------------- 1 | grammar mysql_alter_database; 2 | 3 | import mysql_literal_tokens, mysql_idents; 4 | 5 | alter_database: ALTER (DATABASE | SCHEMA) name alter_database_definition; 6 | alter_database_definition: 7 | (default_character_set | default_collation)+ 8 | | UPGRADE DATA DIRECTORY NAME; 9 | 10 | -------------------------------------------------------------------------------- /src/main/antlr4/imports/mysql_drop.g4: -------------------------------------------------------------------------------- 1 | grammar mysql_drop; 2 | import mysql_literal_tokens, mysql_idents; 3 | 4 | drop_database: DROP (DATABASE | SCHEMA) if_exists? name; 5 | drop_table: DROP TEMPORARY? TABLE if_exists? table_name (',' table_name)* drop_table_options*; 6 | drop_table_options: (RESTRICT | CASCADE); 7 | 8 | if_exists: IF EXISTS; 9 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/InvalidSchemaError.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | public class InvalidSchemaError extends Exception { 4 | public InvalidSchemaError (String message) { super(message); } 5 | private static final long serialVersionUID = 1L; 6 | } 7 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/identity/DataSyncType.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.identity 2 | 3 | /** 4 | * Created by john_liu on 2018/2/7. 5 | */ 6 | object DataSyncType extends Enumeration{ 7 | 8 | type DataSyncType = Value 9 | val NORMAL = Value(0) 10 | val REMEDY = Value(1) 11 | 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/DatabaseAlter.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | public class DatabaseAlter extends SchemaChange { 4 | public String database; 5 | public String charset; 6 | 7 | public DatabaseAlter(String database) { 8 | this.database = database; 9 | } 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/datasink/DataSinkBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.datasink 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.sink.SinkFunc 4 | 5 | /** 6 | * Created by john_liu on 2018/2/7. 7 | */ 8 | trait DataSinkBean[S <: SinkFunc] { 9 | // var dataSinkType:DataSinkType 10 | def dataSinkType: String 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/MaxwellSQLSyntaxError.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | class MaxwellSQLSyntaxError extends RuntimeException { 4 | private static final long serialVersionUID = 140545518818187219L; 5 | 6 | public MaxwellSQLSyntaxError(String message) { 7 | super(message); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/identity/SyncDataType.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.identity; 2 | 3 | public enum SyncDataType { 4 | NORMAL("NORMAL"),REMADY("REMADY"); 5 | private String value; 6 | SyncDataType(String value){ 7 | this.value=value; 8 | } 9 | public String getValue(){ 10 | return value; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/resource/SourceDataType.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.resource; 2 | 3 | public enum SourceDataType { 4 | MYSQL("MYSQL"),MONGO("MONGO"); 5 | private String value; 6 | SourceDataType(String value){ 7 | this.value=value; 8 | } 9 | public String getValue(){ 10 | return value; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/TableTruncate.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | public class TableTruncate extends SchemaChange { 4 | public String database; 5 | final String table; 6 | 7 | public TableTruncate(String database, String table) { 8 | this.database = database; 9 | this.table = table; 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/resource/DataSourceBase.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.resource 2 | 3 | /** 4 | * Created by john_liu on 2018/2/7. 5 | * @tparam S 限定DataSource的类型,使Bean和类型参数绑定 6 | * @author neighborhood.aka.lapalce 7 | */ 8 | trait DataSourceBase[S] { 9 | 10 | // var dataSourceType : DataSourceType 11 | def dataSourceType: String 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/worker/HeartBeatListener.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.worker 2 | 3 | /** 4 | * Created by john_liu on 2018/2/6. 5 | * 6 | */ 7 | trait HeartBeatListener extends worker { 8 | implicit val workerType = WorkerType.Listener 9 | 10 | /** 11 | * 监听心跳 12 | */ 13 | protected def listenHeartBeats: Unit 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/DatabaseDrop.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | public class DatabaseDrop extends SchemaChange { 4 | public String database; 5 | public boolean ifExists; 6 | 7 | public DatabaseDrop(String database, boolean ifExists) { 8 | this.database = database; 9 | this.ifExists = ifExists; 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /src/main/resources/application.properties.templete: -------------------------------------------------------------------------------- 1 | info.build.artifactId=@project.artifactId@ 2 | info.build.version=@project.version@ 3 | #可以自己填 4 | server.port=9000 5 | spring.config.datasource.url=jdbc:mysql://xxx 6 | spring.config.datasource.username=xxx 7 | spring.config.datasource.password=xxx 8 | #这个sda专用的配置项 9 | spring.config.concerned.tableName=11 10 | sda.tableMapping.matedata.url=http://1xx 11 | sda.tableMapping.matedata.token=2sss -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/datasink/SinkDataType.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.datasink; 2 | 3 | public enum SinkDataType { 4 | KAFKA("KAFKA"), HBASE("HBASE"), MYSQL("MYSQL"),HDFS("HDFS"); 5 | private String value; 6 | 7 | SinkDataType(String value) { 8 | this.value = value; 9 | } 10 | 11 | public String getValue() { 12 | return value; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/identity/BaseExtractBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.identity 2 | 3 | /** 4 | * Created by john_liu on 2018/2/7. 5 | */ 6 | trait BaseExtractBean extends BaseBean { 7 | 8 | /** 9 | * 描述 10 | */ 11 | def describe: String = "" 12 | /** 13 | * 数据同步形式 14 | */ 15 | // var dataSyncType : DataSyncType 16 | def dataSyncType: String 17 | } 18 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/worker/WorkerType.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.worker 2 | 3 | /** 4 | * Created by john_liu on 2018/3/14. 5 | */ 6 | object WorkerType extends Enumeration { 7 | type WorkerType = Value 8 | val SyncController = Value(0) 9 | val Fetcher = Value(1) 10 | val Batcher = Value(2) 11 | val Sinker = Value(3) 12 | val Listener = Value(4) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/snapshot/SnapshotStatus.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.snapshot 2 | 3 | /** 4 | * Created by john_liu on 2018/7/9. 5 | */ 6 | object SnapshotStatus extends Enumeration { 7 | type SnapshotStatus = Value 8 | val NO_SNAPSHOT = Value(0) 9 | val ACCUMULATING_DATA = Value(1) 10 | val SUSPEND_4_WAKEUP = Value(2) 11 | val UNKNOWN = Value(3) 12 | 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/offset/LogPositionManager.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.offset 2 | 3 | import scala.util.Try 4 | 5 | /** 6 | * Created by john_liu on 2018/7/24. 7 | */ 8 | trait LogPositionManager[T] { 9 | 10 | def start: Try[Unit] 11 | 12 | def stop: Try[Unit] 13 | 14 | def getLatestIndexBy(destination: String): T 15 | 16 | def persistLogPosition(destination: String, logPosition: T): Unit 17 | } 18 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/identity/BaseBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.identity 2 | 3 | import java.util.Date 4 | 5 | /** 6 | * Created by john_liu on 2018/2/7. 7 | * 标识唯一任务 8 | */ 9 | trait BaseBean { 10 | /** 11 | * 同步任务的唯一id, 这个id表示同步任务的唯一标识 12 | */ 13 | def syncTaskId: String 14 | 15 | protected def createTime: Date 16 | 17 | protected def lastChange: Date 18 | 19 | protected def version = 0L 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/offset/ComparableOffset.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.offset 2 | 3 | /** 4 | * Created by john_liu on 2019/1/10. 5 | */ 6 | trait ComparableOffset[ComparableOffset] { 7 | 8 | def compare(other: ComparableOffset): Boolean 9 | 10 | def compare(other: ComparableOffset, smaller: Boolean): ComparableOffset = { 11 | if (compare(other) ^ smaller) other else this.asInstanceOf[ComparableOffset] 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/util/zookeeper/EsutaryZkClient.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.util.zookeeper 2 | 3 | ;; 4 | 5 | /** 6 | * Created by john_liu on 2018/7/24. 7 | */ 8 | class EsutaryZkClient( 9 | servers: String, 10 | sessionTimeout: Int = Int.MaxValue, 11 | connectionTimeout: Int 12 | ) 13 | // extends ZkClient 14 | { 15 | 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/ReparseSQLException.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | import org.antlr.v4.runtime.misc.ParseCancellationException; 4 | 5 | public class ReparseSQLException extends ParseCancellationException { 6 | private final String sql; 7 | 8 | public ReparseSQLException(String sql) { 9 | this.sql = sql; 10 | } 11 | 12 | public String getSQL() { 13 | return this.sql; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/FloatColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | public class FloatColumnDef extends ColumnDef { 4 | public FloatColumnDef() { } 5 | public FloatColumnDef(String name, String type, int pos) { 6 | super(name, type, pos); 7 | } 8 | 9 | public boolean signed; 10 | 11 | @Override 12 | public String toSQL(Object value) { 13 | return value.toString(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/task/PositionHandler.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.task 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.source.DataSourceConnection 4 | 5 | /** 6 | * 7 | * @tparam A log类型 8 | */ 9 | trait PositionHandler[A] { 10 | 11 | def persistLogPosition(destination:String,logPosition:A):Unit 12 | 13 | def getlatestIndexBy(destination:String):A 14 | 15 | def findStartPosition(conn:DataSourceConnection):A 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/worker/Status.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.worker 2 | 3 | /** 4 | * Created by john_liu on 2018/2/6. 5 | */ 6 | case object Status extends Enumeration { 7 | type Status = Value 8 | val OFFLINE = Value(0) 9 | val ONLINE = Value(1) 10 | val SUSPEND = Value(2) 11 | val ERROR = Value(4) 12 | val RESTARTING = Value(5) 13 | 14 | 15 | val BUSY = Value(6) 16 | val FREE= Value(7) 17 | 18 | 19 | } 20 | 21 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/sink/package.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn 2 | 3 | import com.neighborhood.aka.laplace.estuary.mysql.lifecycle.BinlogPositionInfo 4 | 5 | /** 6 | * Created by john_liu on 2019/1/30. 7 | */ 8 | package object sink { 9 | 10 | private[sink] final case class SqlList(list: List[String], binlogPositionInfo: Option[BinlogPositionInfo], shouldCount: Int, ts: Long = System.currentTimeMillis()) 11 | 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/TableDrop.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | public class TableDrop extends SchemaChange { 4 | public String database; 5 | public String table; 6 | public final boolean ifExists; 7 | 8 | public TableDrop(String database, String table, boolean ifExists) { 9 | this.database = database; 10 | this.table = table; 11 | this.ifExists = ifExists; 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/DecimalColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | import java.math.BigDecimal; 4 | 5 | public class DecimalColumnDef extends ColumnDef { 6 | public DecimalColumnDef(String name, String type, int pos) { 7 | super(name, type, pos); 8 | } 9 | 10 | @Override 11 | public String toSQL(Object value) { 12 | BigDecimal d = (BigDecimal) value; 13 | 14 | return d.toEngineeringString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/DatabaseCreate.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | public class DatabaseCreate extends SchemaChange { 4 | public final String database; 5 | private final boolean ifNotExists; 6 | public final String charset; 7 | 8 | public DatabaseCreate(String database, boolean ifNotExists, String charset) { 9 | this.database = database; 10 | this.ifNotExists = ifNotExists; 11 | this.charset = charset; 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/assembly/server.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd `dirname $0` 3 | if [ "$1" = "start" ]; then 4 | start.sh 5 | else 6 | if [ "$1" = "stop" ]; then 7 | stop.sh 8 | else 9 | if [ "$1" = "debug" ]; then 10 | start.sh debug 11 | else 12 | if [ "$1" = "restart" ]; then 13 | restart.sh 14 | else 15 | if [ "$1" = "dump" ]; then 16 | dump.sh 17 | else 18 | echo "ERROR: Please input argument: start or stop or debug or restart or dump" 19 | exit 1 20 | fi 21 | fi 22 | fi 23 | fi 24 | fi 25 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/source/DataSourceConnection.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.source 2 | 3 | import java.io.IOException 4 | 5 | /** 6 | * Created by john_liu on 2018/3/21. 7 | */ 8 | trait DataSourceConnection { 9 | @throws[IOException] 10 | def connect(): Unit 11 | 12 | @throws[IOException] 13 | def reconnect(): Unit 14 | 15 | @throws[IOException] 16 | def disconnect(): Unit 17 | 18 | def isConnected: Boolean 19 | 20 | def fork:DataSourceConnection 21 | } 22 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/EnumeratedColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | abstract public class EnumeratedColumnDef extends ColumnDef { 4 | 5 | protected String[] enumValues; 6 | 7 | public EnumeratedColumnDef(String name, String type, int pos, String [] enumValues) { 8 | super(name, type, pos); 9 | this.enumValues = enumValues; 10 | } 11 | 12 | public String[] getEnumValues() { 13 | return enumValues; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/datasink/HBaseBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.datasink 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.sink.mysql.MysqlSinkFunc 4 | 5 | /** 6 | * Created by john_liu on 2018/6/1. 7 | * 8 | * @todo 未完成 9 | */ 10 | 11 | trait HBaseBean extends DataSinkBean[MysqlSinkFunc] { 12 | 13 | def HbaseZookeeperQuorum: String 14 | def HabseZookeeperPropertyClientPort: String 15 | override val dataSinkType: String =SinkDataType.HBASE.toString 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/prototype/ProcessingCountPrototype.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.prototype 2 | 3 | import akka.actor.{Actor, ActorLogging} 4 | import com.neighborhood.aka.laplace.estuary.core.lifecycle.worker.ProcessingCounter 5 | 6 | /** 7 | * Created by john_liu on 2019/1/13. 8 | */ 9 | trait ProcessingCountPrototype extends ProcessingCounter with Actor with ActorLogging{ 10 | 11 | /** 12 | * 同步任务标签 13 | */ 14 | def syncTaskId:String 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/trans/MappingFormat.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.trans 2 | 3 | /** 4 | * Created by john_liu on 2018/5/1. 5 | */ 6 | trait MappingFormat[A, B] { 7 | 8 | 9 | 10 | /** 11 | * 拼接json用 12 | */ 13 | val START_JSON = "{" 14 | val END_JSON = "}" 15 | val START_ARRAY = "[" 16 | val END_ARRAY = "]" 17 | val KEY_VALUE_SPLIT = ":" 18 | val ELEMENT_SPLIT = "," 19 | val STRING_CONTAINER = "\"" 20 | 21 | def transform(x: A): B 22 | 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/power/GapTooLargeException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.power 2 | 3 | /** 4 | * Created by john_liu on 2018/7/26. 5 | */ 6 | class GapTooLargeException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends PowerControlException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/fetch/EmptyEntryException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.fetch 2 | 3 | /** 4 | * Created by john_liu on 2018/7/3. 5 | */ 6 | class EmptyEntryException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends FetchDataException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | 14 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/SchemaMatchException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/6/10. 5 | */ 6 | class SchemaMatchException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/snapshot/TimeOverdueException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.snapshot 2 | 3 | /** 4 | * Created by john_liu on 2018/7/9. 5 | */ 6 | class TimeOverdueException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SnapshotException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/batch/MysqlBinlogInOrderBatcherCommand.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.batch 2 | 3 | /** 4 | * Created by john_liu on 2018/10/10. 5 | */ 6 | sealed trait MysqlBinlogInOrderBatcherCommand 7 | 8 | object MysqlBinlogInOrderBatcherCommand { 9 | 10 | case object MysqlBinlogInOrderBatcherStart extends MysqlBinlogInOrderBatcherCommand 11 | 12 | case object MysqlBinlogInOrderBatcherCheckHeartbeats extends MysqlBinlogInOrderBatcherCommand 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/AddColumnMod.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | import com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef.ColumnDef; 4 | 5 | public class AddColumnMod extends ColumnMod { 6 | public ColumnDef definition; 7 | public ColumnPosition position; 8 | 9 | public AddColumnMod(String name, ColumnDef d, ColumnPosition position) { 10 | super(name); 11 | this.definition = d; 12 | this.position = position; 13 | } 14 | 15 | } 16 | 17 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/InvalidDdlException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2019/1/30. 5 | */ 6 | final class InvalidDdlException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/sink/SinkerAbnormalException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.sink 2 | 3 | /** 4 | * Created by john_liu on 2018/6/14. 5 | */ 6 | class SinkerAbnormalException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SinkDataException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/prototype/ActorPrototype.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.prototype 2 | 3 | import akka.actor.{Actor, ActorLogging} 4 | import com.neighborhood.aka.laplace.estuary.core.task.TaskManager 5 | 6 | /** 7 | * Created by john_liu on 2018/5/20. 8 | */ 9 | trait ActorPrototype extends Actor with ActorLogging { 10 | /** 11 | * 任务信息管理器 12 | */ 13 | def taskManager: TaskManager 14 | 15 | /** 16 | * 同步任务id 17 | */ 18 | def syncTaskId:String 19 | } 20 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/control/RestartCommandException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.control 2 | 3 | /** 4 | * Created by john_liu on 2018/5/28. 5 | */ 6 | class RestartCommandException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SyncControlException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/fetch/CannotFindOffsetException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.fetch 2 | 3 | /** 4 | * Created by john_liu on 2018/5/28. 5 | */ 6 | class CannotFindOffsetException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends FetchDataException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/UpsertHBaseTableException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/7/2. 5 | */ 6 | class UpsertHBaseTableException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/UpsertHiveTableException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/7/2. 5 | */ 6 | class UpsertHiveTableException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/sink/MysqlInOrderSinkerEvent.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.sink 2 | 3 | /** 4 | * Created by john_liu on 2018/10/10. 5 | */ 6 | sealed trait MysqlInOrderSinkerEvent 7 | object MysqlInOrderSinkerEvent { 8 | case object MysqlInOrderSinkerStarted extends MysqlInOrderSinkerEvent 9 | 10 | case object MysqlInOrderSinkerOffsetSaved extends MysqlInOrderSinkerEvent 11 | 12 | case object MysqlInOrderSinkerGetAbnormal extends MysqlInOrderSinkerEvent 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/ChangeColumnMod.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | import com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef.ColumnDef; 4 | 5 | public class ChangeColumnMod extends ColumnMod { 6 | public ColumnDef definition; 7 | public ColumnPosition position; 8 | 9 | public ChangeColumnMod(String name, ColumnDef d, ColumnPosition position ) { 10 | super(name); 11 | this.definition = d; 12 | this.position = position; 13 | 14 | } 15 | 16 | } 17 | 18 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/control/WorkerCannotFindException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.control 2 | 3 | /** 4 | * Created by john_liu on 2018/5/28. 5 | */ 6 | class WorkerCannotFindException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SyncControlException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/row/RawJSONString.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.row; 2 | 3 | // we wrap up raw json here and use the type of this class to allow 4 | // tunneling pre-serialized JSON data through a RowMap 5 | 6 | import java.io.Serializable; 7 | 8 | public class RawJSONString implements Serializable { 9 | private static final long serialVersionUID = -5600187114417848732L; 10 | 11 | public final String json; 12 | 13 | public RawJSONString(String json) { 14 | this.json = json; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/control/PartBuildFailureException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.control 2 | 3 | /** 4 | * Created by john_liu on 2018/6/11. 5 | */ 6 | class PartBuildFailureException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SyncControlException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/SchemaInconsistentException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/6/21. 5 | */ 6 | class SchemaInconsistentException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/record/MysqlBinlogInOrderRecorderCommand.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.record 2 | 3 | /** 4 | * Created by john_liu on 2018/10/10. 5 | */ 6 | sealed trait MysqlBinlogInOrderRecorderCommand 7 | 8 | object MysqlBinlogInOrderRecorderCommand { 9 | 10 | case object MysqlBinlogInOrderRecorderSavePosition extends MysqlBinlogInOrderRecorderCommand 11 | 12 | case object MysqlBinlogInOrderRecorderSaveLatestPosition extends MysqlBinlogInOrderRecorderCommand 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/batch/UnsupportedEntryException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.batch 2 | 3 | /** 4 | * Created by john_liu on 2018/7/13. 5 | */ 6 | class UnsupportedEntryException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends BatchException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/SchemaIsNotInitializedException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/6/5. 5 | */ 6 | class SchemaIsNotInitializedException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/UnsupportedDdlTypeException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/7/25. 5 | */ 6 | class UnsupportedDdlTypeException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/snapshot/UnmatchedTaskInfoException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.snapshot 2 | 3 | /** 4 | * Created by john_liu on 2018/7/13. 5 | */ 6 | class UnmatchedTaskInfoException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SnapshotException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/datasink/MysqlSinkBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.datasink 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.credential.MysqlCredentialBean 4 | import com.neighborhood.aka.laplace.estuary.core.sink.mysql.MysqlSinkFunc 5 | 6 | /** 7 | * Created by john_liu on 2019/1/13. 8 | */ 9 | trait MysqlSinkBean extends DataSinkBean[MysqlSinkFunc] { 10 | 11 | def dataSinkType: String = SinkDataType.MYSQL.toString 12 | 13 | /** 14 | * 数据汇Mysql的链接信息 15 | */ 16 | def credential: MysqlCredentialBean 17 | } 18 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/batch/UnsupportedEventTypeException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.batch 2 | 3 | /** 4 | * Created by john_liu on 2018/6/14. 5 | */ 6 | class UnsupportedEventTypeException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends BatchException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | 14 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/SourceSchemaNoFoundException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/8/29. 5 | */ 6 | class SourceSchemaNoFoundException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/NoCorrespondingTableIdException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/6/5. 5 | */ 6 | class NoCorrespondingTableIdException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/RetrieveSchemaFailureException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/6/1. 5 | */ 6 | class RetrieveSchemaFailureException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/sink/KafkaSinkSendFailureException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.sink 2 | 3 | /** 4 | * Created by john_liu on 2018/5/28. 5 | */ 6 | class KafkaSinkSendFailureException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SinkDataException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/snapshot/UnknownSnapshotStatusException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.snapshot 2 | 3 | /** 4 | * Created by john_liu on 2018/7/10. 5 | */ 6 | class UnknownSnapshotStatusException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SnapshotException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/credential/MongoCredentialBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.credential 2 | 3 | import scala.beans.BeanProperty 4 | 5 | /** 6 | * Created by john_liu on 2018/4/24. 7 | */ 8 | @BeanProperty 9 | final case class MongoCredentialBean( 10 | val username: Option[String] = None, 11 | val password: Option[String] = None, 12 | val database: Option[String] = None 13 | ) { 14 | 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/batch/StoreValueParseFailureException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.batch 2 | 3 | /** 4 | * Created by john_liu on 2018/5/30. 5 | */ 6 | class StoreValueParseFailureException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends BatchException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/CannotFindTableCommentException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/10/25. 5 | */ 6 | class CannotFindTableCommentException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/DeplicateInitializationException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/6/4. 5 | */ 6 | class DeplicateInitializationException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/NoCorrespondingSchemaException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/6/6. 5 | */ 6 | class NoCorrespondingSchemaException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/fetch/OutOfFetchRetryThersholdException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.fetch 2 | 3 | /** 4 | * Created by john_liu on 2018/5/28. 5 | */ 6 | class OutOfFetchRetryThersholdException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends FetchDataException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/batch/BatchException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.batch 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.exception.EstuaryException 4 | 5 | /** 6 | * Created by john_liu on 2018/5/30. 7 | */ 8 | abstract class BatchException( 9 | message: => String, 10 | cause: Throwable 11 | ) extends EstuaryException(message, cause) { 12 | def this(message: => String) = this(message, null) 13 | 14 | def this(cause: Throwable) = this("", cause) 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/fetch/NullOfDataSourceConnectionException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.fetch 2 | 3 | /** 4 | * Created by john_liu on 2018/5/28. 5 | */ 6 | class NullOfDataSourceConnectionException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends FetchDataException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/other/TimeoutException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.other 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.exception.EstuaryException 4 | 5 | /** 6 | * Created by john_liu on 2018/9/3. 7 | */ 8 | final class TimeoutException( 9 | message: => String, 10 | cause: Throwable 11 | ) extends EstuaryException(message, cause) { 12 | def this(message: => String) = this(message, null) 13 | 14 | def this(cause: Throwable) = this("", cause) 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/implicit/temp.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.`implicit` 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.trans.{RegTransPlugin, RegTransformation} 4 | 5 | /** 6 | * Created by john_liu on 2018/5/28. 7 | */ 8 | object temp extends App { 9 | val regRules = List(RegTransformation("1", "2"), RegTransformation("23", "34")) 10 | val transPlugin = RegTransPlugin(regRules) 11 | 12 | implicit class trans(str: String) { 13 | def transfer = RegTransPlugin.transfer(str, transPlugin) 14 | } 15 | 16 | println("1".transfer) 17 | } 18 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/SchemaException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.exception.EstuaryException 4 | 5 | /** 6 | * Created by john_liu on 2018/6/5. 7 | */ 8 | abstract class SchemaException ( 9 | message: => String, 10 | cause: Throwable 11 | ) extends EstuaryException(message, cause) { 12 | def this(message: => String) = this(message, null) 13 | 14 | def this(cause: Throwable) = this("", cause) 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/sink/SinkDataException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.sink 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.exception.EstuaryException 4 | 5 | /** 6 | * Created by john_liu on 2018/5/28. 7 | */ 8 | abstract class SinkDataException( 9 | message: => String, 10 | cause: Throwable 11 | ) extends EstuaryException(message, cause) { 12 | def this(message: => String) = this(message, null) 13 | 14 | def this(cause: Throwable) = this("", cause) 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/prototype/PowerAdapterPrototype.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.prototype 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.lifecycle.worker.PowerAdapter 4 | import com.neighborhood.aka.laplace.estuary.core.task.TaskManager 5 | 6 | /** 7 | * Created by john_liu on 2018/5/21. 8 | */ 9 | trait PowerAdapterPrototype extends ActorPrototype with PowerAdapter{ 10 | /** 11 | * 任务信息管理器 12 | */ 13 | val taskManager: TaskManager 14 | 15 | /** 16 | * 同步任务id 17 | */ 18 | override val syncTaskId:String 19 | } 20 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/fetch/FetchDataException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.fetch 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.exception.EstuaryException 4 | 5 | /** 6 | * Created by john_liu on 2018/5/28. 7 | */ 8 | abstract class FetchDataException( 9 | message: => String, 10 | cause: Throwable 11 | ) extends EstuaryException(message, cause) { 12 | def this(message: => String) = this(message, null) 13 | 14 | def this(cause: Throwable) = this("", cause) 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/fetch/UnexpectedEndStream.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.fetch 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.exception.EstuaryException 4 | 5 | /** 6 | * Created by john_liu on 2018/7/4. 7 | */ 8 | class UnexpectedEndStream ( 9 | message: => String, 10 | cause: Throwable 11 | ) extends EstuaryException(message, cause) { 12 | def this(message: => String) = this(message, null) 13 | 14 | def this(cause: Throwable) = this("", cause) 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/EstuaryException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception 2 | 3 | /** 4 | * Created by john_liu on 2018/5/28. 5 | */ 6 | abstract class EstuaryException( 7 | message: => String, 8 | cause: Throwable 9 | ) 10 | extends Exception(message, cause) { 11 | 12 | 13 | def this(message: => String) = this(message, null) 14 | 15 | def this(cause: Throwable) = this("", cause) 16 | 17 | var binlogJournalName = "" 18 | var binlogOffset = 4l 19 | } 20 | 21 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/control/SyncControlException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.control 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.exception.EstuaryException 4 | 5 | /** 6 | * Created by john_liu on 2018/5/28. 7 | */ 8 | abstract class SyncControlException( 9 | message: => String, 10 | cause: Throwable 11 | ) extends EstuaryException(message, cause) { 12 | def this(message: => String) = this(message, null) 13 | 14 | def this(cause: Throwable) = this("", cause) 15 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/fetch/ConcernedDatabaseCannotFoundException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.fetch 2 | 3 | /** 4 | * Created by john_liu on 2018/6/1. 5 | */ 6 | class ConcernedDatabaseCannotFoundException( 7 | message: => String, 8 | cause: Throwable 9 | ) extends FetchDataException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/snapshot/SnapshotException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.snapshot 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.exception.EstuaryException 4 | 5 | /** 6 | * Created by john_liu on 2018/7/9. 7 | */ 8 | abstract class SnapshotException ( 9 | message: => String, 10 | cause: Throwable 11 | ) extends EstuaryException(message, cause) { 12 | def this(message: => String) = this(message, null) 13 | 14 | def this(cause: Throwable) = this("", cause) 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/count/MysqlBinlogInOrderProcessingCounterCommand.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.count 2 | 3 | /** 4 | * Created by john_liu on 2019/1/11. 5 | */ 6 | sealed trait MysqlBinlogInOrderProcessingCounterCommand 7 | 8 | object MysqlBinlogInOrderProcessingCounterCommand { 9 | 10 | case class MysqlBinlogInOrderProcessingCounterUpdateCount(count: Long) extends MysqlBinlogInOrderProcessingCounterCommand 11 | 12 | case object MysqlBinlogInOrderProcessingCounterComputeCount extends MysqlBinlogInOrderProcessingCounterCommand 13 | 14 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/power/PowerControlException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.power 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.exception.EstuaryException 4 | 5 | /** 6 | * Created by john_liu on 2018/7/26. 7 | */ 8 | abstract class PowerControlException( 9 | message: => String, 10 | cause: Throwable 11 | ) extends EstuaryException(message, cause) { 12 | def this(message: => String) = this(message, null) 13 | 14 | def this(cause: Throwable) = this("", cause) 15 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/prototype/ProcessingCounterPrototype.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.prototype 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.lifecycle.worker.ProcessingCounter 4 | import com.neighborhood.aka.laplace.estuary.core.task.TaskManager 5 | 6 | /** 7 | * Created by john_liu on 2018/5/21. 8 | */ 9 | trait ProcessingCounterPrototype extends ActorPrototype with ProcessingCounter{ 10 | /** 11 | * 任务信息管理器 12 | */ 13 | val taskManager: TaskManager 14 | 15 | /** 16 | * 同步任务id 17 | */ 18 | override val syncTaskId:String 19 | } 20 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/schema/UpsertSchemaIntoDatabaseFailureException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/6/10. 5 | */ 6 | class UpsertSchemaIntoDatabaseFailureException ( 7 | message: => String, 8 | cause: Throwable 9 | ) extends SchemaException(message, cause) { 10 | def this(message: => String) = this(message, null) 11 | 12 | def this(cause: Throwable) = this("", cause) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/sink/SinkFunc.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.sink 2 | 3 | /** 4 | * Created by john_liu on 2018/2/7. 5 | * 6 | * @note 实现时必须保证线程安全 7 | * @author neighborhood.aka.laplace 8 | */ 9 | trait SinkFunc { 10 | 11 | /** 12 | * 创造出一个全新链接 13 | * 14 | * @return 15 | */ 16 | def fork: SinkFunc = ??? 17 | 18 | /** 19 | * 生命周期 20 | * 关闭 21 | */ 22 | def close: Unit 23 | 24 | /** 25 | * 生命周期 26 | * 开始 27 | */ 28 | def start: Unit 29 | 30 | /** 31 | * 检测，是否关闭 32 | * 33 | */ 34 | def isTerminated: Boolean 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/listen/MysqlBinlogInOrderListenerCommand.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.listen 2 | 3 | /** 4 | * Created by john_liu on 2018/10/10. 5 | */ 6 | sealed trait MysqlBinlogInOrderListenerCommand 7 | 8 | object MysqlBinlogInOrderListenerCommand { 9 | 10 | case object MysqlBinlogInOrderListenerStart extends MysqlBinlogInOrderListenerCommand 11 | 12 | case object MysqlBinlogInOrderListenerListen extends MysqlBinlogInOrderListenerCommand 13 | 14 | case object MysqlBinlogInOrderListenerStop extends MysqlBinlogInOrderListenerCommand 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/adapt/DefaultMysqlBinlogInOrderPowerAdapter.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.adapt 2 | 3 | import akka.actor.Props 4 | import com.neighborhood.aka.laplace.estuary.core.task.TaskManager 5 | 6 | /** 7 | * Created by john_liu on 2019/1/10. 8 | */ 9 | final class DefaultMysqlBinlogInOrderPowerAdapter(taskManager: TaskManager) extends MysqlBinlogInOrderPowerAdapter(taskManager) 10 | 11 | object DefaultMysqlBinlogInOrderPowerAdapter { 12 | def props(taskManager: TaskManager): Props = Props(new DefaultMysqlBinlogInOrderPowerAdapter(taskManager)) 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/batch/MysqlBinlogInOrderBatcherEvent.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.batch 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.eventsource.EstuaryEvent 4 | 5 | /** 6 | * Created by john_liu on 2018/10/10. 7 | */ 8 | sealed trait MysqlBinlogInOrderBatcherEvent extends EstuaryEvent 9 | 10 | object MysqlBinlogInOrderBatcherEvent { 11 | 12 | case object MysqlBinlogInOrderBatcherStarted extends MysqlBinlogInOrderBatcherEvent 13 | 14 | case object MysqlBinlogInOrderBatcherHeartbeatsChecked extends MysqlBinlogInOrderBatcherEvent 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/sink/MysqlBinlogInOrderSimpleSinker.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.sink 2 | 3 | import akka.actor.{Actor, ActorLogging} 4 | import com.neighborhood.aka.laplace.estuary.core.sink.SinkFunc 5 | import com.neighborhood.aka.laplace.estuary.core.sink.mysql.MysqlSinkFunc 6 | 7 | /** 8 | * Created by john_liu on 2019/1/29. 9 | */ 10 | class MysqlBinlogInOrderSimpleSinker( 11 | sinkFunc: MysqlSinkFunc 12 | ) extends Actor with ActorLogging { 13 | override def receive: Receive = ??? 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/util/JavaCommonUtil.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.util; 2 | 3 | import java.util.regex.Pattern; 4 | 5 | /** 6 | * Created by john_liu on 2018/5/4. 7 | */ 8 | public class JavaCommonUtil { 9 | public static boolean isInteger(String str) { 10 | Pattern pattern = Pattern.compile("^[-\\+]?[\\d]*$"); 11 | return pattern.matcher(str).matches(); 12 | } 13 | 14 | public static boolean isEmpty(Object str) { 15 | return str == null || str.equals(""); 16 | } 17 | 18 | public static boolean nonEmpty(String str) { 19 | return !isEmpty(str); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/exception/other/WorkerInitialFailureException.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.exception.other 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.exception.EstuaryException 4 | 5 | /** 6 | * Created by john_liu on 2019/1/14. 7 | */ 8 | final class WorkerInitialFailureException( 9 | message: => String, 10 | cause: Throwable 11 | ) extends EstuaryException(message, cause) { 12 | def this(message: => String) = this(message, null) 13 | 14 | def this(cause: Throwable) = this("", cause) 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/record/MysqlBinlogInOrderListenerEvent.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.record 2 | 3 | /** 4 | * Created by john_liu on 2018/10/10. 5 | */ 6 | sealed trait MysqlBinlogInOrderPositionRecorderEvent 7 | 8 | object MysqlBinlogInOrderPositionRecorderEvent { 9 | 10 | case object MysqlBinlogInOrderListenerStarted extends MysqlBinlogInOrderPositionRecorderEvent 11 | 12 | case object MysqlBinlogInOrderListenerListened extends MysqlBinlogInOrderPositionRecorderEvent 13 | 14 | case object MysqlBinlogInOrderListenerStopped extends MysqlBinlogInOrderPositionRecorderEvent 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/antlr4/com/neighborhood.aka.laplace/estuary/schema/ddl/mysql.g4: -------------------------------------------------------------------------------- 1 | grammar mysql; 2 | 3 | import mysql_literal_tokens, 4 | mysql_idents, 5 | mysql_alter_table, 6 | mysql_alter_database, 7 | mysql_create_database, 8 | mysql_create_table, 9 | mysql_drop, 10 | mysql_rename, 11 | mysql_view, 12 | mysql_truncate; 13 | 14 | parse: statement? 15 | EOF; 16 | 17 | statement: 18 | alter_table 19 | | alter_view 20 | | alter_database 21 | | create_database 22 | | create_table 23 | | create_view 24 | | drop_database 25 | | drop_table 26 | | drop_view 27 | | rename_table 28 | | truncate_table 29 | | BEGIN 30 | ; 31 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/akkaUtil/EstuaryEventCollector.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.akkaUtil 2 | 3 | import akka.actor.{Actor, ActorLogging} 4 | import com.neighborhood.aka.laplace.estuary.core.eventsource.EstuaryEvent 5 | 6 | /** 7 | * Created by john_liu on 2019/1/14. 8 | * 9 | * 针对时间溯源 eventSource使用 10 | * 收集返回事件 11 | * 另一方面，方便测试 12 | * 13 | * @author neighborhood.aka.laplace 14 | * 15 | */ 16 | trait EstuaryEventCollector extends Actor with ActorLogging { 17 | /** 18 | * 处理event事件 19 | * 20 | * @param event 待处理的事件 21 | * @return 返回的结果 22 | */ 23 | def handleEvent(event: EstuaryEvent): Any 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/worker/worker.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.worker 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.lifecycle.WorkerMessage 4 | 5 | /** 6 | * Created by john_liu on 2018/2/8. 7 | */ 8 | trait worker { 9 | /** 10 | * 错位次数阈值 11 | */ 12 | def errorCountThreshold: Int 13 | 14 | /** 15 | * 错位次数 16 | */ 17 | var errorCount: Int 18 | 19 | /** 20 | * 错误次数超过重试次数时，返回true 21 | */ 22 | def isCrashed: Boolean = errorCount >= errorCountThreshold 23 | 24 | /** 25 | * 错误处理 26 | */ 27 | def processError(e: Throwable, message: WorkerMessage) 28 | 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/bean/SdaRequestBean.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.bean; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * Created by john_liu on 2019/1/21. 7 | */ 8 | public class SdaRequestBean { 9 | 10 | public SdaRequestBean(Map tableMappingRule){ 11 | this.tableMappingRule = tableMappingRule; 12 | } 13 | private Map tableMappingRule; 14 | 15 | public Map getTableMappingRule() { 16 | return tableMappingRule; 17 | } 18 | 19 | public void setTableMappingRule(Map tableMappingRule) { 20 | this.tableMappingRule = tableMappingRule; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/task/package.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core 2 | 3 | import akka.actor.Props 4 | 5 | /** 6 | * Created by john_liu on 2018/5/9. 7 | */ 8 | package object task { 9 | 10 | sealed trait SyncTask { 11 | def props: Props 12 | 13 | def name: String 14 | 15 | def taskType: String = this.getClass.getName 16 | 17 | override def toString: String = { 18 | s"SyncTask,taskType:$taskType,prop:$props,name:$name" 19 | } 20 | } 21 | 22 | final case class Mysql2MysqlSyncTask( 23 | override val props: Props, 24 | override val name: String) extends SyncTask 25 | 26 | 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/antlr4/imports/mysql_view.g4: -------------------------------------------------------------------------------- 1 | grammar mysql_view; 2 | 3 | import mysql_literal_tokens, mysql_idents; 4 | 5 | /* 6 | This in an intentionally incomplete grammar for parsing VIEW statements. 7 | It's designed to parse up to the (SELECT *), as that cruft is too tricky to 8 | capture with a regular-expression based blacklist. 9 | */ 10 | 11 | alter_view: 12 | ALTER view_options* VIEW name; 13 | 14 | create_view: 15 | CREATE (OR REPLACE)? view_options* VIEW name; 16 | 17 | drop_view: 18 | DROP VIEW (IF EXISTS)? name (',' name)* (RESTRICT | CASCADE)?; 19 | 20 | view_options: 21 | ALGORITHM '=' (UNDEFINED | MERGE | TEMPTABLE) 22 | | DEFINER '=' (user | CURRENT_USER) 23 | | SQL SECURITY ( DEFINER | INVOKER ); 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/main/assembly/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd `dirname $0` 3 | BIN_DIR=`pwd` 4 | cd .. 5 | DEPLOY_DIR=`pwd` 6 | if [ -r "$DEPLOY_DIR"/bin/setenv.sh ]; then 7 | . "$DEPLOY_DIR"/bin/setenv.sh 8 | else 9 | echo "Cannot find $DEPLOY_DIR/bin/setenv.sh" 10 | echo "This file is needed to run this program" 11 | exit 1 12 | fi 13 | 14 | 15 | CLOUD_ENV="-Dserver.port=${PORT0} -Dlog.dir=/var/log/app/${productName}/@project.artifactId@-${PORT0}" 16 | 17 | echo -e "Starting the $SERVER_NAME ...\c" 18 | exec java $CLOUD_ENV $JAVA_OPTS $JAVA_MEM_OPTS $JAVA_DEBUG_OPTS $JAVA_JMX_OPTS -classpath $CONF_DIR:$LIB_JARS @main-class@ 19 | 20 | echo "OK!" 21 | PIDS=`ps -f | grep java | grep "$DEPLOY_DIR" | awk '{print $2}'` 22 | echo "PID: $PIDS" 23 | echo "STDOUT: $STDOUT_FILE" -------------------------------------------------------------------------------- /src/main/assembly/start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | export JAVA_HOME=/usr/local/jdk 3 | export PATH=$JAVA_HOME/bin:$PATH 4 | cd `dirname $0` 5 | BIN_DIR=`pwd` 6 | cd .. 7 | DEPLOY_DIR=`pwd` 8 | if [ -r "$DEPLOY_DIR"/bin/setenv.sh ]; then 9 | . "$DEPLOY_DIR"/bin/setenv.sh 10 | else 11 | echo "Cannot find $DEPLOY_DIR/bin/setenv.sh" 12 | echo "This file is needed to run this program" 13 | exit 1 14 | fi 15 | 16 | echo -e "Starting the $SERVER_NAME ...\c" 17 | nohup java $JAVA_OPTS $JAVA_MEM_OPTS $JAVA_DEBUG_OPTS $JAVA_JMX_OPTS -classpath $CONF_DIR:$LIB_JARS com.neighborhood.aka.laplace.estuary.web.App > /dev/null 2>&1 & 18 | 19 | echo "OK!" 20 | PIDS=`ps -f | grep java | grep "$DEPLOY_DIR" | awk '{print $2}'` 21 | echo "PID: $PIDS" 22 | printf -- '\n'; 23 | exit 0; -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/sink/MysqlBinlogInOrderSinkerCommand.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.sink 2 | 3 | import com.neighborhood.aka.laplace.estuary.mysql.lifecycle.BinlogPositionInfo 4 | 5 | /** 6 | * Created by john_liu on 2018/10/10. 7 | */ 8 | sealed trait MysqlBinlogInOrderSinkerCommand 9 | 10 | object MysqlBinlogInOrderSinkerCommand { 11 | 12 | case object MysqlInOrderSinkerStart extends MysqlBinlogInOrderSinkerCommand 13 | 14 | case class MysqlInOrderSinkerGetAbnormal(e: Throwable, offset: Option[BinlogPositionInfo]) extends MysqlBinlogInOrderSinkerCommand 15 | 16 | case object MysqlInOrderSinkerCheckBatch extends MysqlBinlogInOrderSinkerCommand 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/DateColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | public class DateColumnDef extends ColumnDef { 4 | public DateColumnDef(String name, String type, int pos) { 5 | super(name, type, pos); 6 | } 7 | 8 | @Override 9 | public String toSQL(Object value) { 10 | String formatted = DateFormatter.formatDate(value); 11 | if ( formatted == null ) 12 | return null; 13 | else 14 | return "'" + formatted + "'"; 15 | } 16 | 17 | @Override 18 | public Object asJSON(Object value) { 19 | if ( value instanceof Long && (Long) value == Long.MIN_VALUE ) 20 | return "0000-00-00"; 21 | 22 | return DateFormatter.formatDate(value); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/key/PartitionValueJsonKeyPartitioner.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.key 2 | 3 | import java.util 4 | 5 | import org.apache.kafka.clients.producer.Partitioner 6 | import org.apache.kafka.common.Cluster 7 | 8 | /** 9 | * Created by john_liu on 2018/9/29. 10 | */ 11 | class PartitionValueJsonKeyPartitioner extends Partitioner { 12 | override def partition(topic: String, key: scala.Any, keyBytes: Array[Byte], value: scala.Any, valueBytes: Array[Byte], cluster: Cluster): Int = key.asInstanceOf[BaseDataJsonKey].partitionStrategyValue % cluster.partitionCountForTopic(topic) 13 | 14 | 15 | override def close(): Unit = {} 16 | 17 | override def configure(configs: util.Map[String, _]): Unit = {} 18 | } 19 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/YearColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | import java.sql.Date; 4 | import java.util.Calendar; 5 | 6 | public class YearColumnDef extends ColumnDef { 7 | public YearColumnDef(String name, String type, int pos) { 8 | super(name, type, pos); 9 | } 10 | 11 | @Override 12 | public Object asJSON(Object value) { 13 | if ( value instanceof Date ) { 14 | Calendar calendar = Calendar.getInstance(); 15 | calendar.setTime(( java.sql.Date ) value); 16 | return calendar.get(Calendar.YEAR); 17 | } 18 | return value; 19 | } 20 | 21 | @Override 22 | public String toSQL(Object value) { 23 | return ((Integer)value).toString(); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/schema/EventualSinkSchemaHandler.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.schema 2 | 3 | /** 4 | * Created by john_liu on 2018/6/1. 5 | */ 6 | 7 | trait EventualSinkSchemaHandler[A] { 8 | 9 | 10 | /** 11 | * 创建db 12 | */ 13 | def createDb(info: A):Unit 14 | 15 | /** 16 | * 删除db 17 | */ 18 | def dropDb(info:A):Unit 19 | /** 20 | * 创建表 21 | */ 22 | def createTable(info:A):Unit 23 | 24 | /** 25 | * 删除表 26 | */ 27 | def dropTable(info:A):Unit 28 | 29 | /** 30 | * 查看库表是否存在 31 | */ 32 | 33 | def isExists(info:A):Boolean 34 | 35 | /** 36 | * 如果不存在就创建 37 | */ 38 | def createTableIfNotExists(info:A):Unit 39 | 40 | def dropTableIfExists(info:A):Unit 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/SettingConstant.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql 2 | 3 | /** 4 | * Created by john_liu on 2018/4/12. 5 | * 单位秒 6 | */ 7 | object SettingConstant { 8 | 9 | val COMPUTE_COST_CONSTANT = 3 10 | val COMPUTE_COUNT_CONSTANT = 100 //这个单位是ms 11 | val COMPUTE_FIRST_DELAY: Int = 5 12 | val OFFSET_SAVE_CONSTANT: Int = 60 13 | val CHECKSEND_CONSTANT = 20 14 | val BATCHER_START_DELAY = 1 15 | val FETCHER_START_DELAY = 5 16 | val POWER_CONTROL_CONSTANT = 5 //这个单位是ms 17 | val LISTEN_QUERY_TIMEOUT = 5 18 | val TASK_RESTART_DELAY = 10 19 | val HASH_MAPPING_VIRTUAL_NODES_FACTOR = 32 20 | val CHECK_STATUS_INTERVAL = 20 21 | val SINKER_FLUSH_INTERVAL = 300 //这个单位是毫秒 22 | val FAILURE_RETRY_BACKOFF = 100 //ms 23 | } 24 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/fetch/MysqlBinlogInOrderFetcherEvent.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.fetch 2 | 3 | /** 4 | * Created by john_liu on 2018/10/10. 5 | */ 6 | sealed trait MysqlBinlogInOrderFetcherEvent 7 | object MysqlBinlogInOrderFetcherEvent { 8 | case object MysqlBinlogInOrderFetcherStarted extends MysqlBinlogInOrderFetcherEvent 9 | case object MysqlBinlogInOrderFetcherReStarted extends MysqlBinlogInOrderFetcherEvent 10 | case object MysqlBinlogInOrderFetcherSuspended extends MysqlBinlogInOrderFetcherEvent 11 | case object MysqlBinlogInOrderFetcherBusyChanged extends MysqlBinlogInOrderFetcherEvent 12 | case object MysqlBinlogInOrderFetcherFreeChanged extends MysqlBinlogInOrderFetcherEvent 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/control/MysqlBinlogInOrderControllerCommand.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.control 2 | 3 | /** 4 | * Created by john_liu on 2019/1/13. 5 | */ 6 | sealed trait MysqlBinlogInOrderControllerCommand 7 | 8 | object MysqlBinlogInOrderControllerCommand { 9 | 10 | case object MysqlBinlogInOrderControllerRestart extends MysqlBinlogInOrderControllerCommand 11 | 12 | case object MysqlBinlogInOrderControllerStart extends MysqlBinlogInOrderControllerCommand 13 | 14 | case object MysqlBinlogInOrderControllerStopAndRestart extends MysqlBinlogInOrderControllerCommand //针对 online时的人为重启 15 | 16 | case object MysqlBinlogInOrderControllerCheckRunningInfo extends MysqlBinlogInOrderControllerCommand 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/sink/MysqlSinkManagerImp.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.sink 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.sink.mysql.MysqlSinkFunc 4 | import com.neighborhood.aka.laplace.estuary.core.source.MysqlHikariCpConnection 5 | import com.neighborhood.aka.laplace.estuary.core.task.SinkManager 6 | 7 | /** 8 | * Created by john_liu on 2019/1/15. 9 | */ 10 | trait MysqlSinkManagerImp extends SinkManager[MysqlSinkFunc] { 11 | /** 12 | * 数据汇bean 13 | */ 14 | override def sinkBean: MysqlSinkBeanImp 15 | 16 | /** 17 | * 构建数据汇 18 | * 19 | * @return sink 20 | */ 21 | override def buildSink: MysqlSinkFunc = { 22 | new MysqlSinkFunc(new MysqlHikariCpConnection(sinkBean.hikariConfig)) 23 | } 24 | 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/snapshot/MysqlSnapshotCommand.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.snapshot 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.snapshot.SnapshotStateMachine.Mysql2KafkaSnapshotTask 4 | 5 | /** 6 | * Created by john_liu on 2018/10/11. 7 | */ 8 | sealed trait MysqlSnapshotCommand 9 | 10 | object MysqlSnapshotCommand { 11 | 12 | case class MysqlSnapshotStartSnapshotTask(task: Mysql2KafkaSnapshotTask) extends MysqlSnapshotCommand 13 | 14 | case object MysqlSnapshotSuspend extends MysqlSnapshotCommand 15 | 16 | case object MysqlSnapshotTerminate extends MysqlSnapshotCommand 17 | 18 | case object MysqlSnapshotRestore extends MysqlSnapshotCommand 19 | 20 | case object MysqlSnapshotCheckRunningStatus extends MysqlSnapshotCommand 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/EnumColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | public class EnumColumnDef extends EnumeratedColumnDef { 4 | public EnumColumnDef(String name, String type, int pos, String[] enumValues) { 5 | super(name, type, pos, enumValues); 6 | } 7 | 8 | @Override 9 | public String toSQL(Object value) { 10 | return "'" + asString(value) + "'"; 11 | } 12 | 13 | @Override 14 | public String asJSON(Object value) { 15 | return asString(value); 16 | } 17 | 18 | private String asString(Object value) { 19 | if ( value instanceof String ) { 20 | return ( String ) value; 21 | } 22 | Integer i = (Integer) value; 23 | 24 | if ( i == 0 ) 25 | return null; 26 | else 27 | return enumValues[((Integer) value) - 1]; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/task/SinkManager.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.task 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.datasink.DataSinkBean 4 | import com.neighborhood.aka.laplace.estuary.core.sink.SinkFunc 5 | 6 | import scala.util.Try 7 | 8 | /** 9 | * Created by john_liu on 2019/1/13. 10 | */ 11 | trait SinkManager[S <: SinkFunc] { 12 | 13 | /** 14 | * 数据汇bean 15 | */ 16 | def sinkBean: DataSinkBean[S] 17 | 18 | 19 | /** 20 | * 数据汇 21 | */ 22 | def sink:S = sink_ 23 | 24 | private lazy val sink_ :S= buildSink 25 | 26 | /** 27 | * 构建数据汇 28 | * 29 | * @return sink 30 | */ 31 | def buildSink: S 32 | 33 | /** 34 | * 统一关闭资源 35 | */ 36 | def closeSink:Unit = Try{ 37 | if(!sink.isTerminated)sink.close 38 | } 39 | 40 | 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/task/SourceManager.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.task 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.resource.DataSourceBase 4 | import com.neighborhood.aka.laplace.estuary.core.source.DataSourceConnection 5 | 6 | import scala.util.Try 7 | 8 | /** 9 | * Created by john_liu on 2019/1/13. 10 | */ 11 | trait SourceManager[S <: DataSourceConnection] { 12 | /** 13 | * 数据源bean 14 | */ 15 | def sourceBean: DataSourceBase[S] 16 | 17 | /** 18 | * 数据源 19 | */ 20 | def source: S = source_ 21 | 22 | private lazy val source_ : S = buildSource 23 | 24 | /** 25 | * 构建数据源 26 | * 27 | * @return source 28 | */ 29 | def buildSource: S 30 | 31 | /** 32 | * 停止所有资源 33 | */ 34 | def closeSource:Unit = Try{if(source.isConnected)source.disconnect()} 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/TableAlter.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | import java.util.*; 4 | 5 | public class TableAlter extends SchemaChange { 6 | public String database; 7 | public String table; 8 | public ArrayList columnMods; 9 | public String newTableName; 10 | public String newDatabase; 11 | 12 | public String convertCharset; 13 | public String defaultCharset; 14 | public List pks; 15 | 16 | public TableAlter(String database, String table) { 17 | this.database = database; 18 | this.table = table; 19 | this.columnMods = new ArrayList<>(); 20 | } 21 | 22 | @Override 23 | public String toString() { 24 | return "TableAlter"; 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/akkaUtil/DivideDDLRoundRobinRoutingGroup.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.akkaUtil 2 | 3 | import akka.actor.ActorSystem 4 | import akka.dispatch.Dispatchers 5 | import akka.routing.{Group, Router} 6 | 7 | import scala.collection.immutable 8 | 9 | 10 | /** 11 | * Created by john_liu on 2018/4/3. 12 | * @note https://doc.akka.io/docs/akka/current/routing.html 13 | * 参考了Akka官方的Router的自定义 14 | */ 15 | final case class DivideDDLRoundRobinRoutingGroup(routeePaths:immutable.Iterable[String]) extends Group { 16 | 17 | override def paths(system: ActorSystem): immutable.Iterable[String] = routeePaths 18 | 19 | override def createRouter(system: ActorSystem): Router = new Router(new DivideDDLRoundRobinRoutingLogic) 20 | 21 | override def routerDispatcher: String = Dispatchers.DefaultDispatcherId 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/TableCreate.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef.ColumnDef; 7 | 8 | public class TableCreate extends SchemaChange { 9 | public String database; 10 | public String table; 11 | public List columns; 12 | public List pks; 13 | public String charset; 14 | 15 | public String likeDB; 16 | public String likeTable; 17 | public final boolean ifNotExists; 18 | 19 | public TableCreate (String database, String table, boolean ifNotExists) { 20 | this.database = database; 21 | this.table = table; 22 | this.ifNotExists = ifNotExists; 23 | this.columns = new ArrayList<>(); 24 | this.pks = new ArrayList<>(); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/akkaUtil/SyncDaemonCommand.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.akkaUtil 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.task.SyncTask 4 | 5 | /** 6 | * Created by john_liu on 2019/1/16. 7 | */ 8 | sealed trait SyncDaemonCommand 9 | 10 | object SyncDaemonCommand { 11 | 12 | case class ExternalStartCommand(syncTask: SyncTask) extends SyncDaemonCommand 13 | 14 | case object ExternalStartCommand 15 | 16 | 17 | case class ExternalRestartCommand(syncTaskId: String) extends SyncDaemonCommand 18 | 19 | case class ExternalStopCommand(syncTaskId: String) extends SyncDaemonCommand 20 | 21 | case object ExternalStopCommand extends SyncDaemonCommand 22 | 23 | case object ExternalGetAllRunningTask extends SyncDaemonCommand 24 | 25 | case class ExternalGetCertainRunningTask(syncTaskId:String) extends SyncDaemonCommand 26 | 27 | } 28 | 29 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/key/JsonKeySerializer.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.key; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.apache.kafka.common.serialization.Serializer; 5 | 6 | 7 | import java.io.IOException; 8 | import java.util.Map; 9 | 10 | /** 11 | * Created by duming on 16/12/7. 12 | */ 13 | public class JsonKeySerializer implements Serializer 14 | { 15 | private ObjectMapper mapper = new ObjectMapper(); 16 | @Override 17 | public void configure(Map map, boolean b) { 18 | 19 | } 20 | 21 | @Override 22 | public byte[] serialize(String s, Object o) { 23 | try { 24 | return mapper.writeValueAsBytes(o); 25 | } catch (IOException e) { 26 | e.printStackTrace(); 27 | } 28 | return null; 29 | } 30 | 31 | @Override 32 | public void close() { 33 | 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/akkaUtil/ActorRefHolder.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.akkaUtil 2 | 3 | import java.util.concurrent.ConcurrentHashMap 4 | 5 | import akka.actor.{ActorRef, Props} 6 | import com.neighborhood.aka.laplace.estuary.core.akkaUtil.{SyncDaemon, theActorSystem} 7 | 8 | import scala.collection.mutable 9 | 10 | /** 11 | * Created by john_liu on 2018/3/10. 12 | */ 13 | object ActorRefHolder extends theActorSystem { 14 | //todo 初始化守护Actor 15 | //todo 保留重要ActorRef 16 | val syncDaemon = system.actorOf(Props(classOf[SyncDaemon]), "syncDaemon") 17 | val actorRefMap: ConcurrentHashMap[String,ActorRef] = new ConcurrentHashMap[String,ActorRef]() 18 | 19 | 20 | 21 | def addNewTaskActorRef(key: String, value: ActorRef): Boolean = { 22 | if (Option(actorRefMap.get(key)).isEmpty) { 23 | actorRefMap.put(key, value) 24 | true 25 | } else { 26 | false 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/support/KafkaMessage.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.support 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.key.BaseDataJsonKey 4 | 5 | /** 6 | * Created by john_liu on 2018/3/8. 7 | */ 8 | class KafkaMessage { 9 | protected var baseDataJsonKey:BaseDataJsonKey = null 10 | protected var jsonValue:String = null 11 | 12 | 13 | 14 | def getBaseDataJsonKey: BaseDataJsonKey = baseDataJsonKey 15 | 16 | 17 | def this(baseDataJsonKey: BaseDataJsonKey, jsonValue: String) { 18 | this() 19 | this.baseDataJsonKey = baseDataJsonKey 20 | this.jsonValue = jsonValue 21 | } 22 | 23 | 24 | 25 | def setBaseDataJsonKey(baseDataJsonKey: BaseDataJsonKey): Unit = { 26 | this.baseDataJsonKey = baseDataJsonKey 27 | } 28 | 29 | 30 | def getJsonValue: String = jsonValue 31 | 32 | def setJsonValue(jsonValue: String): Unit = { 33 | this.jsonValue = jsonValue 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/JsonColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | 4 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization.json.JsonBinary; 5 | import com.neighborhood.aka.laplace.estuary.mysql.schema.row.RawJSONString; 6 | 7 | import java.io.IOException; 8 | 9 | 10 | 11 | public class JsonColumnDef extends ColumnDef { 12 | public JsonColumnDef(String name, String type, int pos) { 13 | super(name, type, pos); 14 | } 15 | 16 | @Override 17 | public Object asJSON(Object value) { 18 | try { 19 | byte[] bytes = (byte[]) value; 20 | String jsonString = bytes.length > 0 ? JsonBinary.parseAsString(bytes) : "null"; 21 | return new RawJSONString(jsonString); 22 | } catch (IOException e) { 23 | throw new RuntimeException(e); 24 | } 25 | } 26 | 27 | @Override 28 | public String toSQL(Object value) { 29 | return null; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/EventData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event; 17 | 18 | import java.io.Serializable; 19 | 20 | /** 21 | * @author Stanley Shyiko 22 | */ 23 | public interface EventData extends Serializable { 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/package.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core 2 | 3 | /** 4 | * Created by john_liu on 2018/2/9. 5 | */ 6 | package object lifecycle { 7 | 8 | sealed trait WorkerMessage{def msg: Any} 9 | 10 | final case class SyncControllerMessage(override val msg: Any) extends WorkerMessage 11 | 12 | final case class ListenerMessage(override val msg: Any) extends WorkerMessage 13 | 14 | final case class SinkerMessage(override val msg: Any) extends WorkerMessage 15 | 16 | final case class FetcherMessage(override val msg: Any) extends WorkerMessage 17 | 18 | final case class BatcherMessage(override val msg: Any) extends WorkerMessage 19 | 20 | final case class RecorderMessage(override val msg: Any) extends WorkerMessage 21 | 22 | final case class SnapshotJudgerMessage(override val msg: Any) extends WorkerMessage 23 | 24 | final case class PowerAdapterMessage(override val msg: Any) extends WorkerMessage 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/key/JsonKeyPartitioner.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.key; 2 | 3 | import org.apache.kafka.clients.producer.Partitioner; 4 | import org.apache.kafka.common.Cluster; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * Created by duming on 16/12/8. 10 | */ 11 | public class JsonKeyPartitioner implements Partitioner { 12 | @Override 13 | public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { 14 | BaseDataJsonKey opsKey = (BaseDataJsonKey) key; 15 | int tmp = (int) opsKey.getSyncTaskSequence() % cluster.partitionCountForTopic(topic); 16 | 17 | //根据唯一键值来进行数据分发, 这个partioion 值必 须在kafka的partition范围内, 如果不在, 会报TimeoutException, 并且得不到任何提示. 18 | return tmp < 0 ? -tmp : tmp; 19 | } 20 | 21 | @Override 22 | public void close() { 23 | 24 | } 25 | 26 | @Override 27 | public void configure(Map configs) { 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/trans/package.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.plugin.Plugin 4 | 5 | /** 6 | * Created by john_liu on 2018/5/26. 7 | */ 8 | package object trans { 9 | 10 | sealed trait TransPlugin extends Plugin 11 | 12 | case class RegTransformation(reg: String, target: String) 13 | 14 | case class RegTransPlugin(regs: List[RegTransformation]) { 15 | lazy val compiledRegMap = regs.map(x => (x.reg.r -> x.target)) 16 | 17 | def transfer(string: String): String = { 18 | lazy val re = compiledRegMap 19 | .withFilter( 20 | _._1 21 | .findFirstIn(string) 22 | .isDefined 23 | ) 24 | .map(_._2) 25 | 26 | if (re.isEmpty) string else re(0) 27 | } 28 | 29 | } 30 | case object RegTransPlugin{ 31 | def transfer(str:String,regTransPlugin: RegTransPlugin):String = { 32 | regTransPlugin.transfer(str) 33 | } 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/utils/ValidationUtils.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.utils; 2 | 3 | import org.springframework.util.Assert; 4 | 5 | import java.util.List; 6 | 7 | public class ValidationUtils { 8 | 9 | public static void notAllNull(String message, Object... obj) { 10 | int i = 0; 11 | for (Object o : obj) { 12 | if (o != null) { 13 | i++; 14 | } 15 | } 16 | Assert.isTrue(i > 0, message); 17 | } 18 | 19 | public static void notNull(Object obj, String message) { 20 | Assert.notNull(obj, message); 21 | } 22 | 23 | public static void notblank(String obj, String message) { 24 | Assert.isTrue(obj.trim() != "",message); 25 | } 26 | public static void notZero(long num,String message) { 27 | Assert.isTrue(num != 0,message); 28 | } 29 | public static void notEmpty(List obj, String message) { 30 | Assert.isTrue(!obj.isEmpty(),message); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/ddl/ColumnPosition.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl; 2 | 3 | import com.neighborhood.aka.laplace.estuary.mysql.schema.defs.Table; 4 | 5 | public class ColumnPosition { 6 | enum Position { FIRST, AFTER, DEFAULT }; 7 | 8 | public Position position; 9 | public String afterColumn; 10 | 11 | public ColumnPosition() { 12 | this.position = Position.DEFAULT; 13 | } 14 | 15 | public int index(Table t, Integer defaultIndex) throws InvalidSchemaError { 16 | switch(position) { 17 | case FIRST: 18 | return 0; 19 | case DEFAULT: 20 | if ( defaultIndex != null ) 21 | return defaultIndex; 22 | else 23 | return t.getColumnList().size(); 24 | 25 | case AFTER: 26 | int afterIdx = t.findColumnIndex(afterColumn); 27 | if ( afterIdx == -1 ) 28 | throw new InvalidSchemaError("Could not find column " + afterColumn + " (needed in AFTER statement)"); 29 | return afterIdx + 1; 30 | } 31 | return -1; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/assembly/stop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd `dirname $0` 3 | BIN_DIR=`pwd` 4 | cd .. 5 | DEPLOY_DIR=`pwd` 6 | CONF_DIR=$DEPLOY_DIR/conf 7 | 8 | SERVER_NAME=`sed '/spring.application.name/!d;s/.*=//' conf/application.properties | tr -d '\r'` 9 | 10 | if [ -z "$SERVER_NAME" ]; then 11 | SERVER_NAME=`hostname` 12 | fi 13 | 14 | PIDS=`ps -ef | grep java | grep "$CONF_DIR" |awk '{print $2}'` 15 | if [ -z "$PIDS" ]; then 16 | echo "ERROR: The $SERVER_NAME does not started!" 17 | exit 1 18 | fi 19 | 20 | if [ "$1" != "skip" ]; then 21 | $BIN_DIR/dump.sh 22 | fi 23 | 24 | echo -e "Stopping the $SERVER_NAME ...\c" 25 | for PID in $PIDS ; do 26 | kill $PID > /dev/null 2>&1 27 | done 28 | 29 | COUNT=0 30 | while [ $COUNT -lt 1 ]; do 31 | echo -e ".\c" 32 | sleep 1 33 | COUNT=1 34 | for PID in $PIDS ; do 35 | PID_EXIST=`ps -f -p $PID | grep java` 36 | if [ -n "$PID_EXIST" ]; then 37 | COUNT=0 38 | break 39 | fi 40 | done 41 | done 42 | 43 | echo "OK!" 44 | echo "PID: $PIDS" 45 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/TimeColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | import java.sql.Time; 4 | import java.sql.Timestamp; 5 | 6 | public class TimeColumnDef extends ColumnDefWithLength { 7 | public TimeColumnDef(String name, String type, int pos, Long columnLength) { 8 | super(name, type, pos, columnLength); 9 | } 10 | 11 | protected String formatValue(Object value) { 12 | if ( value instanceof Timestamp ) { 13 | Time time = new Time(((Timestamp) value).getTime()); 14 | String timeAsStr = String.valueOf(time); 15 | 16 | return appendFractionalSeconds(timeAsStr, ((Timestamp) value).getNanos(), this.columnLength); 17 | 18 | } else if ( value instanceof Long ) { 19 | Time time = new Time((Long) value / 1000); 20 | String timeAsStr = String.valueOf(time); 21 | 22 | return appendFractionalSeconds(timeAsStr, (int) ((Long) value % 1000000) * 1000, this.columnLength); 23 | } else { 24 | return String.valueOf((Time) value); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/adapt/MysqlBinlogInOrderPowerAdapterEvent.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.adapt 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.eventsource.EstuaryEvent 4 | 5 | /** 6 | * Created by john_liu on 2019/1/19. 7 | */ 8 | sealed trait MysqlBinlogInOrderPowerAdapterEvent extends EstuaryEvent 9 | 10 | object MysqlBinlogInOrderPowerAdapterEvent { 11 | case class MysqlBinlogInOrderPowerAdapterCostUpdated(cost: Long) extends MysqlBinlogInOrderPowerAdapterEvent 12 | 13 | case class MysqlBinlogInOrderPowerAdapterFetcherCountUpdated(count: Long) extends MysqlBinlogInOrderPowerAdapterEvent //special case 因为fetcher的计数特性 14 | 15 | case object MysqlBinlogInOrderPowerAdapterControlled extends MysqlBinlogInOrderPowerAdapterEvent 16 | 17 | case object MysqlBinlogInOrderPowerAdapterCostComputed extends MysqlBinlogInOrderPowerAdapterEvent 18 | 19 | case class MysqlBinlogInOrderPowerAdapterFetchDelayed(delay:Long) extends MysqlBinlogInOrderPowerAdapterEvent 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/utils/MysqlBinlogParser.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.utils 2 | 3 | import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert 4 | import com.alibaba.otter.canal.protocol.CanalEntry 5 | import com.taobao.tddl.dbsync.binlog.LogEvent 6 | 7 | /** 8 | * Created by john_liu on 2018/2/2. 9 | */ 10 | final class MysqlBinlogParser extends LogEventConvert { 11 | 12 | private var filterTimestamp: Long = 0 13 | start() 14 | 15 | def parse(eventOption: Option[LogEvent]): Option[CanalEntry.Entry] = eventOption.flatMap { 16 | event => 17 | event.getHeader.getType match { 18 | case LogEvent.ROTATE_EVENT => Option(parse(event)) 19 | case LogEvent.QUERY_EVENT => Option(parse(event)) 20 | case LogEvent.XID_EVENT => Option(parse(event)) 21 | case _ => if (event.getWhen *1000 >= filterTimestamp) Option(parse(event)) else None 22 | 23 | } 24 | } 25 | 26 | 27 | 28 | def setFilterTimestamp(ts: Long) = this.filterTimestamp = ts 29 | } 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/config/RestTemplateConfig.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.http.client.ClientHttpRequestFactory; 6 | import org.springframework.http.client.SimpleClientHttpRequestFactory; 7 | import org.springframework.web.client.RestTemplate; 8 | 9 | /** 10 | * Created by john_liu on 2019/1/21. 11 | */ 12 | @Configuration 13 | public class RestTemplateConfig { 14 | 15 | @Bean("restTemplate") 16 | public RestTemplate restTemplate(ClientHttpRequestFactory factory) { 17 | return new RestTemplate(factory); 18 | } 19 | 20 | @Bean 21 | public ClientHttpRequestFactory simpleClientHttpRequestFactory() { 22 | SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); 23 | factory.setReadTimeout(5000);//单位为ms 24 | factory.setConnectTimeout(5000);//单位为ms 25 | return factory; 26 | } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/DateTimeColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | import java.sql.Timestamp; 4 | 5 | public class DateTimeColumnDef extends ColumnDefWithLength { 6 | public DateTimeColumnDef(String name, String type, int pos, Long columnLength) { 7 | super(name, type, pos, columnLength); 8 | } 9 | 10 | final private boolean isTimestamp = getType().equals("timestamp"); 11 | protected String formatValue(Object value) { 12 | // special case for those broken mysql dates. 13 | if ( value instanceof Long ) { 14 | Long v = (Long) value; 15 | if ( v == Long.MIN_VALUE || (v == 0L && isTimestamp) ) 16 | return appendFractionalSeconds("0000-00-00 00:00:00", 0, columnLength); 17 | } 18 | 19 | Timestamp ts = DateFormatter.extractTimestamp(value); 20 | String dateString = DateFormatter.formatDateTime(value, ts); 21 | if ( dateString == null ) 22 | return null; 23 | else 24 | return appendFractionalSeconds(dateString, ts.getNanos(), columnLength); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/adapt/MysqlBinlogInOrderPowerAdapterCommand.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.adapt 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.eventsource.EstuaryCommand 4 | 5 | /** 6 | * Created by john_liu on 2018/10/10. 7 | */ 8 | sealed trait MysqlBinlogInOrderPowerAdapterCommand extends EstuaryCommand 9 | 10 | object MysqlBinlogInOrderPowerAdapterCommand { 11 | 12 | case class MysqlBinlogInOrderPowerAdapterUpdateCost(cost: Long) extends MysqlBinlogInOrderPowerAdapterCommand 13 | 14 | case class MysqlBinlogInOrderPowerAdapterUpdateFetcherCount(count: Long) extends MysqlBinlogInOrderPowerAdapterCommand //special case 因为fetcher的计数特性 15 | 16 | case object MysqlBinlogInOrderPowerAdapterControl extends MysqlBinlogInOrderPowerAdapterCommand 17 | 18 | case object MysqlBinlogInOrderPowerAdapterComputeCost extends MysqlBinlogInOrderPowerAdapterCommand 19 | 20 | case class MysqlBinlogInOrderPowerAdapterDelayFetch(delay:Long) extends MysqlBinlogInOrderPowerAdapterCommand 21 | 22 | } 23 | 24 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/key/PartitionStrategy.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.key; 2 | 3 | import org.omg.CORBA.TRANSACTION_MODE; 4 | 5 | /** 6 | * Created by john_liu on 2018/5/6. 7 | */ 8 | public enum PartitionStrategy { 9 | MOD("MOD"), 10 | PRIMARY_KEY("PRIMARY_KEY"), 11 | DATABASE_TABLE("DATABASE_TABLE"), 12 | TRANSACTION("TRANSACTION"); //todo 严格按照事务实现 13 | 14 | 15 | private String value; 16 | 17 | PartitionStrategy(String value) { 18 | this.value = value; 19 | } 20 | 21 | public String getValue() { 22 | return value; 23 | } 24 | 25 | public static PartitionStrategy fromString(String str) { 26 | switch (str.toUpperCase()) { 27 | case "MOD":return PartitionStrategy.MOD; 28 | case "TRANSACTION":return PartitionStrategy.TRANSACTION; 29 | case "DATABASE_TABLE":return PartitionStrategy.DATABASE_TABLE; 30 | case "PRIMARY_KEY":return PartitionStrategy.PRIMARY_KEY; 31 | default:return PartitionStrategy.PRIMARY_KEY; //默认是primaryKey 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/App.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web 2 | 3 | import org.springframework.boot.SpringApplication 4 | import org.springframework.boot.autoconfigure.{EnableAutoConfiguration, SpringBootApplication} 5 | import org.springframework.boot.web.servlet.ServletComponentScan 6 | 7 | import org.springframework.boot.autoconfigure.dao.PersistenceExceptionTranslationAutoConfiguration 8 | import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration 9 | import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration 10 | 11 | /** 12 | * Created by john_liu on 2018/3/11. 13 | */ 14 | @EnableAutoConfiguration(exclude = Array( 15 | classOf[MongoAutoConfiguration], 16 | classOf[MongoDataAutoConfiguration], 17 | classOf[PersistenceExceptionTranslationAutoConfiguration])) 18 | @SpringBootApplication 19 | @ServletComponentScan(basePackages = Array("com.neighborhood.aka.laplace.web")) 20 | class App { 21 | 22 | } 23 | 24 | 25 | object App { 26 | def main(args: Array[String]): Unit = { 27 | SpringApplication.run(classOf[App]) 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/SetColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | 6 | import org.apache.commons.lang3.StringUtils; 7 | 8 | public class SetColumnDef extends EnumeratedColumnDef { 9 | public SetColumnDef(String name, String type, int pos, String[] enumValues) { 10 | super(name, type, pos, enumValues); 11 | } 12 | 13 | @Override 14 | public String toSQL(Object value) { 15 | return "'" + StringUtils.join(asList(value), "'") + "'"; 16 | } 17 | 18 | @Override 19 | public Object asJSON(Object value) { 20 | return asList(value); 21 | } 22 | 23 | private ArrayList asList(Object value) { 24 | if ( value instanceof String ) { 25 | return new ArrayList<>(Arrays.asList((( String ) value).split(","))); 26 | } 27 | ArrayList values = new ArrayList<>(); 28 | long v = (Long) value; 29 | for(int i = 0; i < enumValues.length; i++) { 30 | if ( ((v >> i) & 1) == 1 ) { 31 | values.add(enumValues[i]); 32 | } 33 | } 34 | return values; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/EventHeader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event; 17 | 18 | import java.io.Serializable; 19 | 20 | /** 21 | * @author Stanley Shyiko 22 | */ 23 | public interface EventHeader extends Serializable { 24 | 25 | long getTimestamp(); 26 | EventType getEventType(); 27 | long getServerId(); 28 | long getHeaderLength(); 29 | long getDataLength(); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/assembly/descriptor.xml: -------------------------------------------------------------------------------- 1 | 2 | assembly 3 | 4 | zip 5 | 6 | 7 | 8 | lib 9 | false 10 | 11 | 12 | 13 | 14 | ${project.basedir}/src/main/resources 15 | /conf 16 | 17 | 18 | **/* 19 | 20 | 0644 21 | true 22 | 23 | 24 | ${project.basedir}/src/main/assembly 25 | /bin 26 | true 27 | 0755 28 | 29 | *.sh 30 | 31 | unix 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/MissingTableMapEventException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import java.io.IOException; 19 | 20 | /** 21 | * @author Stanley Shyiko 22 | */ 23 | public class MissingTableMapEventException extends IOException { 24 | 25 | public MissingTableMapEventException(String message) { 26 | super(message); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/util/spring/SpringHttpRequestUtil.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.util.spring 2 | 3 | import org.slf4j.LoggerFactory 4 | import org.springframework.http.HttpHeaders 5 | import org.springframework.web.client.RestTemplate 6 | 7 | import scala.reflect.ClassTag 8 | import scala.util.Try 9 | 10 | /** 11 | * Created by john_liu on 2018/7/25. 12 | */ 13 | object SpringHttpRequestUtil { 14 | 15 | import scala.reflect._ 16 | 17 | private lazy val restTemplate = new RestTemplate 18 | private lazy val log = LoggerFactory.getLogger("SpringHttpRequestUtil") 19 | 20 | def doPost[A, B:ClassTag](url: String, request: A, uriParam: AnyRef*): Try[B] = Try { 21 | log.info(s"start post $url:$request") 22 | 23 | 24 | restTemplate.postForObject(url, request, classTag[B].runtimeClass, uriParam).asInstanceOf[B] 25 | } 26 | 27 | def doGet[A:ClassTag](url: String, uriParam: Any*): Try[A] = Try { 28 | log.info(s"start get $url.$uriParam") 29 | lazy val runtimeClass = classTag[A].runtimeClass 30 | restTemplate.getForObject(url, runtimeClass, uriParam).asInstanceOf[A] 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/batch/mappings/CanalEntryMappingFormat.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.batch.mappings 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.key.PartitionStrategy 4 | import com.neighborhood.aka.laplace.estuary.core.trans.MappingFormat 5 | import com.neighborhood.aka.laplace.estuary.mysql.lifecycle.EntryKeyClassifier 6 | import com.typesafe.config.Config 7 | import org.slf4j.LoggerFactory 8 | 9 | /** 10 | * Created by john_liu on 2019/1/10. 11 | */ 12 | trait CanalEntryMappingFormat[R] extends MappingFormat[EntryKeyClassifier, R] { 13 | 14 | 15 | protected lazy val logger = LoggerFactory.getLogger(classOf[CanalEntryMappingFormat[R]]) 16 | 17 | 18 | /** 19 | * 分区策略 20 | * 21 | * @return 22 | */ 23 | def partitionStrategy: PartitionStrategy 24 | 25 | /** 26 | * 同步任务Id 27 | */ 28 | def syncTaskId: String 29 | 30 | /** 31 | * 任务开始时间 32 | */ 33 | def syncStartTime: Long 34 | 35 | /** 36 | * 是否开启Schema管理 37 | * 38 | * @return 39 | */ 40 | def schemaComponentIsOn: Boolean 41 | 42 | /** 43 | * config 44 | */ 45 | def config: Config 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/BigIntColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | import java.math.BigInteger; 4 | 5 | public class BigIntColumnDef extends ColumnDef { 6 | static private final BigInteger longlong_max = BigInteger.ONE.shiftLeft(64); 7 | 8 | protected boolean signed; 9 | 10 | public BigIntColumnDef(String name, String type, int pos, boolean signed) { 11 | super(name, type, pos); 12 | this.signed = signed; 13 | } 14 | 15 | private Object toNumeric(Object value) { 16 | if ( value instanceof BigInteger ) { 17 | return value; 18 | } 19 | Long l = (Long)value; 20 | if ( l < 0 && !signed ) 21 | return longlong_max.add(BigInteger.valueOf(l)); 22 | else 23 | return Long.valueOf(l); 24 | } 25 | @Override 26 | public String toSQL(Object value) { 27 | return toNumeric(value).toString(); 28 | } 29 | 30 | @Override 31 | public Object asJSON(Object value) { 32 | return toNumeric(value); 33 | } 34 | 35 | public boolean isSigned() { 36 | return signed; 37 | } 38 | 39 | public void setSigned(boolean signed) { 40 | this.signed = signed; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/bean/SnapshotRequestBean.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.bean; 2 | 3 | /** 4 | * Created by john_liu on 2018/7/26. 5 | */ 6 | 7 | public class SnapshotRequestBean extends TaskRequestBean { 8 | private String syncTaskId; 9 | private String snapshotTaskId; 10 | private Long targetTimestamp; 11 | private int kafkaPartition; 12 | 13 | public String getSyncTaskId() { 14 | return syncTaskId; 15 | } 16 | 17 | public void setSyncTaskId(String syncTaskId) { 18 | this.syncTaskId = syncTaskId; 19 | } 20 | 21 | public String getSnapshotTaskId() { 22 | return snapshotTaskId; 23 | } 24 | 25 | public void setSnapshotTaskId(String snapshotTaskId) { 26 | this.snapshotTaskId = snapshotTaskId; 27 | } 28 | 29 | public Long getTargetTimestamp() { 30 | return targetTimestamp; 31 | } 32 | 33 | public void setTargetTimestamp(Long targetTimestamp) { 34 | this.targetTimestamp = targetTimestamp; 35 | } 36 | 37 | public int getKafkaPartition() { 38 | return kafkaPartition; 39 | } 40 | 41 | public void setKafkaPartition(int kafkaPartition) { 42 | this.kafkaPartition = kafkaPartition; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/bean/MysqlSinkBean.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.bean; 2 | 3 | 4 | 5 | /** 6 | * Created by john_liu on 2019/1/17. 7 | */ 8 | public class MysqlSinkBean { 9 | 10 | private MysqlCredentialRequestBean credential; 11 | private boolean isAutoCommit; 12 | private long connectionTimeout; 13 | private int maximumPoolSize; 14 | 15 | public MysqlCredentialRequestBean getCredential() { 16 | return credential; 17 | } 18 | 19 | public void setCredential(MysqlCredentialRequestBean credential) { 20 | this.credential = credential; 21 | } 22 | 23 | public boolean isAutoCommit() { 24 | return isAutoCommit; 25 | } 26 | 27 | public void setAutoCommit(boolean autoCommit) { 28 | isAutoCommit = autoCommit; 29 | } 30 | 31 | public long getConnectionTimeout() { 32 | return connectionTimeout; 33 | } 34 | 35 | public void setConnectionTimeout(long connectionTimeout) { 36 | this.connectionTimeout = connectionTimeout; 37 | } 38 | 39 | public int getMaximumPoolSize() { 40 | return maximumPoolSize; 41 | } 42 | 43 | public void setMaximumPoolSize(int maximumPoolSize) { 44 | this.maximumPoolSize = maximumPoolSize; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/fetch/MysqlBinlogInOrderFetcherCommand.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.fetch 2 | 3 | /** 4 | * Created by john_liu on 2018/10/10. 5 | */ 6 | sealed trait MysqlBinlogInOrderFetcherCommand 7 | 8 | object MysqlBinlogInOrderFetcherCommand { 9 | 10 | case object MysqlBinlogInOrderFetcherStart extends MysqlBinlogInOrderFetcherCommand 11 | 12 | case object MysqlBinlogInOrderFetcherRestart extends MysqlBinlogInOrderFetcherCommand 13 | 14 | case object MysqlBinlogInOrderFetcherSuspend extends MysqlBinlogInOrderFetcherCommand 15 | 16 | case object MysqlBinlogInOrderFetcherBusy extends MysqlBinlogInOrderFetcherCommand 17 | 18 | case object MysqlBinlogInOrderFetcherFree extends MysqlBinlogInOrderFetcherCommand 19 | 20 | case object MysqlBinlogInOrderResume extends MysqlBinlogInOrderFetcherCommand 21 | 22 | case object MysqlBinlogInOrderFetcherPrefetch extends MysqlBinlogInOrderFetcherCommand 23 | 24 | case object MysqlBinlogInOrderFetcherFetch extends MysqlBinlogInOrderFetcherCommand 25 | 26 | case object MysqlBinlogInOrderFetcherNonBlockingFetch extends MysqlBinlogInOrderFetcherCommand 27 | 28 | case class MysqlBinlogInOrderFetcherUpdateDelay(delay: Long) extends MysqlBinlogInOrderFetcherCommand 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/util/message/MessageSender.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.util.message 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.util.spring.SpringHttpRequestUtil 4 | 5 | import scala.reflect.ClassTag 6 | import scala.util.Try 7 | 8 | /** 9 | * Created by john_liu on 2018/7/25. 10 | */ 11 | 12 | object MessageSender { 13 | /** 14 | * 15 | * @param contents 发送内容 16 | * @param mobileList 手机号 17 | * @param senderName 发送者 18 | * @param url 调用的url 19 | * @tparam A 返回值类型 20 | * @return 21 | */ 22 | def sendMessage[A:ClassTag](contents: List[String], mobileList: List[String], senderName: String)(url: String): Try[A] = { 23 | import scala.collection.JavaConversions._ 24 | lazy val message = MessageBody.buildMessage(contents, mobileList, senderName) 25 | SpringHttpRequestUtil.doPost[MessageBody, A](url, message) 26 | } 27 | 28 | /** 29 | * 发送短信，返回值String 30 | * @param contents 31 | * @param mobileList 32 | * @param senderName 33 | * @param url 34 | * @return 35 | */ 36 | def sendMessageReturnWithinString(contents: List[String], mobileList: List[String], senderName: String = "estuary")(url: String): Try[String] = sendMessage[String](contents, mobileList, senderName)(url) 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/config/SwaggerConfig.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import springfox.documentation.builders.ApiInfoBuilder; 6 | import springfox.documentation.builders.PathSelectors; 7 | import springfox.documentation.service.ApiInfo; 8 | import springfox.documentation.spi.DocumentationType; 9 | import springfox.documentation.spring.web.plugins.Docket; 10 | import springfox.documentation.swagger2.annotations.EnableSwagger2; 11 | 12 | /** 13 | * Created by john_liu on 2017/11/6. 14 | */ 15 | 16 | @Configuration 17 | @EnableSwagger2 18 | public class SwaggerConfig { 19 | 20 | @Bean 21 | public Docket userApi() { 22 | return new Docket(DocumentationType.SWAGGER_2) 23 | .apiInfo(apiInfo()) 24 | .select() 25 | .paths(PathSelectors.regex("/api/.*")) 26 | .build(); 27 | } 28 | 29 | private ApiInfo apiInfo() { 30 | return new ApiInfoBuilder() 31 | .title("Estuary") 32 | .description("数海数据流式同步系统") 33 | .license("Apache License Version 2.0") 34 | .version("2.0") 35 | .build(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/prototype/SourceDataSpecialBatcherPrototype.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.prototype 2 | 3 | 4 | import akka.actor.ActorRef 5 | import com.neighborhood.aka.laplace.estuary.core.lifecycle.worker.SourceDataBatcher 6 | import com.neighborhood.aka.laplace.estuary.core.task.TaskManager 7 | 8 | /** 9 | * Created by john_liu on 2019/1/9. 10 | */ 11 | trait SourceDataSpecialBatcherPrototype extends ActorPrototype with SourceDataBatcher { 12 | /** 13 | * 事件收集器 14 | */ 15 | def eventCollector: Option[ActorRef] 16 | 17 | /** 18 | * sinker 的ActorRef 19 | */ 20 | def sinker: ActorRef 21 | 22 | /** 23 | * 任务信息管理器 24 | */ 25 | def taskManager: TaskManager 26 | 27 | 28 | override def preStart(): Unit = { 29 | log.info(s"init special batcher,id:$syncTaskId") 30 | } 31 | 32 | override def postStop(): Unit = { 33 | 34 | } 35 | 36 | override def preRestart(reason: Throwable, message: Option[Any]): Unit = { 37 | log.info(s"special batcher process preRestart,id:$syncTaskId") 38 | super.preRestart(reason, message) 39 | } 40 | 41 | override def postRestart(reason: Throwable): Unit = { 42 | log.info(s"special batcher process postRestart,id:$syncTaskId") 43 | super.postRestart(reason) 44 | } 45 | } 46 | 47 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/PreviousGtidSetEventData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Juan Olivares 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event; 17 | 18 | /** 19 | * @author Juan Olivares 20 | */ 21 | public class PreviousGtidSetEventData implements EventData { 22 | 23 | private final String gtidSet; 24 | 25 | public PreviousGtidSetEventData(String gtidSet) { 26 | this.gtidSet = gtidSet; 27 | } 28 | 29 | public String getGtidSet() { 30 | return gtidSet; 31 | } 32 | 33 | @Override 34 | public String toString() { 35 | return "PreviousGtidSetEventData {gtidSet='" + gtidSet + "'}"; 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/EventDataDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import com.neighborhood.aka.laplace.estuary.mysql.schema.io.ByteArrayInputStream; 19 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.EventData; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * @param event data this deserializer is responsible for 25 | * @author Stanley Shyiko 26 | */ 27 | public interface EventDataDeserializer { 28 | 29 | T deserialize(ByteArrayInputStream inputStream) throws IOException; 30 | } 31 | -------------------------------------------------------------------------------- /src/main/resources/application.conf.templete: -------------------------------------------------------------------------------- 1 | akka { 2 | jvm-exit-on-fatal-error = true 3 | loggers = [ 4 | "akka.event.slf4j.Slf4jLogger" 5 | , "com.neighborhood.aka.laplace.estuary.core.akkaUtil.EstuaryEventListener" 6 | ] 7 | 8 | log-dead-letters = 10 9 | log-dead-letters-during-shutdown = on 10 | 11 | loglevel = "DEBUG" 12 | // logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" 13 | stdout-loglevel = "OFF" 14 | 15 | pinned-dispatcher { 16 | type = PinnedDispatcher 17 | } 18 | fetcher-dispatcher { 19 | type = Dispatcher 20 | executor = "fork-join-executor" 21 | fork-join-executor { 22 | parallelism-min = 1 23 | parallelism-factor = 1 24 | parallelism-max = 2 25 | } 26 | throughput = 50 27 | } 28 | batcher-dispatcher { 29 | type = Dispatcher 30 | executor = "fork-join-executor" 31 | fork-join-executor { 32 | parallelism-min = 1 33 | parallelism-factor = 8 34 | parallelism-max = 200 35 | } 36 | throughput = 50 37 | } 38 | sinker-dispatcher { 39 | type = Dispatcher 40 | executor = "fork-join-executor" 41 | fork-join-executor { 42 | parallelism-min = 5 43 | parallelism-factor = 30 44 | parallelism-max = 200 45 | } 46 | throughput = 1 47 | } 48 | } 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/task/Mysql2MysqlTaskInfoBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.task 2 | 3 | import java.util.Date 4 | 5 | import com.neighborhood.aka.laplace.estuary.bean.identity.BaseExtractBean 6 | import com.neighborhood.aka.laplace.estuary.mysql.sink.MysqlSinkBeanImp 7 | import com.neighborhood.aka.laplace.estuary.mysql.source.MysqlSourceBeanImp 8 | 9 | /** 10 | * Created by john_liu on 2019/1/15. 11 | * 12 | * @author neighborhood.aka.laplace 13 | */ 14 | final case class Mysql2MysqlTaskInfoBean( 15 | val sourceBean: MysqlSourceBeanImp, 16 | val sinkBean: MysqlSinkBeanImp, 17 | val taskRunningInfoBean: MysqlTaskInfoBeanImp, 18 | val sdaBean:Option[SdaBean] =None 19 | ) extends BaseExtractBean { 20 | 21 | 22 | /** 23 | * 同步任务的唯一id, 这个id表示同步任务的唯一标识 24 | */ 25 | override val syncTaskId: String = taskRunningInfoBean.syncTaskId 26 | 27 | /** 28 | * 数据同步形式 29 | */ 30 | override def dataSyncType: String = taskRunningInfoBean.dataSyncType 31 | 32 | override protected val createTime: Date = new Date() 33 | 34 | override protected val lastChange: Date = createTime 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/EventHeaderDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import com.neighborhood.aka.laplace.estuary.mysql.schema.io.ByteArrayInputStream; 19 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.EventHeader; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * @param event header this deserializer is responsible for 25 | * @author Stanley Shyiko 26 | */ 27 | public interface EventHeaderDeserializer { 28 | 29 | T deserialize(ByteArrayInputStream inputStream) throws IOException; 30 | } 31 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/akkaUtil/DivideDDLRoundRobinRoutingLogic.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.akkaUtil 2 | 3 | import akka.routing.{RoundRobinRoutingLogic, Routee, RoutingLogic} 4 | import com.alibaba.otter.canal.protocol.CanalEntry 5 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType 6 | 7 | import scala.collection.immutable 8 | 9 | /** 10 | * Created by john_liu on 2018/4/3. 11 | * 12 | * @note https://doc.akka.io/docs/akka/current/routing.html 13 | * 参考了Akka官方的Router的自定义 14 | */ 15 | class DivideDDLRoundRobinRoutingLogic extends RoutingLogic { 16 | val roundRobin = RoundRobinRoutingLogic() 17 | 18 | override def select(message: Any, routees: immutable.IndexedSeq[Routee]): Routee = { 19 | val ddlBatcher = routees.head 20 | routees(0) 21 | message match { 22 | case entry: CanalEntry.Entry if (isConcernedDDL(entry.getHeader.getEventType)) => 23 | ddlBatcher 24 | case _ => roundRobin.select(message, routees.drop(0)) 25 | } 26 | 27 | } 28 | 29 | /** 30 | * @param eventType binlog事件类型 31 | * 目前仅处理Alter一种DDL 32 | */ 33 | def isConcernedDDL(eventType: CanalEntry.EventType): Boolean = { 34 | eventType match { 35 | case EventType.ALTER => 36 | true 37 | case _ => false 38 | } 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/NullEventDataDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import com.neighborhood.aka.laplace.estuary.mysql.schema.io.ByteArrayInputStream; 19 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.EventData; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * @author Stanley Shyiko 25 | */ 26 | public class NullEventDataDeserializer implements EventDataDeserializer { 27 | 28 | @Override 29 | public EventData deserialize(ByteArrayInputStream inputStream) throws IOException { 30 | return null; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/key/MultipleJsonKeyPartitionerJava.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.key; 2 | 3 | import org.apache.kafka.clients.producer.Partitioner; 4 | import org.apache.kafka.common.Cluster; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * Created by john_liu on 2018/5/10. 10 | */ 11 | public class MultipleJsonKeyPartitionerJava implements Partitioner { 12 | private int partitionByPrimaryKey(Object key, int partitions) { 13 | return key.hashCode() % partitions; 14 | } 15 | 16 | private int partitionByMod(long mod, int partitions) { 17 | return (int) (mod % partitions); 18 | } 19 | 20 | private int partitionByDbAndTable(String db, String tb, int partitions) { 21 | return (db + "-" + tb).hashCode() % partitions; 22 | } 23 | 24 | @Override 25 | 26 | public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { 27 | //todo 28 | BinlogKey theKey = (BinlogKey) key; 29 | int partitions = cluster.partitionCountForTopic(topic); 30 | return partitionByPrimaryKey(theKey.getPrimaryKeyValue(), partitions); 31 | } 32 | 33 | @Override 34 | public void close() { 35 | 36 | } 37 | 38 | @Override 39 | public void configure(Map configs) { 40 | 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/SdaSchemaMappingRule.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema 2 | 3 | /** 4 | * Created by john_liu on 2019/1/25. 5 | * 6 | * sda tableMappingRule 7 | * 8 | * @author neighborhood.aka.laplace 9 | */ 10 | final class SdaSchemaMappingRule( 11 | private val tableMappingRule: Map[String, String] 12 | ) { 13 | private lazy val databaseMappingRule: Map[String, String] = tableMappingRule.map { 14 | case (k, v) => (k.split('.')(0) -> v.split('.')(0)) 15 | } 16 | 17 | /** 18 | * 获取mapping name 19 | * 20 | * warning:如果通过db.tb获取不到的话。返回的mapping是`(sdaDb->tb)` 21 | * 22 | * @param dbName 数据库名称 23 | * @param tableName 表名称 24 | * @return (db,tb) 25 | */ 26 | def getMappingName(dbName: String, tableName: String): (String, String) = { 27 | tableMappingRule 28 | .get(s"$dbName.$tableName") 29 | .map { kv => (kv.split('.')(0), kv.split('.')(1)) } //todo 可以优化 30 | .getOrElse((getDatabaseMappingName(dbName).get, tableName)) //找不到情况是使用(sdaDb->tb) 如果找不到我们期望扔出异常 31 | } 32 | 33 | /** 34 | * 匹配原库->sda库对应明后才能 35 | * 36 | * @param dbName 数据库名称 37 | * @return sda库对应名称 38 | */ 39 | def getDatabaseMappingName(dbName: String): Option[String] = databaseMappingRule.get(dbName) 40 | } 41 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/ChecksumType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | /** 19 | * @see 20 | * 21 | * MySQL --binlog-checksum option 22 | * 23 | * @author Stanley Shyiko 24 | */ 25 | public enum ChecksumType { 26 | 27 | NONE(0), CRC32(4); 28 | 29 | private int length; 30 | 31 | private ChecksumType(int length) { 32 | this.length = length; 33 | } 34 | 35 | public int getLength() { 36 | return length; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/XidEventData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event; 17 | 18 | /** 19 | * @author Stanley Shyiko 20 | */ 21 | public class XidEventData implements EventData { 22 | 23 | private long xid; 24 | 25 | public long getXid() { 26 | return xid; 27 | } 28 | 29 | public void setXid(long xid) { 30 | this.xid = xid; 31 | } 32 | 33 | @Override 34 | public String toString() { 35 | final StringBuilder sb = new StringBuilder(); 36 | sb.append("XidEventData"); 37 | sb.append("{xid=").append(xid); 38 | sb.append('}'); 39 | return sb.toString(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/credential/MysqlCredentialBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.credential 2 | 3 | import scala.beans.BeanProperty 4 | 5 | /** 6 | * Created by john_liu on 2018/2/7. 7 | */ 8 | @BeanProperty 9 | final case class MysqlCredentialBean(/** 10 | * 服务器地址 11 | */ 12 | val address: String, 13 | 14 | /** 15 | * 服务器端口号 16 | */ 17 | val port: Int, 18 | 19 | /** 20 | * 服务器用户名 21 | */ 22 | val username: String, 23 | 24 | /** 25 | * 服务器密码 26 | */ 27 | val password: String, 28 | 29 | /** 30 | * 服务器密码 31 | */ 32 | val defaultDatabase: Option[String] = None 33 | ) extends DataSourceCredentialBean { 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/GeometryColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | import com.vividsolutions.jts.geom.Geometry; 4 | import com.vividsolutions.jts.io.ParseException; 5 | import com.vividsolutions.jts.io.WKBReader; 6 | 7 | import java.util.Arrays; 8 | 9 | /** 10 | * Created by ben on 12/30/15. 11 | */ 12 | public class GeometryColumnDef extends ColumnDef { 13 | public GeometryColumnDef(String name, String type, int pos) { 14 | super(name, type, pos); 15 | } 16 | 17 | @Override 18 | public Object asJSON(Object value) { 19 | Geometry geometry = null; 20 | if ( value instanceof Geometry ) { 21 | geometry = (Geometry) value; 22 | } else if ( value instanceof byte[] ) { 23 | byte []bytes = (byte[]) value; 24 | 25 | // mysql sprinkles 4 mystery bytes on top of the GIS data. 26 | bytes = Arrays.copyOfRange(bytes, 4, bytes.length); 27 | final WKBReader reader = new WKBReader(); 28 | 29 | try { 30 | geometry = reader.read(bytes); 31 | } catch ( ParseException e ) { 32 | throw new RuntimeException("Could not parse geometry: " + e); 33 | } 34 | 35 | } else { 36 | throw new RuntimeException("Could not parse geometry column value: " + value); 37 | } 38 | 39 | return geometry.toText(); 40 | } 41 | 42 | @Override 43 | public String toSQL(Object value) { 44 | return null; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/bean/TableNameMappingBean.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.bean; 2 | 3 | /** 4 | * Created by john_liu on 2019/1/22. 5 | */ 6 | public class TableNameMappingBean { 7 | 8 | private String sourceType; // 来源类型 9 | private String sourceTable; // 老表名 10 | private String newDb; // 新库名 11 | private String newTable; // 新表名 12 | private String sourceDb; 13 | 14 | public String getSourceType() { 15 | return sourceType; 16 | } 17 | 18 | public void setSourceType(String sourceType) { 19 | this.sourceType = sourceType; 20 | } 21 | 22 | public String getSourceTable() { 23 | return sourceTable; 24 | } 25 | 26 | public void setSourceTable(String sourceTable) { 27 | this.sourceTable = sourceTable; 28 | } 29 | 30 | public String getNewDb() { 31 | return newDb; 32 | } 33 | 34 | public void setNewDb(String newDb) { 35 | this.newDb = newDb; 36 | } 37 | 38 | public String getNewTable() { 39 | return newTable; 40 | } 41 | 42 | public void setNewTable(String newTable) { 43 | this.newTable = newTable; 44 | } 45 | 46 | public String getSourceDb() { 47 | return sourceDb; 48 | } 49 | 50 | public void setSourceDb(String sourceDb) { 51 | this.sourceDb = sourceDb; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/prototype/SourceDataBatcherPrototype.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.prototype 2 | 3 | import akka.actor.ActorRef 4 | import com.neighborhood.aka.laplace.estuary.core.lifecycle.worker.SourceDataBatcher 5 | import com.neighborhood.aka.laplace.estuary.core.trans.MappingFormat 6 | 7 | /** 8 | * Created by john_liu on 2018/5/20. 9 | */ 10 | trait SourceDataBatcherPrototype[A, B] extends ActorPrototype with SourceDataBatcher { 11 | 12 | 13 | def mappingFormat: MappingFormat[A, B] 14 | 15 | /** 16 | * sinker 的ActorRef 17 | */ 18 | def sinker: ActorRef 19 | 20 | /** 21 | * 编号 22 | */ 23 | def num: Int 24 | 25 | /** 26 | * 核心转换方法 27 | * 28 | * @param a 待转换的 29 | * @return 转换结果 30 | */ 31 | def transform(a: A): B = mappingFormat.transform(a) // todo 这么干有问题 32 | 33 | override def preStart(): Unit = { 34 | log.info(s"init batcher$num,id:$syncTaskId") 35 | } 36 | 37 | override def postStop(): Unit = { 38 | 39 | } 40 | 41 | override def preRestart(reason: Throwable, message: Option[Any]): Unit = { 42 | log.info(s"batcher$num process preRestart,id:$syncTaskId") 43 | super.preRestart(reason, message) 44 | } 45 | 46 | override def postRestart(reason: Throwable): Unit = { 47 | log.info(s"batcher$num process postRestart,id:$syncTaskId") 48 | super.postRestart(reason) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/RowsQueryEventData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Patrick Prasse 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event; 17 | 18 | /** 19 | * @author Patrick Prasse 20 | */ 21 | public class RowsQueryEventData implements EventData { 22 | 23 | private String query; 24 | 25 | public String getQuery() { 26 | return query; 27 | } 28 | 29 | public void setQuery(String query) { 30 | this.query = query; 31 | } 32 | 33 | public String toString() { 34 | final StringBuilder sb = new StringBuilder(); 35 | sb.append("RowsQueryEventData"); 36 | sb.append("{query='").append(query).append('\''); 37 | sb.append('}'); 38 | return sb.toString(); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/ByteArrayEventData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event; 17 | 18 | /** 19 | * @author Stanley Shyiko 20 | */ 21 | public class ByteArrayEventData implements EventData { 22 | 23 | private byte[] data; 24 | 25 | public byte[] getData() { 26 | return data; 27 | } 28 | 29 | public void setData(byte[] data) { 30 | this.data = data; 31 | } 32 | 33 | @Override 34 | public String toString() { 35 | final StringBuilder sb = new StringBuilder(); 36 | sb.append("ByteArrayEventData"); 37 | sb.append("{dataLength=").append(data.length); 38 | sb.append('}'); 39 | return sb.toString(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/source/MysqlSourceBeanImp.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.source 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.credential.MysqlCredentialBean 4 | import com.neighborhood.aka.laplace.estuary.bean.resource.MysqlSourceBean 5 | 6 | /** 7 | * Created by john_liu on 2019/1/13. 8 | * Mysql数据源构建需要的信息Bean 9 | * 10 | * @author neighborhood.aka.laplace 11 | */ 12 | final case class MysqlSourceBeanImp( 13 | override val master: MysqlCredentialBean, 14 | override val concernedDatabase: List[String], 15 | override val ignoredDatabase: List[String], 16 | override val filterPattern: Option[String] = None, 17 | override val filterBlackPattern: Option[String] = None 18 | )( 19 | override val filterQueryDcl: Boolean = false, 20 | override val filterQueryDml: Boolean = false, 21 | override val filterQueryDdl: Boolean = false, 22 | override val filterRows: Boolean = false, 23 | override val filterTableError: Boolean = true //默认开启 24 | ) extends MysqlSourceBean 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/fetch/DefaultMysqlBinlogInOrderDirectFetcher.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.fetch 2 | 3 | import akka.actor.{ActorRef, Props} 4 | import com.alibaba.otter.canal.protocol.CanalEntry 5 | import com.neighborhood.aka.laplace.estuary.core.task.TaskManager 6 | import com.neighborhood.aka.laplace.estuary.mysql.source.MysqlSourceManagerImp 7 | 8 | /** 9 | * Created by john_liu on 2019/1/12. 10 | * 11 | * @author neighborhood.aka.laplace 12 | */ 13 | final class DefaultMysqlBinlogInOrderDirectFetcher( 14 | override val taskManager: MysqlSourceManagerImp with TaskManager, 15 | override val downStream: ActorRef) extends MysqlBinlogInOrderDirectFetcher(taskManager, downStream) { 16 | override protected def executeDdl(entry: CanalEntry.Entry): Unit = throw new UnsupportedOperationException(s"DefaultMysqlBinlogInOrderDirectFetcher does not support execute ddl,:id:$syncTaskId ") 17 | } 18 | 19 | object DefaultMysqlBinlogInOrderDirectFetcher { 20 | val name: String = DefaultMysqlBinlogInOrderDirectFetcher.getClass.getName.stripSuffix("$") 21 | 22 | def props(taskManager: MysqlSourceManagerImp with TaskManager 23 | , downStream: ActorRef 24 | ): Props = Props(new DefaultMysqlBinlogInOrderDirectFetcher( 25 | taskManager, 26 | downStream 27 | )) 28 | } 29 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/bean/Mysql2MysqlRequestBean.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.bean; 2 | 3 | /** 4 | * Created by john_liu on 2019/1/17. 5 | */ 6 | public class Mysql2MysqlRequestBean extends TaskRequestBean { 7 | 8 | private MysqlSourceBean mysqlSourceBean; 9 | private MysqlSinkBean mysqlSinkBean; 10 | private Mysql2MysqlRunningInfoBean mysql2MysqlRunningInfoBean; 11 | private SdaRequestBean sdaBean; 12 | 13 | public SdaRequestBean getSdaBean() { 14 | return sdaBean; 15 | } 16 | 17 | public void setSdaBean(SdaRequestBean sdaBean) { 18 | this.sdaBean = sdaBean; 19 | } 20 | 21 | public MysqlSourceBean getMysqlSourceBean() { 22 | return mysqlSourceBean; 23 | } 24 | 25 | public void setMysqlSourceBean(MysqlSourceBean mysqlSourceBean) { 26 | this.mysqlSourceBean = mysqlSourceBean; 27 | } 28 | 29 | public MysqlSinkBean getMysqlSinkBean() { 30 | return mysqlSinkBean; 31 | } 32 | 33 | public void setMysqlSinkBean(MysqlSinkBean mysqlSinkBean) { 34 | this.mysqlSinkBean = mysqlSinkBean; 35 | } 36 | 37 | public Mysql2MysqlRunningInfoBean getMysql2MysqlRunningInfoBean() { 38 | return mysql2MysqlRunningInfoBean; 39 | } 40 | 41 | public void setMysql2MysqlRunningInfoBean(Mysql2MysqlRunningInfoBean mysql2MysqlRunningInfoBean) { 42 | this.mysql2MysqlRunningInfoBean = mysql2MysqlRunningInfoBean; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/ddl/Antlr4DdlParserTest.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.ddl 2 | 3 | import com.neighborhood.aka.laplace.estuary.UnitSpec 4 | import com.neighborhood.aka.laplace.estuary.mysql.schema.defs.ddl.{AddColumnMod, TableAlter} 5 | import com.neighborhood.aka.laplace.estuary.mysql.schema.{Parser, SdaSchemaMappingRule} 6 | 7 | /** 8 | * Created by john_liu on 2019/2/14. 9 | */ 10 | class Sda4DdlParserTest extends UnitSpec { 11 | 12 | val mappingRuleMap = Map("a.a" -> "a_map.a_map", "b.b" -> "b_map.b_map") 13 | val schemaMappingRule = new SdaSchemaMappingRule(mappingRuleMap) 14 | val alterTable1 = "ALTER TABLE a.a ADD col1 text DEFAULT 'hello';" 15 | 16 | 17 | "test 1" should "successfully handle Alter table" in { 18 | val schemaChange = Parser.parseAndReplace(alterTable1, "a_map", schemaMappingRule) 19 | assert(schemaChange.isInstanceOf[TableAlter]) 20 | val tableAlter = schemaChange.asInstanceOf[TableAlter] 21 | assert(tableAlter.database == "a_map") 22 | assert(tableAlter.table == "a_map") 23 | assert(tableAlter.newDatabase == "a_map") 24 | assert(tableAlter.newTableName == "a_map") 25 | val addColumnMod = tableAlter.columnMods.get(0).asInstanceOf[AddColumnMod] 26 | assert(addColumnMod.definition.getName == "col1") 27 | assert(addColumnMod.definition.getType == "text") 28 | assert(addColumnMod.definition.getDefaultValue == "'hello'") 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/BitColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | import java.math.BigInteger; 4 | import java.util.BitSet; 5 | 6 | public class BitColumnDef extends ColumnDef { 7 | public BitColumnDef(String name, String type, int pos) { 8 | super(name, type, pos); 9 | } 10 | 11 | @Override 12 | public Object asJSON(Object value) { 13 | byte[] bytes; 14 | if( value instanceof Boolean ){ 15 | bytes = new byte[]{(byte) (( Boolean ) value ? 1 : 0)}; 16 | } else if ( value instanceof BitSet ) { 17 | BitSet bs = (BitSet) value; 18 | bytes = bs.toByteArray(); 19 | } else { 20 | bytes = (byte[]) value; 21 | } 22 | if ( bytes.length == 8 && ((bytes[7] & 0xFF) > 127) ) { 23 | return bytesToBigInteger(bytes); 24 | } else { 25 | return bytesToLong(bytes); 26 | } 27 | } 28 | 29 | private BigInteger bytesToBigInteger(byte[] bytes) { 30 | BigInteger res = BigInteger.ZERO; 31 | 32 | for (int i = 0; i < bytes.length; i++) { 33 | res = res.add(BigInteger.valueOf(bytes[i] & 0xFF).shiftLeft(i * 8)); 34 | } 35 | 36 | return res; 37 | } 38 | 39 | private Long bytesToLong(byte[] bytes) { 40 | long res = 0; 41 | 42 | for (int i = 0; i < bytes.length; i++) 43 | res += ((bytes[i] & 0xFF) << ( i * 8 )); 44 | 45 | return res; 46 | } 47 | 48 | @Override 49 | public String toSQL(Object value) { 50 | return asJSON(value).toString(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/XidEventDataDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.XidEventData; 19 | import com.neighborhood.aka.laplace.estuary.mysql.schema.io.ByteArrayInputStream; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * @author Stanley Shyiko 25 | */ 26 | public class XidEventDataDeserializer implements EventDataDeserializer { 27 | 28 | @Override 29 | public XidEventData deserialize(ByteArrayInputStream inputStream) throws IOException { 30 | XidEventData eventData = new XidEventData(); 31 | eventData.setXid(inputStream.readLong(8)); 32 | return eventData; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/EventDataDeserializationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.EventHeader; 19 | 20 | import java.io.IOException; 21 | 22 | /** 23 | * @author Stanley Shyiko 24 | */ 25 | public class EventDataDeserializationException extends IOException { 26 | 27 | private EventHeader eventHeader; 28 | 29 | public EventDataDeserializationException(EventHeader eventHeader, Throwable cause) { 30 | super("Failed to deserialize data of " + eventHeader, cause); 31 | this.eventHeader = eventHeader; 32 | } 33 | 34 | public EventHeader getEventHeader() { 35 | return eventHeader; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/sink/mysql/MysqlSinkFunc.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.sink.mysql 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.sink.SinkFunc 4 | import com.neighborhood.aka.laplace.estuary.core.source.{MysqlHikariCpConnection, MysqlJdbcConnection} 5 | 6 | import scala.util.Try 7 | 8 | /** 9 | * Created by john_liu on 2019/1/11. 10 | * 11 | * mysql的sink 12 | * 13 | * @author neighborhood.aka.laplace 14 | */ 15 | final class MysqlSinkFunc( 16 | mysqlHikariCpConnection: MysqlHikariCpConnection 17 | ) extends SinkFunc { 18 | /** 19 | * 生命周期 20 | * 关闭 21 | */ 22 | override def close: Unit = mysqlHikariCpConnection.disconnect() 23 | 24 | /** 25 | * 生命周期 26 | * 开始 27 | */ 28 | override def start: Unit = mysqlHikariCpConnection.connect() 29 | 30 | /** 31 | * 检测，是否关闭 32 | * 33 | * @return 34 | */ 35 | override def isTerminated: Boolean = !mysqlHikariCpConnection.isConnected 36 | 37 | def insertSql(sql: String): Try[Int] = mysqlHikariCpConnection.insertSql(sql) 38 | 39 | def insertBatchSql(sqls: List[String]): Try[List[Int]] = mysqlHikariCpConnection.insertBatchSql(sqls) 40 | 41 | def getJdbcConnection: java.sql.Connection = mysqlHikariCpConnection.getConnection 42 | 43 | def queryAsScalaList(sql: String): List[Map[String, AnyRef]] = { 44 | import MysqlJdbcConnection._ 45 | getJdbcConnection.selectSqlAndClose(sql) 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/offset/ZooKeeperLogPositionManager.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.offset 2 | 3 | import com.alibaba.fastjson.serializer.SerializerFeature 4 | import com.alibaba.otter.canal.common.utils.JsonUtils 5 | import com.alibaba.otter.canal.common.zookeeper.{ZkClientx, ZookeeperPathUtils} 6 | import org.I0Itec.zkclient.exception.ZkNoNodeException 7 | import org.springframework.util.Assert 8 | 9 | import scala.util.Try 10 | 11 | /** 12 | * Created by john_liu on 2018/5/3. 13 | */ 14 | trait ZooKeeperLogPositionManager[T] extends LogPositionManager[T] { 15 | 16 | private var zkClientx: ZkClientx = null 17 | 18 | def start(): Try[Unit] = Try { 19 | 20 | Assert.notNull(zkClientx) 21 | } 22 | 23 | def stop(): Try[Unit] = Try { 24 | 25 | } 26 | 27 | def getLatestIndexBy(destination: String): T 28 | 29 | //todo 30 | def persistLogPosition(destination: String, logPosition: T): Unit = { 31 | 32 | val path = ZookeeperPathUtils.getParsePath(destination) 33 | val data = JsonUtils.marshalToByte(logPosition, SerializerFeature.BeanToArray) 34 | try 35 | zkClientx.writeData(path, data) 36 | catch { 37 | case e: ZkNoNodeException => 38 | zkClientx.createPersistent(path, data, true) 39 | } 40 | } 41 | 42 | // ================== setter / getter ================= 43 | def setZkClientx(zkClientx: ZkClientx): Unit = { 44 | this.zkClientx = zkClientx 45 | } 46 | 47 | def getZkClient = zkClientx 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/sink/MysqlSinkBeanImp.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.sink 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.credential.MysqlCredentialBean 4 | import com.neighborhood.aka.laplace.estuary.bean.datasink.MysqlSinkBean 5 | import com.zaxxer.hikari.HikariConfig 6 | 7 | /** 8 | * Created by john_liu on 2019/1/13. 9 | * Mysql作为数据汇的Bean实现 10 | * 11 | * @author neighborhood.aka.laplace 12 | */ 13 | final case class MysqlSinkBeanImp( 14 | override val credential: MysqlCredentialBean 15 | 16 | )( 17 | val isAutoCommit: Option[Boolean] = None, 18 | val connectionTimeout: Option[Long] = None, 19 | val maximumPoolSize: Option[Int] = None 20 | ) extends MysqlSinkBean { 21 | 22 | 23 | lazy val hikariConfig: HikariConfig = { 24 | val re = new HikariConfig() 25 | re.setJdbcUrl(s"jdbc:mysql://${credential.address}:${credential.port}/?useUnicode=true&characterEncoding=UTF-8") 26 | re.setUsername(credential.username) 27 | re.setPassword(credential.password) 28 | credential.defaultDatabase.flatMap(Option(_)).map(re.setSchema(_)) 29 | isAutoCommit.flatMap(Option(_)).map(re.setAutoCommit(_)) 30 | connectionTimeout.flatMap(Option(_)).map(re.setConnectionTimeout(_)) 31 | maximumPoolSize.flatMap(Option(_)).map(re.setMaximumPoolSize(_)) 32 | re 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/util/SimpleEstuaryRingBuffer.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.util 2 | 3 | import scala.reflect.ClassTag 4 | import scala.reflect.runtime.universe._ 5 | 6 | /** 7 | * Created by john_liu on 2019/1/20. 8 | * 9 | * 简单的ringBuffer实现 10 | * 11 | * 并不线程安全 12 | * 并没有保证读写的原子性 13 | * 14 | * @tparam A 任意元素 15 | * @author neighborhood.aka.laplace 16 | * 17 | */ 18 | final class SimpleEstuaryRingBuffer[A: ClassTag](private val size: Int = 31)(implicit ev: TypeTag[A]) { 19 | private val length = size + 1 20 | private val buffer = new Array[A](length) 21 | @volatile private var head = 0; //读指针 22 | @volatile private var tail = 0; //写指针 23 | 24 | def isEmpty: Boolean = head == tail 25 | 26 | def isFull: Boolean = head == (tail + 1) % size 27 | 28 | def elemNum: Int = { //需要测试 29 | val diff = tail - head 30 | if (diff >= 0) diff 31 | else size + diff 32 | } 33 | 34 | def put(x: A): Boolean = if (isFull) false 35 | else { 36 | buffer(tail) = x 37 | tail = (tail + 1) % size 38 | true 39 | } 40 | 41 | 42 | def get: Option[A] = { 43 | val re = if (isEmpty) None else { 44 | val r = Option(buffer(head)) 45 | buffer(head) = null.asInstanceOf[A] 46 | r 47 | } 48 | head = (head + 1) % size 49 | re 50 | } 51 | 52 | def foreach[B](f: A => B): Unit = while (!isEmpty) { 53 | get.map(f) 54 | } 55 | 56 | def clear = while (!isEmpty) { 57 | get 58 | } 59 | 60 | def peek: A = buffer(head) 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/ByteArrayEventDataDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import com.neighborhood.aka.laplace.estuary.mysql.schema.io.ByteArrayInputStream; 19 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.ByteArrayEventData; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * @author Stanley Shyiko 25 | */ 26 | public class ByteArrayEventDataDeserializer implements EventDataDeserializer { 27 | 28 | @Override 29 | public ByteArrayEventData deserialize(ByteArrayInputStream inputStream) throws IOException { 30 | ByteArrayEventData eventData = new ByteArrayEventData(); 31 | eventData.setData(inputStream.read(inputStream.available())); 32 | return eventData; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/IntVarEventDataDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.IntVarEventData; 19 | import com.neighborhood.aka.laplace.estuary.mysql.schema.io.ByteArrayInputStream; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * @author Stanley Shyiko 25 | */ 26 | public class IntVarEventDataDeserializer implements EventDataDeserializer { 27 | 28 | @Override 29 | public IntVarEventData deserialize(ByteArrayInputStream inputStream) throws IOException { 30 | IntVarEventData event = new IntVarEventData(); 31 | event.setType(inputStream.readInteger(1)); 32 | event.setValue(inputStream.readLong(8)); 33 | return event; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/bean/MysqlCredentialRequestBean.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.bean; 2 | 3 | /** 4 | * Created by john_liu on 2019/1/18. 5 | */ 6 | public class MysqlCredentialRequestBean { 7 | 8 | 9 | private String address; 10 | 11 | /** 12 | * 服务器端口号 13 | */ 14 | private int port; 15 | 16 | /** 17 | * 服务器用户名 18 | */ 19 | private String username; 20 | 21 | /** 22 | * 服务器密码 23 | */ 24 | private String password; 25 | 26 | /** 27 | * 服务器密码 28 | */ 29 | private String defaultDatabase; 30 | 31 | public String getAddress() { 32 | return address; 33 | } 34 | 35 | public void setAddress(String address) { 36 | this.address = address; 37 | } 38 | 39 | public int getPort() { 40 | return port; 41 | } 42 | 43 | public void setPort(int port) { 44 | this.port = port; 45 | } 46 | 47 | public String getUsername() { 48 | return username; 49 | } 50 | 51 | public void setUsername(String username) { 52 | this.username = username; 53 | } 54 | 55 | public String getPassword() { 56 | return password; 57 | } 58 | 59 | public void setPassword(String password) { 60 | this.password = password; 61 | } 62 | 63 | public String getDefaultDatabase() { 64 | return defaultDatabase; 65 | } 66 | 67 | public void setDefaultDatabase(String defaultDatabase) { 68 | this.defaultDatabase = defaultDatabase; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/RowsQueryEventDataDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Patrick Prasse 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.RowsQueryEventData; 19 | import com.neighborhood.aka.laplace.estuary.mysql.schema.io.ByteArrayInputStream; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * @author Patrick Prasse 25 | */ 26 | public class RowsQueryEventDataDeserializer implements EventDataDeserializer { 27 | 28 | @Override 29 | public RowsQueryEventData deserialize(ByteArrayInputStream inputStream) throws IOException { 30 | RowsQueryEventData eventData = new RowsQueryEventData(); 31 | inputStream.readInteger(1); // ignored 32 | eventData.setQuery(inputStream.readString(inputStream.available())); 33 | return eventData; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/RotateEventDataDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.RotateEventData; 19 | import com.neighborhood.aka.laplace.estuary.mysql.schema.io.ByteArrayInputStream; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * @author Stanley Shyiko 25 | */ 26 | public class RotateEventDataDeserializer implements EventDataDeserializer { 27 | 28 | @Override 29 | public RotateEventData deserialize(ByteArrayInputStream inputStream) throws IOException { 30 | RotateEventData eventData = new RotateEventData(); 31 | eventData.setBinlogPosition(inputStream.readLong(8)); 32 | eventData.setBinlogFilename(inputStream.readString(inputStream.available())); 33 | return eventData; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/tablemeta/package.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry 4 | import com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef.ColumnDef 5 | 6 | /** 7 | * Created by john_liu on 2019/1/23. 8 | */ 9 | package object tablemeta { 10 | 11 | sealed trait ColumnInfo 12 | 13 | /** 14 | * 入海口的MysqlColumnInfo定义 15 | * 16 | * @param name 列名称 17 | * @param index 索引 18 | * @param mysqlType mysqlType 19 | */ 20 | final case class EstuaryMysqlColumnInfo(name: String, index: Int, mysqlType: String) extends ColumnInfo 21 | 22 | /** 23 | * 入海口的Mysql表定义 24 | * 25 | * @param schemaName 库名称 26 | * @param tableName 表名称 27 | * @param columns 列 28 | */ 29 | final case class EstuaryMysqlTableMeta(schemaName: String, tableName: String, columns: List[EstuaryMysqlColumnInfo]) { 30 | val columnNum = columns.size 31 | val columnInfoMap = columns.map(x => (x.name -> x)).toMap 32 | } 33 | 34 | 35 | implicit final class ColumnDefEstuaryMysqlColumnInfoSyntax(column: ColumnDef) { 36 | def toEstuaryMysqlColumnInfo: EstuaryMysqlColumnInfo = { 37 | EstuaryMysqlColumnInfo(column.getName, column.getPos, column.getType) 38 | } 39 | } 40 | 41 | implicit final class CanalColumnEstuaryMysqlColumnInfoSyntax(column: CanalEntry.Column) { 42 | def toEstuaryMysqlColumnInfo: EstuaryMysqlColumnInfo = { 43 | EstuaryMysqlColumnInfo(name = column.getName, index = column.getIndex, mysqlType = column.getMysqlType) 44 | } 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/batch/imp/MysqlBinlogInOrderMysqlBatcher.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.batch.imp 2 | 3 | import akka.actor.{ActorRef, Props} 4 | import com.neighborhood.aka.laplace.estuary.core.task.TaskManager 5 | import com.neighborhood.aka.laplace.estuary.core.trans.MappingFormat 6 | import com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.batch.MysqlBinlogInOrderBatcher 7 | import com.neighborhood.aka.laplace.estuary.mysql.lifecycle.{EntryKeyClassifier, MysqlRowDataInfo} 8 | import com.neighborhood.aka.laplace.estuary.mysql.source.MysqlSourceManagerImp 9 | 10 | /** 11 | * Created by john_liu on 2019/1/15. 12 | * 13 | * @author neighborhood.aka.laplace 14 | */ 15 | final class MysqlBinlogInOrderMysqlBatcher(override val taskManager: MysqlSourceManagerImp with TaskManager, 16 | override val sinker: ActorRef, 17 | override val num: Int) extends MysqlBinlogInOrderBatcher[MysqlRowDataInfo](taskManager, sinker, num) { 18 | 19 | override val mappingFormat: MappingFormat[EntryKeyClassifier, MysqlRowDataInfo] = taskManager.batchMappingFormat.get.asInstanceOf[MappingFormat[EntryKeyClassifier, MysqlRowDataInfo]] 20 | 21 | } 22 | 23 | object MysqlBinlogInOrderMysqlBatcher { 24 | val name: String = MysqlBinlogInOrderMysqlBatcher.getClass.getName.stripSuffix("$") 25 | 26 | def props(taskManager: MysqlSourceManagerImp with TaskManager, 27 | sinker: ActorRef, 28 | num: Int): Props = Props(new MysqlBinlogInOrderMysqlBatcher(taskManager, sinker, num)) 29 | } 30 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/resource/MongoSourceBean.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.resource 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.credential.MongoCredentialBean 4 | 5 | /** 6 | * Created by john_liu on 2018/4/25. 7 | * 8 | * @todo 有问题 9 | */ 10 | trait MongoSourceBean extends DataSourceBase[MysqlSourceBean] { 11 | override val dataSourceType = SourceDataType.MONGO.toString 12 | val MONGODB_CR = "MONGODB-CR" 13 | val SCRAM_SHA_1 = "SCRAM-SHA-1"; 14 | 15 | val mongoCredentials: Option[List[MongoCredentialBean]] 16 | val hosts: List[String] 17 | val port: Int 18 | val concernedNs: Array[String] = Array.empty 19 | val ignoredNs: Array[String] = Array.empty 20 | /** 21 | * 读取数据时, 对于有replication set 复本集的collection是使用什么策略 22 | * primary, 23 | * primaryPreferred, 24 | * secondary, 25 | * secondaryPreferred, 26 | * nearest 27 | */ 28 | val readPreference = "secondaryPreferred" 29 | /** 30 | * 写数据时的设置, 对于有replication set 复本集的collection是使用什么策略 31 | * majority, 32 | * normal, 33 | * journaled, 34 | * acknowledged, 35 | * replica_acknowledged, 36 | * journal_safe, 37 | * fsynced, 38 | * unacknowledged, 39 | * fsync_safe, 40 | * safe, 41 | * replicas_safe, 42 | * w1, 43 | * w2, 44 | * w3 45 | */ 46 | val writeConcern = "majority" 47 | var authMechanism = SCRAM_SHA_1 48 | 49 | override def toString: String = "MongoBean{" + ", hosts=" + hosts.mkString(",") + ", port=" + port + ", authMechanism='" + authMechanism + '\'' + ", readPreference='" + readPreference + '\'' + ", writeConcern='" + writeConcern + '\'' + '}' 50 | } 51 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/defs/columndef/IntColumnDef.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.schema.defs.columndef; 2 | 3 | public class IntColumnDef extends ColumnDef { 4 | public int bits; 5 | 6 | protected boolean signed; 7 | 8 | public IntColumnDef(String name, String type, int pos, boolean signed) { 9 | super(name, type, pos); 10 | this.signed = signed; 11 | this.bits = bitsFromType(type); 12 | } 13 | 14 | 15 | private long castUnsigned(Integer i, long max_value) { 16 | if ( i < 0 ) 17 | return max_value + i; 18 | else 19 | return i; 20 | } 21 | 22 | private Long toLong(Object value) { 23 | 24 | if ( value instanceof Long ) { 25 | return ( Long ) value; 26 | } 27 | 28 | if ( value instanceof Boolean ) { 29 | return ( Boolean ) value ? 1l: 0l; 30 | } 31 | 32 | Integer i = (Integer) value; 33 | 34 | if (signed) 35 | return Long.valueOf(i); 36 | 37 | long res = castUnsigned(i, 1L << this.bits); 38 | return Long.valueOf(res); 39 | 40 | } 41 | @Override 42 | public String toSQL(Object value) { 43 | return toLong(value).toString(); 44 | } 45 | 46 | @Override 47 | public Object asJSON(Object value) { 48 | return toLong(value); 49 | } 50 | 51 | private final static int bitsFromType(String type) { 52 | switch(type) { 53 | case "tinyint": 54 | return 8; 55 | case "smallint": 56 | return 16; 57 | case "mediumint": 58 | return 24; 59 | case "int": 60 | return 32; 61 | default: 62 | return 0; 63 | } 64 | } 65 | 66 | public boolean isSigned() { 67 | return signed; 68 | } 69 | 70 | public void setSigned(boolean signed) { 71 | this.signed = signed; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/GtidEventData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Patrick Prasse 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event; 17 | 18 | /** 19 | * @author Patrick Prasse 20 | */ 21 | public class GtidEventData implements EventData { 22 | 23 | public static final byte COMMIT_FLAG = 1; 24 | 25 | private String gtid; 26 | private byte flags; 27 | 28 | public String getGtid() { 29 | return gtid; 30 | } 31 | 32 | public void setGtid(String gtid) { 33 | this.gtid = gtid; 34 | } 35 | 36 | public byte getFlags() { 37 | return flags; 38 | } 39 | 40 | public void setFlags(byte flags) { 41 | this.flags = flags; 42 | } 43 | 44 | public String toString() { 45 | final StringBuilder sb = new StringBuilder(); 46 | sb.append("GtidEventData"); 47 | sb.append("{flags=").append(flags).append(", gtid='").append(gtid).append('\''); 48 | sb.append('}'); 49 | return sb.toString(); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/prototype/DataSourceFetcherPrototype.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.prototype 2 | 3 | import com.neighborhood.aka.laplace.estuary.core.lifecycle.worker.SourceDataFetcher 4 | import com.neighborhood.aka.laplace.estuary.core.source.DataSourceConnection 5 | import com.neighborhood.aka.laplace.estuary.core.task.SourceManager 6 | 7 | /** 8 | * Created by john_liu on 2018/5/30. 9 | */ 10 | trait DataSourceFetcherPrototype[source <: DataSourceConnection] extends ActorPrototype with SourceDataFetcher { 11 | 12 | /** 13 | * 数据源资源管理器 14 | */ 15 | def sourceManager: SourceManager[source] 16 | 17 | /** 18 | * 数据源链接 19 | */ 20 | lazy val connection = sourceManager.source //取消fork，使得线程可以被杀死 21 | 22 | /* 23 | * 用于在最终的数据汇建立相应的schema信息 24 | */ 25 | def initEventualSinkSchema: Unit = {} 26 | 27 | /** 28 | * ********************* Actor生命周期 ******************* 29 | */ 30 | override def preStart(): Unit = { 31 | log.info(s"fetcher switch to offline,id:$syncTaskId") 32 | if (connection.isConnected) connection.disconnect() 33 | //状态置为offline 34 | } 35 | 36 | override def postRestart(reason: Throwable): Unit = { 37 | log.info(s"fetcher processing postRestart,id:$syncTaskId") 38 | super.postRestart(reason) 39 | 40 | } 41 | 42 | override def postStop(): Unit = { 43 | log.info(s"fetcher processing postStop,id:$syncTaskId") 44 | connection.disconnect() 45 | } 46 | 47 | override def preRestart(reason: Throwable, message: Option[Any]): Unit = { 48 | log.info(s"fetcher processing preRestart,id:$syncTaskId") 49 | context.become(receive) 50 | super.preRestart(reason, message) 51 | } 52 | 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/akkaUtil/EstuaryEventListener.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.akkaUtil 2 | 3 | 4 | import akka.actor.{Actor, ActorLogging, Props} 5 | import akka.event.Logging._ 6 | import com.neighborhood.aka.laplace.estuary.core.util.message.MessageSender 7 | import com.typesafe.config.ConfigFactory 8 | 9 | class EstuaryEventListener extends Actor with ActorLogging { 10 | 11 | val TIME_INTERVAL = 5 * 60 * 1000 //5min 以ms为单位 12 | val config = ConfigFactory.load() 13 | val url = config.getString("error.monitor.url") 14 | val mobilelist = List(config.getString("error.monitor.mobiles")) 15 | val ip = if (config.hasPath("app.server.ip")) config.getString("app.server.ip") else "unknown" 16 | val port = if (config.hasPath("app.server.port")) config.getInt("app.server.port") else -1 17 | var lastSendTime = 0l 18 | 19 | override def receive = { 20 | case InitializeLogger(_) => sender() ! LoggerInitialized 21 | 22 | case Error(cause, logSource, logClass, message) => { 23 | def buildAndSendErrorMessage = { 24 | // 信息内容 25 | lazy val contents = List(s"exception:${cause},cause:${cause.getCause},logSource:$logSource,logClass:$logClass,message $message,host:$ip:$port") 26 | MessageSender.sendMessageReturnWithinString(contents, mobilelist)(url).toOption 27 | .fold(log.error(s"message send failure"))(_ => log.info("message send success")) 28 | } 29 | 30 | lazy val now = System.currentTimeMillis() 31 | if (now - lastSendTime > TIME_INTERVAL) { 32 | buildAndSendErrorMessage 33 | lastSendTime = now 34 | } 35 | 36 | } 37 | } 38 | } 39 | 40 | object EstuaryEventListener { 41 | def props: Props = Props(new EstuaryEventListener) 42 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/lifecycle/prototype/SourceDataBatcherManagerPrototype.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.lifecycle.prototype 2 | 3 | import akka.actor.ActorRef 4 | import com.neighborhood.aka.laplace.estuary.core.lifecycle.worker.SourceDataBatcher 5 | import com.neighborhood.aka.laplace.estuary.core.sink.SinkFunc 6 | import com.neighborhood.aka.laplace.estuary.core.source.DataSourceConnection 7 | import com.neighborhood.aka.laplace.estuary.core.task.{SinkManager, SourceManager, TaskManager} 8 | 9 | /** 10 | * Created by john_liu on 2018/5/20. 11 | * 12 | * @tparam A Source 13 | * @tparam B Sink 14 | * @author neighborhood.aka.laplace 15 | */ 16 | trait SourceDataBatcherManagerPrototype[A <: DataSourceConnection, B <: SinkFunc] extends ActorPrototype with SourceDataBatcher { 17 | /** 18 | * specialInfoSender的名称 19 | */ 20 | val specialInfoSenderName = "specialInfoSender" 21 | /** 22 | * router的名称 23 | */ 24 | val routerName: String = "router" 25 | 26 | /** 27 | * 事件收集器 28 | */ 29 | def eventCollector: Option[ActorRef] 30 | 31 | /** 32 | * 是否是最上层的manager 33 | */ 34 | def isHead: Boolean 35 | 36 | /** 37 | * sinker 的ActorRef 38 | */ 39 | def sinker: ActorRef 40 | 41 | /** 42 | * 任务信息管理器 43 | */ 44 | def taskManager: TaskManager 45 | 46 | /** 47 | * 数据源资源管理器 48 | */ 49 | def sourceManager: SourceManager[A] = ??? 50 | 51 | /** 52 | * 数据汇管理器 53 | */ 54 | def sinkManager: SinkManager[B] = ??? 55 | 56 | /** 57 | * 编号 58 | */ 59 | def num: Int 60 | 61 | /** 62 | * 同步任务id 63 | */ 64 | def syncTaskId: String 65 | 66 | /** 67 | * 初始化Batchers 68 | */ 69 | protected def initBatchers: Unit 70 | } 71 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/Event.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event; 17 | 18 | import java.io.Serializable; 19 | 20 | /** 21 | * @author Stanley Shyiko 22 | */ 23 | public class Event implements Serializable { 24 | 25 | private EventHeader header; 26 | private EventData data; 27 | 28 | public Event(EventHeader header, EventData data) { 29 | this.header = header; 30 | this.data = data; 31 | } 32 | 33 | @SuppressWarnings("unchecked") 34 | public T getHeader() { 35 | return (T) header; 36 | } 37 | 38 | @SuppressWarnings("unchecked") 39 | public T getData() { 40 | return (T) data; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | final StringBuilder sb = new StringBuilder(); 46 | sb.append("Event"); 47 | sb.append("{header=").append(header); 48 | sb.append(", data=").append(data); 49 | sb.append('}'); 50 | return sb.toString(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/web/config/ConfigDataSourceConfig.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.web.config; 2 | 3 | import com.zaxxer.hikari.HikariConfig; 4 | import com.zaxxer.hikari.HikariDataSource; 5 | import org.springframework.beans.factory.annotation.Qualifier; 6 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 7 | import org.springframework.context.annotation.*; 8 | import org.springframework.core.env.Environment; 9 | import org.springframework.jdbc.core.JdbcTemplate; 10 | 11 | import javax.annotation.Resource; 12 | import javax.sql.DataSource; 13 | 14 | /** 15 | * Created by meyer on 2017/1/15. 16 | */ 17 | @Configuration 18 | @PropertySource("application.properties") 19 | @EnableConfigurationProperties 20 | public class ConfigDataSourceConfig { 21 | 22 | @Resource 23 | private Environment env; 24 | 25 | 26 | @Primary 27 | @Lazy 28 | @Bean(name = "configDataSource") 29 | public DataSource dataSource1() { 30 | final String url = this.env.getProperty("spring.config.datasource.url"); 31 | final String username = this.env.getProperty("spring.config.datasource.username"); 32 | final String password = this.env.getProperty("spring.config.datasource.password"); 33 | final HikariConfig config = new HikariConfig(); 34 | config.setJdbcUrl(url); 35 | config.setUsername(username); 36 | config.setPassword(password); 37 | config.setMaximumPoolSize(5); 38 | DataSource dds = new HikariDataSource(config); 39 | return dds; 40 | } 41 | 42 | 43 | @Lazy 44 | @Bean(name = "configJdbcTemplate") 45 | public JdbcTemplate JdbcTemplate1(@Qualifier("configDataSource") DataSource dataSource) { 46 | return new JdbcTemplate(dataSource); 47 | } 48 | 49 | 50 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/bean/key/MultipleJsonKeyPartitioner.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.bean.key 2 | 3 | import java.util 4 | 5 | import org.apache.kafka.clients.producer.Partitioner 6 | import org.apache.kafka.common.Cluster 7 | import org.slf4j.LoggerFactory 8 | 9 | /** 10 | * Created by john_liu on 2018/5/6. 11 | * 12 | * @todo 13 | */ 14 | class MultipleJsonKeyPartitioner extends Partitioner { 15 | val logger = LoggerFactory.getLogger(classOf[MultipleJsonKeyPartitioner]) 16 | 17 | private def partitionByPrimaryKey(key: Any)(implicit partitions: Int): Int = { 18 | key.hashCode() % partitions 19 | } 20 | 21 | private def partitionByMod(mod: Long)(implicit partitions: Int): Int = (mod % partitions) toInt 22 | 23 | private def partitionByDbAndTable(db: String, tb: String)(implicit partitions: Int): Int = s"$db-$tb".hashCode % partitions 24 | 25 | override def partition(topic: String, key: Any, keyBytes: Array[Byte], value: Any, valueBytes: Array[Byte], cluster: Cluster): Int = { 26 | implicit val partitions: Int = cluster.partitionCountForTopic(topic) 27 | key match { 28 | case x: BinlogKey => { 29 | x.getPartitionStrategy match { 30 | case PartitionStrategy.MOD => math.abs(partitionByMod(x.getSyncTaskSequence)) 31 | case PartitionStrategy.PRIMARY_KEY => math.abs(partitionByPrimaryKey(x.getPrimaryKeyValue)) 32 | case _ => ??? 33 | } 34 | } 35 | case x: OplogKey => { 36 | x.getPartitionStrategy match { 37 | case PartitionStrategy.PRIMARY_KEY => math.abs(partitionByPrimaryKey(x.getMongoOpsUuid)) 38 | case _ => ??? 39 | } 40 | } 41 | } 42 | } 43 | 44 | override def close(): Unit = {} 45 | 46 | override def configure(configs: util.Map[String, _]): Unit = {} 47 | } 48 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/RotateEventData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event; 17 | 18 | /** 19 | * @author Stanley Shyiko 20 | */ 21 | public class RotateEventData implements EventData { 22 | 23 | private String binlogFilename; 24 | private long binlogPosition; 25 | 26 | public String getBinlogFilename() { 27 | return binlogFilename; 28 | } 29 | 30 | public void setBinlogFilename(String binlogFilename) { 31 | this.binlogFilename = binlogFilename; 32 | } 33 | 34 | public long getBinlogPosition() { 35 | return binlogPosition; 36 | } 37 | 38 | public void setBinlogPosition(long binlogPosition) { 39 | this.binlogPosition = binlogPosition; 40 | } 41 | 42 | @Override 43 | public String toString() { 44 | final StringBuilder sb = new StringBuilder(); 45 | sb.append("RotateEventData"); 46 | sb.append("{binlogFilename='").append(binlogFilename).append('\''); 47 | sb.append(", binlogPosition=").append(binlogPosition); 48 | sb.append('}'); 49 | return sb.toString(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/count/MysqlInOrderProcessingCounter.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.count 2 | 3 | import akka.actor.Props 4 | import com.neighborhood.aka.laplace.estuary.core.lifecycle._ 5 | import com.neighborhood.aka.laplace.estuary.core.lifecycle.prototype.ProcessingCountPrototype 6 | import com.neighborhood.aka.laplace.estuary.core.task.TaskManager 7 | import com.neighborhood.aka.laplace.estuary.mysql.SettingConstant 8 | import com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.count.MysqlBinlogInOrderProcessingCounterCommand._ 9 | 10 | /** 11 | * Created by john_liu on 2018/5/8. 12 | * 计数器，用于计算数量 13 | * 14 | * @author neighborhood.aka.laplace 15 | */ 16 | final class MysqlInOrderProcessingCounter( 17 | override val taskManager: TaskManager 18 | ) extends ProcessingCountPrototype { 19 | override val syncTaskId = taskManager.syncTaskId 20 | 21 | override def receive: Receive = { 22 | case FetcherMessage(MysqlBinlogInOrderProcessingCounterUpdateCount(x: Long)) => addFetchCount(x) 23 | case BatcherMessage(MysqlBinlogInOrderProcessingCounterUpdateCount(x: Long)) => addBatchCount(x) 24 | case SinkerMessage(MysqlBinlogInOrderProcessingCounterUpdateCount(x: Long)) => { 25 | addSinkCount(x) 26 | } 27 | case SyncControllerMessage(MysqlBinlogInOrderProcessingCounterComputeCount) => { 28 | updateRecord(SettingConstant.COMPUTE_COUNT_CONSTANT) 29 | log.debug(s"set fetch count $fetchCount,batch count $batchCount,sink count $sinkCount,id:$syncTaskId") 30 | } 31 | } 32 | 33 | 34 | } 35 | 36 | object MysqlInOrderProcessingCounter { 37 | def props(taskManager: TaskManager): Props = Props(new MysqlInOrderProcessingCounter(taskManager)) 38 | } 39 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/FormatDescriptionEventDataDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.FormatDescriptionEventData; 19 | import com.neighborhood.aka.laplace.estuary.mysql.schema.io.ByteArrayInputStream; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * @author Stanley Shyiko 25 | */ 26 | public class FormatDescriptionEventDataDeserializer implements EventDataDeserializer { 27 | 28 | @Override 29 | public FormatDescriptionEventData deserialize(ByteArrayInputStream inputStream) throws IOException { 30 | FormatDescriptionEventData eventData = new FormatDescriptionEventData(); 31 | eventData.setBinlogVersion(inputStream.readInteger(2)); 32 | eventData.setServerVersion(inputStream.readString(50).trim()); 33 | inputStream.skip(4); // redundant, present in a header 34 | eventData.setHeaderLength(inputStream.readInteger(1)); 35 | // lengths for all event types 36 | return eventData; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/IntVarEventData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event; 17 | 18 | /** 19 | * @author Stanley Shyiko 20 | */ 21 | public class IntVarEventData implements EventData { 22 | 23 | /** 24 | * Type indicating whether the value is meant to be used for the LAST_INSERT_ID() invocation (should be equal 1) or 25 | * AUTO_INCREMENT column (should be equal 2). 26 | */ 27 | private int type; 28 | private long value; 29 | 30 | public int getType() { 31 | return type; 32 | } 33 | 34 | public void setType(int type) { 35 | this.type = type; 36 | } 37 | 38 | public long getValue() { 39 | return value; 40 | } 41 | 42 | public void setValue(long value) { 43 | this.value = value; 44 | } 45 | 46 | @Override 47 | public String toString() { 48 | final StringBuilder sb = new StringBuilder(); 49 | sb.append("IntVarEventData"); 50 | sb.append("{type=").append(type); 51 | sb.append(", value=").append(value); 52 | sb.append('}'); 53 | return sb.toString(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/util/SupportUtil.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.util 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.exception.other.TimeoutException 4 | import com.neighborhood.aka.laplace.estuary.core.task.TaskManager 5 | import org.slf4j.LoggerFactory 6 | 7 | import scala.annotation.tailrec 8 | 9 | /** 10 | * Created by john_liu on 2018/9/3. 11 | */ 12 | object SupportUtil { 13 | private lazy val logger = LoggerFactory.getLogger(SupportUtil.getClass) 14 | 15 | /** 16 | * 判断当前数据是否发送完毕(fetch == sink) 17 | * 18 | * @param taskManager 任务管理器 19 | * @return 20 | */ 21 | def sendCurrentAllDataFinish(taskManager: TaskManager): Boolean = { 22 | lazy val fetchCount = taskManager.fetchCount.get() 23 | lazy val sinkCount = taskManager.batchCount.get() 24 | lazy val sameCount = fetchCount == sinkCount 25 | sameCount 26 | } 27 | 28 | /** 29 | * 循环判断当前数据是否发送完毕 30 | * 31 | * @param taskManager 任务管理器 32 | * @param timeout 超时时间，单位是ms 33 | * @param startTs 开始的时间戳 34 | * @throws TimeoutException 35 | */ 36 | @tailrec 37 | @throws[TimeoutException] 38 | def loopWaiting4SendCurrentAllDataFinish(taskManager: TaskManager, timeout: Option[Long] = None, startTs: Long = System.currentTimeMillis()): Unit = { 39 | lazy val currentTs = System.currentTimeMillis() 40 | lazy val totalCost = currentTs - startTs 41 | lazy val isTimeout = timeout.fold(false)(t => totalCost >= t) 42 | if (isTimeout) { 43 | logger.warn(s"time has been run out when loopWaiting4SendDataFinish,currentTs:$currentTs,timeOut:$timeout,startTs:$startTs") 44 | throw new TimeoutException(s"time has been run out when loopWaiting4SendDataFinish,currentTs:$currentTs,timeOut:$timeout,startTs:$startTs") 45 | } else loopWaiting4SendCurrentAllDataFinish(taskManager, timeout, startTs) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/batch/mappings/DefaultCanalEntry2RowDataInfoMappingFormat.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.batch.mappings 2 | 3 | import com.neighborhood.aka.laplace.estuary.bean.key.PartitionStrategy 4 | import com.neighborhood.aka.laplace.estuary.mysql.lifecycle 5 | import com.neighborhood.aka.laplace.estuary.mysql.lifecycle.MysqlRowDataInfo 6 | import com.neighborhood.aka.laplace.estuary.mysql.schema.tablemeta.MysqlTableSchemaHolder 7 | import com.typesafe.config.Config 8 | 9 | /** 10 | * Created by john_liu on 2019/1/13. 11 | */ 12 | final class DefaultCanalEntry2RowDataInfoMappingFormat( 13 | override val partitionStrategy: PartitionStrategy, 14 | override val syncTaskId: String, 15 | override val syncStartTime: Long, 16 | override val schemaComponentIsOn: Boolean, 17 | override val config: Config, 18 | override val isCheckSchema: Boolean = true, 19 | override val schemaHolder: Option[MysqlTableSchemaHolder] 20 | ) extends CanalEntry2RowDataInfoMappingFormat { 21 | 22 | override def transform(x: lifecycle.EntryKeyClassifier): MysqlRowDataInfo = { 23 | val entry = x.entry 24 | val header = entry.getHeader 25 | val tableName = header.getTableName 26 | val dbName = header.getSchemaName 27 | val dmlType = header.getEventType 28 | val columnList = x.columnList 29 | checkAndGetMysqlRowDataInfo(dbName, tableName, dmlType, columnList, entry) 30 | } 31 | 32 | 33 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/lifecycle/reborn/sink/MysqlBinlogInOrderMysqlSinkerManager.scala: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.mysql.lifecycle.reborn.sink 2 | 3 | import akka.actor.{ActorRef, Props} 4 | import com.neighborhood.aka.laplace.estuary.core.task.TaskManager 5 | import com.neighborhood.aka.laplace.estuary.mysql.sink.MysqlSinkManagerImp 6 | 7 | /** 8 | * Created by john_liu on 2019/1/14. 9 | * MysqlSink的SinkManager 10 | * 11 | * @note 需要提供: 12 | * $sinkerName 默认加载Mysql2Mysql 13 | * @author neighborhood.aka.laplace 14 | */ 15 | final class MysqlBinlogInOrderMysqlSinkerManager(override val taskManager: MysqlSinkManagerImp with TaskManager) extends MysqlBinlogInOrderSinkerManager(taskManager) { 16 | 17 | 18 | /** 19 | * position记录器 20 | */ 21 | override lazy val positionRecorder: Option[ActorRef] = taskManager.positionRecorder 22 | 23 | override val sinkerName: String = "sinker" 24 | 25 | /** 26 | * 初始化sinkers 27 | */ 28 | override protected def initSinkers: Unit = { 29 | val sinkTypeName = sinkerNameToLoad.get(sinkerName).getOrElse(MysqlBinlogInOrderMysqlSinker.name) 30 | log.info(s"MysqlBinlogInOrderMysqlSinkerManager init sinkers,sinkerName:$sinkTypeName,num:$sinkerNum,id:$syncTaskId") 31 | val sinkerList = (1 to sinkerNum).map(index => MysqlBinlogInOrderSinker.buildMysqlBinlogInOrderSinker(sinkTypeName, taskManager, index).withDispatcher("akka.sinker-dispatcher")).map(context.actorOf(_)).toList 32 | taskManager.sinkerList = sinkerList 33 | log.info(s"sinkList has been updated into taskManager,id:$syncTaskId") 34 | } 35 | } 36 | 37 | object MysqlBinlogInOrderMysqlSinkerManager { 38 | lazy val name = MysqlBinlogInOrderMysqlSinkerManager.getClass.getName.stripSuffix("$") 39 | 40 | def props(taskManager: MysqlSinkManagerImp with TaskManager): Props = Props(new MysqlBinlogInOrderMysqlSinkerManager(taskManager)) 41 | } -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/mysql/schema/event/deserialization/QueryEventDataDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Stanley Shyiko 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.neighborhood.aka.laplace.estuary.mysql.schema.event.deserialization; 17 | 18 | import com.neighborhood.aka.laplace.estuary.mysql.schema.io.ByteArrayInputStream; 19 | import com.neighborhood.aka.laplace.estuary.mysql.schema.event.QueryEventData; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * @author Stanley Shyiko 25 | */ 26 | public class QueryEventDataDeserializer implements EventDataDeserializer { 27 | 28 | @Override 29 | public QueryEventData deserialize(ByteArrayInputStream inputStream) throws IOException { 30 | QueryEventData eventData = new QueryEventData(); 31 | eventData.setThreadId(inputStream.readLong(4)); 32 | eventData.setExecutionTime(inputStream.readLong(4)); 33 | inputStream.skip(1); // length of the name of the database 34 | eventData.setErrorCode(inputStream.readInteger(2)); 35 | inputStream.skip(inputStream.readInteger(2)); // status variables block 36 | eventData.setDatabase(inputStream.readZeroTerminatedString()); 37 | eventData.setSql(inputStream.readString(inputStream.available())); 38 | return eventData; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/scala/com/neighborhood/aka/laplace/estuary/core/util/message/MessageBody.java: -------------------------------------------------------------------------------- 1 | package com.neighborhood.aka.laplace.estuary.core.util.message; 2 | 3 | import java.util.List; 4 | 5 | public class MessageBody { 6 | private String batchNo;//不用填 7 | private String bsCode ="SYSTEM";//默认的SYSTEM 8 | private List messageContents;//内容 必填 9 | private List mobiles;//必填 10 | private int senderId =1; 11 | private String senderName;//必填 12 | 13 | public static MessageBody buildMessage(List messageContents,List mobiles,String senderName) { 14 | MessageBody msb = new MessageBody(); 15 | msb.setMessageContents(messageContents); 16 | msb.setMobiles(messageContents); 17 | msb.setSenderName(senderName); 18 | return msb; 19 | } 20 | public String getBatchNo() { 21 | return batchNo; 22 | } 23 | 24 | public void setBatchNo(String batchNo) { 25 | this.batchNo = batchNo; 26 | } 27 | 28 | public String getBsCode() { 29 | return bsCode; 30 | } 31 | 32 | public void setBsCode(String bsCode) { 33 | this.bsCode = bsCode; 34 | } 35 | 36 | public List getMessageContents() { 37 | return messageContents; 38 | } 39 | 40 | public void setMessageContents(List messageContents) { 41 | this.messageContents = messageContents; 42 | } 43 | 44 | public List getMobiles() { 45 | return mobiles; 46 | } 47 | 48 | public void setMobiles(List mobiles) { 49 | this.mobiles = mobiles; 50 | } 51 | 52 | public int getSenderId() { 53 | return senderId; 54 | } 55 | 56 | public void setSenderId(int senderId) { 57 | this.senderId = senderId; 58 | } 59 | 60 | public String getSenderName() { 61 | return senderName; 62 | } 63 | 64 | public void setSenderName(String senderName) { 65 | this.senderName = senderName; 66 | } 67 | } 68 | --------------------------------------------------------------------------------