├── .github └── issue_template.md ├── .gitignore ├── LICENSE.txt ├── README.md ├── RELEASE.txt ├── client-adapter ├── README.md ├── common │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── client │ │ └── adapter │ │ ├── OuterAdapter.java │ │ └── support │ │ ├── CanalClientConfig.java │ │ ├── Constant.java │ │ ├── DatasourceConfig.java │ │ ├── Dml.java │ │ ├── EtlResult.java │ │ ├── ExtensionLoader.java │ │ ├── JdbcTypeUtil.java │ │ ├── MappingConfigsLoader.java │ │ ├── MessageUtil.java │ │ ├── OuterAdapterConfig.java │ │ ├── Result.java │ │ ├── SPI.java │ │ ├── URLClassExtensionLoader.java │ │ └── Util.java ├── elasticsearch │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── alibaba │ │ │ │ └── otter │ │ │ │ └── canal │ │ │ │ └── client │ │ │ │ └── adapter │ │ │ │ └── es │ │ │ │ ├── ESAdapter.java │ │ │ │ ├── config │ │ │ │ ├── ESSyncConfig.java │ │ │ │ ├── ESSyncConfigLoader.java │ │ │ │ ├── SchemaItem.java │ │ │ │ └── SqlParser.java │ │ │ │ ├── monitor │ │ │ │ └── ESConfigMonitor.java │ │ │ │ ├── service │ │ │ │ ├── ESEtlService.java │ │ │ │ └── ESSyncService.java │ │ │ │ └── support │ │ │ │ ├── ESSyncUtil.java │ │ │ │ └── ESTemplate.java │ │ └── resources │ │ │ ├── META-INF │ │ │ └── canal │ │ │ │ └── com.alibaba.otter.canal.client.adapter.OuterAdapter │ │ │ └── es │ │ │ └── mytest_user.yml │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── alibaba │ │ │ └── otter │ │ │ └── canal │ │ │ └── client │ │ │ └── adapter │ │ │ └── es │ │ │ └── test │ │ │ ├── ConfigLoadTest.java │ │ │ ├── SqlParseTest.java │ │ │ ├── TestConstant.java │ │ │ └── sync │ │ │ ├── Common.java │ │ │ ├── LabelSyncJoinSub2Test.java │ │ │ ├── LabelSyncJoinSubTest.java │ │ │ ├── RoleSyncJoinOne2Test.java │ │ │ ├── RoleSyncJoinOneTest.java │ │ │ ├── UserSyncJoinOneTest.java │ │ │ ├── UserSyncSingleTest.java │ │ │ ├── db_schema.sql │ │ │ └── es_mapping.json │ │ └── resources │ │ ├── es │ │ └── mytest_user_single.yml_ │ │ ├── log4j2-test.xml │ │ └── logback-test.xml ├── hbase │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── alibaba │ │ │ └── otter │ │ │ └── canal │ │ │ └── client │ │ │ └── adapter │ │ │ └── hbase │ │ │ ├── HbaseAdapter.java │ │ │ ├── config │ │ │ ├── MappingConfig.java │ │ │ └── MappingConfigLoader.java │ │ │ ├── monitor │ │ │ └── HbaseConfigMonitor.java │ │ │ ├── service │ │ │ ├── HbaseEtlService.java │ │ │ └── HbaseSyncService.java │ │ │ └── support │ │ │ ├── HRow.java │ │ │ ├── HbaseTemplate.java │ │ │ ├── PhType.java │ │ │ ├── PhTypeUtil.java │ │ │ ├── Type.java │ │ │ └── TypeUtil.java │ │ └── resources │ │ ├── META-INF │ │ └── canal │ │ │ └── com.alibaba.otter.canal.client.adapter.OuterAdapter │ │ └── hbase │ │ └── mytest_person2.yml ├── launcher │ ├── pom.xml │ └── src │ │ └── main │ │ ├── assembly │ │ ├── dev.xml │ │ └── release.xml │ │ ├── bin │ │ ├── restart.sh │ │ ├── startup.bat │ │ ├── startup.sh │ │ └── stop.sh │ │ ├── java │ │ └── com │ │ │ └── alibaba │ │ │ └── otter │ │ │ └── canal │ │ │ └── adapter │ │ │ └── launcher │ │ │ ├── CanalAdapterApplication.java │ │ │ ├── common │ │ │ ├── EtlLock.java │ │ │ ├── Mode.java │ │ │ └── SyncSwitch.java │ │ │ ├── config │ │ │ ├── AdapterCanalConfig.java │ │ │ ├── CuratorClient.java │ │ │ └── SpringContext.java │ │ │ ├── loader │ │ │ ├── AbstractCanalAdapterWorker.java │ │ │ ├── CanalAdapterKafkaWorker.java │ │ │ ├── CanalAdapterLoader.java │ │ │ ├── CanalAdapterRocketMQWorker.java │ │ │ ├── CanalAdapterService.java │ │ │ └── CanalAdapterWorker.java │ │ │ ├── monitor │ │ │ └── ApplicationConfigMonitor.java │ │ │ └── rest │ │ │ └── CommonRest.java │ │ └── resources │ │ ├── application.yml │ │ └── es │ │ ├── mytest_user.json │ │ ├── mytest_user.yml │ │ └── test1_user.yml ├── logger │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── alibaba │ │ │ └── otter │ │ │ └── canal │ │ │ └── client │ │ │ └── adapter │ │ │ └── logger │ │ │ └── LoggerAdapterExample.java │ │ └── resources │ │ └── META-INF │ │ └── canal │ │ └── com.alibaba.otter.canal.client.adapter.OuterAdapter ├── pom.xml └── rdb │ ├── pom.xml │ └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── alibaba │ │ │ └── otter │ │ │ └── canal │ │ │ └── client │ │ │ └── adapter │ │ │ └── rdb │ │ │ ├── RdbAdapter.java │ │ │ ├── config │ │ │ ├── ConfigLoader.java │ │ │ └── MappingConfig.java │ │ │ ├── monitor │ │ │ └── RdbConfigMonitor.java │ │ │ ├── service │ │ │ ├── RdbEtlService.java │ │ │ └── RdbSyncService.java │ │ │ └── support │ │ │ ├── BatchExecutor.java │ │ │ ├── SingleDml.java │ │ │ └── SyncUtil.java │ └── resources │ │ ├── META-INF │ │ └── canal │ │ │ └── com.alibaba.otter.canal.client.adapter.OuterAdapter │ │ └── rdb │ │ └── mytest_user.yml │ └── test │ ├── java │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── client │ │ └── adapter │ │ └── rdb │ │ └── test │ │ ├── ConfigLoadTest.java │ │ ├── DBTest.java │ │ ├── TestConstant.java │ │ └── sync │ │ ├── Common.java │ │ └── OracleSyncTest.java │ └── resources │ ├── log4j2-test.xml │ ├── logback-test.xml │ └── rdb │ └── mytest_user.yml ├── client ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── client │ │ ├── CanalConnector.java │ │ ├── CanalConnectors.java │ │ ├── CanalMQConnector.java │ │ ├── CanalMessageDeserializer.java │ │ ├── CanalNodeAccessStrategy.java │ │ ├── impl │ │ ├── ClusterCanalConnector.java │ │ ├── ClusterNodeAccessStrategy.java │ │ ├── ServerNotFoundException.java │ │ ├── SimpleCanalConnector.java │ │ ├── SimpleNodeAccessStrategy.java │ │ └── running │ │ │ ├── ClientRunningData.java │ │ │ ├── ClientRunningListener.java │ │ │ └── ClientRunningMonitor.java │ │ ├── kafka │ │ ├── KafkaCanalConnector.java │ │ └── MessageDeserializer.java │ │ └── rocketmq │ │ ├── ConsumerBatchMessage.java │ │ └── RocketMQCanalConnector.java │ └── test │ └── java │ ├── com │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── client │ │ └── running │ │ ├── AbstractZkTest.java │ │ ├── ClientRunningTest.java │ │ ├── kafka │ │ ├── AbstractKafkaTest.java │ │ ├── CanalKafkaClientExample.java │ │ └── KafkaClientRunningTest.java │ │ └── rocketmq │ │ ├── AbstractRocektMQTest.java │ │ └── CanalRocketMQClientExample.java │ └── logback.xml ├── codeformat.xml ├── codetemplates.xml ├── common ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ ├── alibaba │ │ └── otter │ │ │ └── canal │ │ │ └── common │ │ │ ├── AbstractCanalLifeCycle.java │ │ │ ├── CanalException.java │ │ │ ├── CanalLifeCycle.java │ │ │ ├── alarm │ │ │ ├── CanalAlarmHandler.java │ │ │ └── LogAlarmHandler.java │ │ │ ├── utils │ │ │ ├── AddressUtils.java │ │ │ ├── BooleanMutex.java │ │ │ ├── CanalToStringStyle.java │ │ │ ├── JsonUtils.java │ │ │ ├── NamedThreadFactory.java │ │ │ └── UriUtils.java │ │ │ └── zookeeper │ │ │ ├── ByteSerializer.java │ │ │ ├── StringSerializer.java │ │ │ ├── ZkClientx.java │ │ │ ├── ZooKeeperx.java │ │ │ ├── ZookeeperPathUtils.java │ │ │ └── running │ │ │ ├── ServerRunningData.java │ │ │ ├── ServerRunningListener.java │ │ │ ├── ServerRunningMonitor.java │ │ │ └── ServerRunningMonitors.java │ │ └── google │ │ └── common │ │ └── collect │ │ └── MigrateMap.java │ └── test │ └── java │ ├── com │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── common │ │ ├── AbstractZkTest.java │ │ └── ServerRunningTest.java │ └── logback.xml ├── dbsync ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── taobao │ │ └── tddl │ │ └── dbsync │ │ └── binlog │ │ ├── CharsetConversion.java │ │ ├── DirectLogFetcher.java │ │ ├── FileLogFetcher.java │ │ ├── JsonConversion.java │ │ ├── JsonDiffConversion.java │ │ ├── LogBuffer.java │ │ ├── LogContext.java │ │ ├── LogDecoder.java │ │ ├── LogEvent.java │ │ ├── LogFetcher.java │ │ ├── LogPosition.java │ │ └── event │ │ ├── AppendBlockLogEvent.java │ │ ├── BeginLoadQueryLogEvent.java │ │ ├── CreateFileLogEvent.java │ │ ├── DeleteFileLogEvent.java │ │ ├── DeleteRowsLogEvent.java │ │ ├── ExecuteLoadLogEvent.java │ │ ├── ExecuteLoadQueryLogEvent.java │ │ ├── FormatDescriptionLogEvent.java │ │ ├── GtidLogEvent.java │ │ ├── HeartbeatLogEvent.java │ │ ├── IgnorableLogEvent.java │ │ ├── IncidentLogEvent.java │ │ ├── IntvarLogEvent.java │ │ ├── LoadLogEvent.java │ │ ├── LogHeader.java │ │ ├── PreviousGtidsLogEvent.java │ │ ├── QueryLogEvent.java │ │ ├── RandLogEvent.java │ │ ├── RotateLogEvent.java │ │ ├── RowsLogBuffer.java │ │ ├── RowsLogEvent.java │ │ ├── RowsQueryLogEvent.java │ │ ├── StartLogEventV3.java │ │ ├── StopLogEvent.java │ │ ├── TableMapLogEvent.java │ │ ├── TransactionContextLogEvent.java │ │ ├── UnknownLogEvent.java │ │ ├── UpdateRowsLogEvent.java │ │ ├── UserVarLogEvent.java │ │ ├── ViewChangeEvent.java │ │ ├── WriteRowsLogEvent.java │ │ ├── XaPrepareLogEvent.java │ │ ├── XidLogEvent.java │ │ └── mariadb │ │ ├── AnnotateRowsEvent.java │ │ ├── BinlogCheckPointLogEvent.java │ │ ├── MariaGtidListLogEvent.java │ │ ├── MariaGtidLogEvent.java │ │ └── StartEncryptionLogEvent.java │ └── test │ ├── java │ └── com │ │ └── taobao │ │ └── tddl │ │ └── dbsync │ │ ├── FetcherPerformanceTest.java │ │ └── binlog │ │ ├── BaseLogFetcherTest.java │ │ ├── DirectLogFetcherTest.java │ │ ├── FileLogFetcherTest.java │ │ └── LogBufferTest.java │ └── resources │ ├── binlog │ └── mysql-bin.000001 │ └── dummy.txt ├── deployer ├── pom.xml └── src │ └── main │ ├── assembly │ ├── dev.xml │ └── release.xml │ ├── bin │ ├── restart.sh │ ├── startup.bat │ ├── startup.sh │ └── stop.sh │ ├── java │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── deployer │ │ ├── CanalConstants.java │ │ ├── CanalController.java │ │ ├── CanalLauncher.java │ │ ├── InstanceConfig.java │ │ └── monitor │ │ ├── InstanceAction.java │ │ ├── InstanceConfigMonitor.java │ │ ├── ManagerInstanceConfigMonitor.java │ │ └── SpringInstanceConfigMonitor.java │ └── resources │ ├── canal.properties │ ├── example │ └── instance.properties │ ├── logback.xml │ ├── metrics │ └── Canal_instances_tmpl.json │ └── spring │ ├── base-instance.xml │ ├── default-instance.xml │ ├── file-instance.xml │ ├── group-instance.xml │ ├── memory-instance.xml │ └── tsdb │ ├── h2-tsdb.xml │ ├── mysql-tsdb.xml │ ├── sql-map │ ├── sqlmap-config.xml │ ├── sqlmap_history.xml │ └── sqlmap_snapshot.xml │ └── sql │ └── create_table.sql ├── docker ├── Dockerfile ├── base │ └── Dockerfile ├── build.sh ├── image │ ├── admin │ │ ├── app.sh │ │ ├── bin │ │ │ ├── clean_log │ │ │ └── clean_log.sh │ │ └── health.sh │ └── alidata │ │ ├── bin │ │ ├── exec_rc_local.sh │ │ ├── lark-wait │ │ └── main.sh │ │ ├── init │ │ ├── 02init-sshd.sh │ │ └── fix-hosts.py │ │ └── lib │ │ └── proc.sh └── run.sh ├── driver ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── parse │ │ └── driver │ │ └── mysql │ │ ├── MysqlConnector.java │ │ ├── MysqlQueryExecutor.java │ │ ├── MysqlUpdateExecutor.java │ │ ├── packets │ │ ├── CommandPacket.java │ │ ├── GTIDSet.java │ │ ├── HeaderPacket.java │ │ ├── IPacket.java │ │ ├── MysqlGTIDSet.java │ │ ├── PacketWithHeaderPacket.java │ │ ├── UUIDSet.java │ │ ├── client │ │ │ ├── BinlogDumpCommandPacket.java │ │ │ ├── BinlogDumpGTIDCommandPacket.java │ │ │ ├── ClientAuthenticationPacket.java │ │ │ ├── QueryCommandPacket.java │ │ │ ├── QuitCommandPacket.java │ │ │ ├── RegisterSlaveCommandPacket.java │ │ │ └── SemiAckCommandPacket.java │ │ └── server │ │ │ ├── DataPacket.java │ │ │ ├── EOFPacket.java │ │ │ ├── ErrorPacket.java │ │ │ ├── FieldPacket.java │ │ │ ├── HandshakeInitializationPacket.java │ │ │ ├── OKPacket.java │ │ │ ├── Reply323Packet.java │ │ │ ├── ResultSetHeaderPacket.java │ │ │ ├── ResultSetPacket.java │ │ │ └── RowDataPacket.java │ │ ├── socket │ │ ├── BioSocketChannel.java │ │ ├── BioSocketChannelPool.java │ │ ├── NettySocketChannel.java │ │ ├── NettySocketChannelPool.java │ │ ├── SocketChannel.java │ │ └── SocketChannelPool.java │ │ └── utils │ │ ├── BinlogDumpCommandBuilder.java │ │ ├── ByteHelper.java │ │ ├── ChannelBufferHelper.java │ │ ├── CharsetUtil.java │ │ ├── LengthCodedStringReader.java │ │ ├── MSC.java │ │ ├── MySQLPasswordEncrypter.java │ │ └── PacketManager.java │ └── test │ └── java │ └── com │ └── alibaba │ └── otter │ └── canal │ └── parse │ └── driver │ └── mysql │ ├── CharsetUtilTest.java │ ├── MysqlConnectorTest.java │ ├── MysqlGTIDSetTest.java │ └── UUIDSetTest.java ├── example ├── pom.xml └── src │ └── main │ ├── assembly │ ├── dev.xml │ └── release.xml │ ├── bin │ ├── init.sh │ ├── startup.bat │ ├── startup.sh │ └── stop.sh │ ├── conf │ └── logback.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── example │ │ ├── AbstractCanalClientTest.java │ │ ├── ClusterCanalClientTest.java │ │ ├── SimpleCanalClientPermanceTest.java │ │ ├── SimpleCanalClientTest.java │ │ └── db │ │ ├── AbstractDbClient.java │ │ ├── CanalConnectorClient.java │ │ ├── MysqlLoadLauncher.java │ │ ├── PropertyPlaceholderConfigurer.java │ │ ├── ServiceLocator.java │ │ ├── dialect │ │ ├── AbstractDbDialect.java │ │ ├── AbstractSqlTemplate.java │ │ ├── DbDialect.java │ │ ├── SqlTemplate.java │ │ ├── TableType.java │ │ └── mysql │ │ │ ├── MysqlDialect.java │ │ │ └── MysqlSqlTemplate.java │ │ ├── mysql │ │ ├── AbstractMysqlClient.java │ │ └── MysqlClient.java │ │ └── utils │ │ ├── ByteArrayConverter.java │ │ ├── DdlUtils.java │ │ ├── SqlTimestampConverter.java │ │ └── SqlUtils.java │ └── resources │ ├── client-spring.xml │ ├── client.properties │ └── logback.xml ├── filter ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── filter │ │ ├── CanalEventFilter.java │ │ ├── PatternUtils.java │ │ ├── aviater │ │ ├── AviaterELFilter.java │ │ ├── AviaterRegexFilter.java │ │ ├── AviaterSimpleFilter.java │ │ └── RegexFunction.java │ │ └── exception │ │ └── CanalFilterException.java │ └── test │ └── java │ └── com │ └── alibaba │ └── otter │ └── canal │ └── filter │ ├── AviaterFilterTest.java │ └── MutliAviaterFilterTest.java ├── images ├── QPS.PNG ├── TPS.PNG ├── basic.PNG ├── clients.PNG ├── delay.PNG ├── empty.PNG ├── idle.PNG ├── instance.PNG ├── latency.PNG ├── network.PNG ├── overview.PNG ├── remain_events.PNG ├── remain_mem.PNG ├── reqs.PNG ├── rows.PNG ├── store.PNG ├── throughput.PNG └── transactions.PNG ├── instance ├── core │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── instance │ │ └── core │ │ ├── AbstractCanalInstance.java │ │ ├── CanalInstance.java │ │ ├── CanalInstanceGenerator.java │ │ └── CanalMQConfig.java ├── manager │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── instance │ │ └── manager │ │ ├── CanalConfigClient.java │ │ ├── CanalInstanceWithManager.java │ │ ├── ManagerCanalInstanceGenerator.java │ │ └── model │ │ ├── Canal.java │ │ ├── CanalParameter.java │ │ └── CanalStatus.java ├── pom.xml └── spring │ ├── pom.xml │ └── src │ ├── main │ └── java │ │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── instance │ │ └── spring │ │ ├── CanalInstanceWithSpring.java │ │ ├── SpringCanalInstanceGenerator.java │ │ └── support │ │ ├── PropertyPlaceholderConfigurer.java │ │ └── SocketAddressEditor.java │ └── test │ ├── java │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── instance │ │ └── spring │ │ └── integrated │ │ ├── DefaultSpringInstanceTest.java │ │ ├── GroupSpringInstanceTest.java │ │ └── MemorySpringInstanceTest.java │ └── resources │ ├── canal.properties │ ├── retl │ └── instance.properties │ └── spring │ ├── default-instance.xml │ ├── file-instance.xml │ ├── group-instance.xml │ └── memory-instance.xml ├── meta ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── meta │ │ ├── CanalMetaManager.java │ │ ├── FileMixedMetaManager.java │ │ ├── MemoryMetaManager.java │ │ ├── MixedMetaManager.java │ │ ├── PeriodMixedMetaManager.java │ │ ├── ZooKeeperMetaManager.java │ │ └── exception │ │ └── CanalMetaManagerException.java │ └── test │ └── java │ └── com │ └── alibaba │ └── otter │ └── canal │ └── meta │ ├── AbstractMetaManagerTest.java │ ├── AbstractZkTest.java │ ├── FileMixedMetaManagerTest.java │ ├── MemoryMetaManagerTest.java │ ├── MixedMetaManagerTest.java │ ├── PeriodMixedMetaManagerTest.java │ └── ZooKeeperMetaManagerTest.java ├── parse ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── alibaba │ │ │ └── otter │ │ │ └── canal │ │ │ └── parse │ │ │ ├── CanalEventParser.java │ │ │ ├── CanalHASwitchable.java │ │ │ ├── exception │ │ │ ├── CanalHAException.java │ │ │ ├── CanalParseException.java │ │ │ ├── PositionNotFoundException.java │ │ │ ├── ServerIdNotMatchException.java │ │ │ └── TableIdNotFoundException.java │ │ │ ├── ha │ │ │ ├── CanalHAController.java │ │ │ └── HeartBeatHAController.java │ │ │ ├── inbound │ │ │ ├── AbstractBinlogParser.java │ │ │ ├── AbstractEventParser.java │ │ │ ├── BinlogParser.java │ │ │ ├── ErosaConnection.java │ │ │ ├── EventTransactionBuffer.java │ │ │ ├── HeartBeatCallback.java │ │ │ ├── MultiStageCoprocessor.java │ │ │ ├── ParserExceptionHandler.java │ │ │ ├── SinkFunction.java │ │ │ ├── TableMeta.java │ │ │ ├── group │ │ │ │ └── GroupEventParser.java │ │ │ └── mysql │ │ │ │ ├── AbstractMysqlEventParser.java │ │ │ │ ├── DbsyncMysqlEventParser.java │ │ │ │ ├── LocalBinLogConnection.java │ │ │ │ ├── LocalBinlogEventParser.java │ │ │ │ ├── MysqlConnection.java │ │ │ │ ├── MysqlEventParser.java │ │ │ │ ├── MysqlMultiStageCoprocessor.java │ │ │ │ ├── SlaveEntryPosition.java │ │ │ │ ├── dbsync │ │ │ │ ├── DirectLogFetcher.java │ │ │ │ ├── LogEventConvert.java │ │ │ │ └── TableMetaCache.java │ │ │ │ ├── ddl │ │ │ │ ├── DdlResult.java │ │ │ │ ├── DruidDdlParser.java │ │ │ │ └── SimpleDdlParser.java │ │ │ │ ├── local │ │ │ │ ├── BinLogFileQueue.java │ │ │ │ └── BufferedFileDataInput.java │ │ │ │ ├── rds │ │ │ │ ├── BinlogDownloadQueue.java │ │ │ │ ├── HttpHelper.java │ │ │ │ ├── RdsBinlogEventParserProxy.java │ │ │ │ ├── RdsBinlogOpenApi.java │ │ │ │ ├── RdsLocalBinlogEventParser.java │ │ │ │ ├── data │ │ │ │ │ ├── BinlogFile.java │ │ │ │ │ ├── DescribeBinlogFileResult.java │ │ │ │ │ ├── RdsBackupPolicy.java │ │ │ │ │ └── RdsItem.java │ │ │ │ └── request │ │ │ │ │ ├── AbstractRequest.java │ │ │ │ │ ├── DescribeBackupPolicyRequest.java │ │ │ │ │ └── DescribeBinlogFilesRequest.java │ │ │ │ └── tsdb │ │ │ │ ├── DatabaseTableMeta.java │ │ │ │ ├── DefaultTableMetaTSDBFactory.java │ │ │ │ ├── MemoryTableMeta.java │ │ │ │ ├── TableMetaTSDB.java │ │ │ │ ├── TableMetaTSDBBuilder.java │ │ │ │ ├── TableMetaTSDBFactory.java │ │ │ │ └── dao │ │ │ │ ├── MetaBaseDAO.java │ │ │ │ ├── MetaHistoryDAO.java │ │ │ │ ├── MetaHistoryDO.java │ │ │ │ ├── MetaSnapshotDAO.java │ │ │ │ └── MetaSnapshotDO.java │ │ │ ├── index │ │ │ ├── AbstractLogPositionManager.java │ │ │ ├── CanalLogPositionManager.java │ │ │ ├── FailbackLogPositionManager.java │ │ │ ├── FileMixedLogPositionManager.java │ │ │ ├── MemoryLogPositionManager.java │ │ │ ├── MetaLogPositionManager.java │ │ │ ├── MixedLogPositionManager.java │ │ │ ├── PeriodMixedLogPositionManager.java │ │ │ └── ZooKeeperLogPositionManager.java │ │ │ └── support │ │ │ ├── AuthenticationInfo.java │ │ │ └── HaAuthenticationInfo.java │ └── resources │ │ └── ddl │ │ ├── derby │ │ ├── meta_history.sql │ │ └── meta_snapshot.sql │ │ ├── h2 │ │ ├── meta_history.sql │ │ └── meta_snapshot.sql │ │ └── mysql │ │ ├── meta_history.sql │ │ └── meta_snapshot.sql │ └── test │ ├── java │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── parse │ │ ├── DirectLogFetcherTest.java │ │ ├── MysqlBinlogDumpPerformanceTest.java │ │ ├── MysqlBinlogEventPerformanceTest.java │ │ ├── MysqlBinlogParsePerformanceTest.java │ │ ├── helper │ │ └── TimeoutChecker.java │ │ ├── inbound │ │ ├── EventTransactionBufferTest.java │ │ ├── TableMetaCacheTest.java │ │ ├── group │ │ │ ├── DummyEventStore.java │ │ │ └── GroupEventPaserTest.java │ │ └── mysql │ │ │ ├── LocalBinlogDumpTest.java │ │ │ ├── LocalBinlogEventParserTest.java │ │ │ ├── MysqlDumpTest.java │ │ │ ├── MysqlEventParserTest.java │ │ │ ├── RdsBinlogEventParserProxyTest.java │ │ │ ├── RdsBinlogOpenApiTest.java │ │ │ ├── RdsLocalBinlogDumpTest.java │ │ │ ├── SimpleDdlParserTest.java │ │ │ └── tsdb │ │ │ ├── FastsqlSchemaTest.java │ │ │ ├── MemoryTableMetaTest.java │ │ │ ├── MemoryTableMeta_DDL_Test.java │ │ │ ├── MetaHistoryDAOTest.java │ │ │ ├── MetaSnapshotDAOTest.java │ │ │ ├── TableMetaManagerBuilderTest.java │ │ │ └── TableMetaManagerTest.java │ │ ├── index │ │ ├── AbstractLogPositionManagerTest.java │ │ ├── AbstractZkTest.java │ │ ├── FileMixedLogPositionManagerTest.java │ │ ├── MemoryLogPositionManagerTest.java │ │ ├── MetaLogPositionManagerTest.java │ │ ├── MixedLogPositionManagerTest.java │ │ ├── PeriodMixedLogPositionManagerTest.java │ │ └── ZooKeeperLogPositionManagerTest.java │ │ └── stub │ │ └── AbstractCanalEventSinkTest.java │ └── resources │ ├── binlog │ ├── mysql-bin.000001 │ ├── mysql-bin.000002 │ └── tsdb │ │ ├── mysql-bin.000001 │ │ ├── mysql-bin.000002 │ │ └── mysql-bin.000003 │ ├── ddl │ ├── create.sql │ ├── ddl_test1.sql │ └── ddl_test2.sql │ ├── dummy.txt │ └── tsdb │ ├── derby-tsdb.xml │ ├── h2-tsdb.xml │ ├── mysql-tsdb.xml │ └── sql-map │ ├── sqlmap-config.xml │ ├── sqlmap_history.xml │ └── sqlmap_snapshot.xml ├── pom.xml ├── prometheus ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── prometheus │ │ ├── CanalInstanceExports.java │ │ ├── InstanceRegistry.java │ │ ├── PrometheusProvider.java │ │ ├── PrometheusService.java │ │ └── impl │ │ ├── EntryCollector.java │ │ ├── MetaCollector.java │ │ ├── ParserCollector.java │ │ ├── PrometheusCanalEventDownStreamHandler.java │ │ ├── PrometheusClientInstanceProfiler.java │ │ ├── SinkCollector.java │ │ └── StoreCollector.java │ └── resources │ └── META-INF │ └── services │ └── com.alibaba.otter.canal.spi.CanalMetricsProvider ├── protocol ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── alibaba │ └── otter │ └── canal │ └── protocol │ ├── CanalEntry.java │ ├── CanalPacket.java │ ├── CanalProtocol.proto │ ├── ClientIdentity.java │ ├── EntryProtocol.proto │ ├── FlatMessage.java │ ├── Message.java │ ├── exception │ └── CanalClientException.java │ └── position │ ├── EntryPosition.java │ ├── LogIdentity.java │ ├── LogPosition.java │ ├── MetaqPosition.java │ ├── Position.java │ ├── PositionRange.java │ └── TimePosition.java ├── server ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ ├── common │ │ ├── CanalMessageSerializer.java │ │ ├── MQMessageUtils.java │ │ └── MQProperties.java │ │ ├── kafka │ │ ├── CanalKafkaProducer.java │ │ └── MessageSerializer.java │ │ ├── rocketmq │ │ └── CanalRocketMQProducer.java │ │ ├── server │ │ ├── CanalMQStarter.java │ │ ├── CanalServer.java │ │ ├── CanalServerStarter.java │ │ ├── CanalService.java │ │ ├── embedded │ │ │ └── CanalServerWithEmbedded.java │ │ ├── exception │ │ │ └── CanalServerException.java │ │ └── netty │ │ │ ├── CanalServerWithNetty.java │ │ │ ├── CanalServerWithNettyProfiler.java │ │ │ ├── ClientInstanceProfiler.java │ │ │ ├── NettyUtils.java │ │ │ ├── handler │ │ │ ├── ClientAuthenticationHandler.java │ │ │ ├── FixedHeaderFrameDecoder.java │ │ │ ├── HandshakeInitializationHandler.java │ │ │ └── SessionHandler.java │ │ │ └── listener │ │ │ └── ChannelFutureAggregator.java │ │ └── spi │ │ ├── CanalMQProducer.java │ │ ├── CanalMetricsProvider.java │ │ ├── CanalMetricsService.java │ │ └── NopCanalMetricsService.java │ └── test │ └── java │ └── com │ └── alibaba │ └── otter │ └── canal │ └── server │ ├── BaseCanalServerWithEmbededTest.java │ ├── CanalServerWithEmbedded_FileModeTest.java │ ├── CanalServerWithEmbedded_StandaloneTest.java │ ├── CanalServerWithEmbedded_StandbyTest.java │ ├── CanalServerWithNettyTest.java │ └── ProtocolTest.java ├── sink ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── alibaba │ │ └── otter │ │ └── canal │ │ └── sink │ │ ├── AbstractCanalEventDownStreamHandler.java │ │ ├── AbstractCanalEventSink.java │ │ ├── CanalEventDownStreamHandler.java │ │ ├── CanalEventSink.java │ │ ├── entry │ │ ├── EntryEventSink.java │ │ ├── HeartBeatEntryEventHandler.java │ │ └── group │ │ │ ├── GroupBarrier.java │ │ │ ├── GroupEventSink.java │ │ │ ├── TimelineBarrier.java │ │ │ └── TimelineTransactionBarrier.java │ │ └── exception │ │ └── CanalSinkException.java │ └── test │ └── java │ └── com │ └── alibaba │ └── otter │ └── canal │ └── sink │ ├── GroupEventSinkTest.java │ └── stub │ └── DummyEventStore.java └── store ├── pom.xml └── src ├── main └── java │ └── com │ └── alibaba │ └── otter │ └── canal │ └── store │ ├── AbstractCanalGroupStore.java │ ├── AbstractCanalStoreScavenge.java │ ├── CanalEventStore.java │ ├── CanalGroupEventStore.java │ ├── CanalStoreConstants.java │ ├── CanalStoreException.java │ ├── CanalStoreScavenge.java │ ├── StoreInfo.java │ ├── helper │ └── CanalEventUtils.java │ ├── memory │ └── MemoryEventStoreWithBuffer.java │ └── model │ ├── BatchMode.java │ ├── Event.java │ └── Events.java └── test └── java └── com └── alibaba └── otter └── cancel └── store └── memory └── buffer ├── MemoryEventStoreBase.java ├── MemoryEventStoreMemBatchTest.java ├── MemoryEventStoreMultiThreadTest.java ├── MemoryEventStorePutAndGetTest.java └── MemoryEventStoreRollbackAndAckTest.java /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | ### environment 2 | 3 | * canal version 4 | * mysql version 5 | 6 | ### Issue Description 7 | 8 | 9 | ### Steps to reproduce 10 | 11 | ### Expected behaviour 12 | 13 | ### Actual behaviour 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .svn/ 2 | target/ 3 | test-output/ 4 | *.class 5 | .classpath 6 | .project 7 | .settings/ 8 | tmp 9 | temp 10 | *.log 11 | antx.properties 12 | otter.properties 13 | jtester.properties 14 | .idea/ 15 | *.iml 16 | .DS_Store 17 | *.tar.gz 18 | *.rpm 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 背景 2 | canal的具体使用，请参考阿里官方的代码，已经有非常详细的说明，这里就不多说明了。 3 | 地址：https://github.com/alibaba/canal 4 | 在canal具体的使用中，遇到一些bug以及不便之处，拉了一个分支进行修改，如有兴趣，欢迎大家补充。 5 | 6 | # 说明 7 | 8 | ## ES初始化 9 | 一个全新的ES无法使用，因为没有创建es index和mapping,增加了对应的功能。 10 | ES配置文件mapping节点增加两个参数： 11 | ``` yml 12 | enablefieldmap: true 13 | fieldmap: 14 | id: "text" 15 | BuildingId: "text" 16 | HouseNum: "text" 17 | Floors: "text" 18 | IdProjectInfo: "text" 19 | HouseDigitNum: "text" 20 | BuildingNum: "text" 21 | BuildingName: "text" 22 | Name: "text" 23 | projectid: "text" 24 | bIdProjectInfo: "text" 25 | cinitid: "text" 26 | pCommunityId: "text" 27 | ``` 28 | 29 | enablefieldmap 是否需要自动生成fieldmap，默认为false,如果需要启动的时候就生成这设置为true,并且设置 30 | fieldmap,类似es mapping中每个字段的类型。 31 | 32 | ## esconfig bug处理 33 | 代码中获取binlog的日志处理时，必须要获取数据库名，但是当获取binlog为type query时，是无法获取 34 | 数据库名的，此处有bug，导致出现 "Outer adapter write failed" ,且未输出错误日志，修复此bug. 35 | 36 | ## 后续计划 37 | 增加rabbit MQ的支持 38 | 增加redis的支持（建议实现） -------------------------------------------------------------------------------- /RELEASE.txt: -------------------------------------------------------------------------------- 1 | release link : https://alibaba.github.com/canal/release.html 2 | 3 | download link : https://github.com/alibaba/canal/releases 4 | -------------------------------------------------------------------------------- /client-adapter/common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | canal.client-adapter 5 | com.alibaba.otter 6 | 1.1.3-SNAPSHOT 7 | 8 | 4.0.0 9 | com.alibaba.otter 10 | client-adapter.common 11 | jar 12 | canal client adapter common module for otter ${project.version} 13 | 14 | 15 | com.alibaba.otter 16 | canal.protocol 17 | 1.1.3-SNAPSHOT 18 | 19 | 20 | joda-time 21 | joda-time 22 | 2.9.4 23 | 24 | 25 | com.alibaba 26 | druid 27 | 1.1.9 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.adapter.support; 2 | 3 | public class Constant { 4 | 5 | public static final String CONF_DIR = "conf"; 6 | } 7 | -------------------------------------------------------------------------------- /client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/EtlResult.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.adapter.support; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * ETL的结果对象 7 | * 8 | * @author rewerma @ 2018-10-20 9 | * @version 1.0.0 10 | */ 11 | public class EtlResult implements Serializable { 12 | 13 | private static final long serialVersionUID = 4250522736289866505L; 14 | 15 | private boolean succeeded = false; 16 | 17 | private String resultMessage; 18 | 19 | private String errorMessage; 20 | 21 | public boolean getSucceeded() { 22 | return succeeded; 23 | } 24 | 25 | public void setSucceeded(boolean succeeded) { 26 | this.succeeded = succeeded; 27 | } 28 | 29 | public String getResultMessage() { 30 | return resultMessage; 31 | } 32 | 33 | public void setResultMessage(String resultMessage) { 34 | this.resultMessage = resultMessage; 35 | } 36 | 37 | public String getErrorMessage() { 38 | return errorMessage; 39 | } 40 | 41 | public void setErrorMessage(String errorMessage) { 42 | this.errorMessage = errorMessage; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Result.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.adapter.support; 2 | 3 | import java.io.Serializable; 4 | import java.util.Date; 5 | 6 | /** 7 | * 用于rest的结果返回对象 8 | * 9 | * @author rewerma @ 2018-10-20 10 | * @version 1.0.0 11 | */ 12 | public class Result implements Serializable { 13 | 14 | private static final long serialVersionUID = -3276409502352405716L; 15 | public Integer code = 20000; 16 | public Object data; 17 | public String message; 18 | public Date sysTime; 19 | 20 | public static Result createSuccess(String message) { 21 | Result result = new Result(); 22 | result.setMessage(message); 23 | return result; 24 | } 25 | 26 | public Integer getCode() { 27 | return code; 28 | } 29 | 30 | public void setCode(Integer code) { 31 | this.code = code; 32 | } 33 | 34 | public Object getData() { 35 | return data; 36 | } 37 | 38 | public void setData(Object data) { 39 | this.data = data; 40 | } 41 | 42 | public String getMessage() { 43 | return message; 44 | } 45 | 46 | public void setMessage(String message) { 47 | this.message = message; 48 | } 49 | 50 | public Date getSysTime() { 51 | return sysTime; 52 | } 53 | 54 | public void setSysTime(Date sysTime) { 55 | this.sysTime = sysTime; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/SPI.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.adapter.support; 2 | 3 | import java.lang.annotation.Documented; 4 | import java.lang.annotation.ElementType; 5 | import java.lang.annotation.Retention; 6 | import java.lang.annotation.RetentionPolicy; 7 | import java.lang.annotation.Target; 8 | 9 | /** 10 | * SPI装载器注解 11 | * 12 | * @author rewerma @ 2018-10-20 13 | * @version 1.0.0 14 | */ 15 | @Documented 16 | @Retention(RetentionPolicy.RUNTIME) 17 | @Target({ ElementType.TYPE }) 18 | public @interface SPI { 19 | 20 | // Default SPI name 21 | String value() default ""; 22 | } 23 | -------------------------------------------------------------------------------- /client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.adapter.es.config; 2 | 3 | import java.util.LinkedHashMap; 4 | import java.util.Map; 5 | 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.yaml.snakeyaml.Yaml; 9 | 10 | import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; 11 | 12 | /** 13 | * ES 配置装载器 14 | * 15 | * @author rewerma 2018-11-01 16 | * @version 1.0.0 17 | */ 18 | public class ESSyncConfigLoader { 19 | 20 | private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class); 21 | 22 | public static synchronized Map load() { 23 | logger.info("## Start loading es mapping config ... "); 24 | 25 | Map esSyncConfig = new LinkedHashMap<>(); 26 | 27 | Map configContentMap = MappingConfigsLoader.loadConfigs("es"); 28 | configContentMap.forEach((fileName, content) -> { 29 | ESSyncConfig config = new Yaml().loadAs(content, ESSyncConfig.class); 30 | try { 31 | config.validate(); 32 | } catch (Exception e) { 33 | throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e); 34 | } 35 | esSyncConfig.put(fileName, config); 36 | }); 37 | 38 | logger.info("## ES mapping config loaded"); 39 | return esSyncConfig; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /client-adapter/elasticsearch/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter: -------------------------------------------------------------------------------- 1 | es=com.alibaba.otter.canal.client.adapter.es.ESAdapter -------------------------------------------------------------------------------- /client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml: -------------------------------------------------------------------------------- 1 | dataSourceKey: defaultDS 2 | destination: example 3 | esMapping: 4 | _index: mytest_user 5 | _type: _doc 6 | _id: _id 7 | # pk: id 8 | sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name, 9 | a.c_time as _c_time, c.labels as _labels from user a 10 | left join role b on b.id=a.role_id 11 | left join (select user_id, group_concat(label order by id desc separator ';') as labels from label 12 | group by user_id) c on c.user_id=a.id" 13 | # objFields: 14 | # _labels: array:; 15 | etlCondition: "where a.c_time>='{0}'" 16 | commitBatch: 3000 17 | -------------------------------------------------------------------------------- /client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql: -------------------------------------------------------------------------------- 1 | -- ---------------------------- 2 | -- Table structure for label 3 | -- ---------------------------- 4 | DROP TABLE IF EXISTS `label`; 5 | CREATE TABLE `label` ( 6 | `id` bigint(20) NOT NULL AUTO_INCREMENT, 7 | `user_id` bigint(20) NOT NULL, 8 | `label` varchar(30) NOT NULL, 9 | PRIMARY KEY (`id`) 10 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; 11 | 12 | -- ---------------------------- 13 | -- Table structure for role 14 | -- ---------------------------- 15 | DROP TABLE IF EXISTS `role`; 16 | CREATE TABLE `role` ( 17 | `id` bigint(20) NOT NULL AUTO_INCREMENT, 18 | `role_name` varchar(30) NOT NULL, 19 | PRIMARY KEY (`id`) 20 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; 21 | 22 | -- ---------------------------- 23 | -- Table structure for user 24 | -- ---------------------------- 25 | DROP TABLE IF EXISTS `user`; 26 | CREATE TABLE `user` ( 27 | `id` bigint(20) NOT NULL AUTO_INCREMENT, 28 | `name` varchar(30) NOT NULL, 29 | `c_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, 30 | `role_id` bigint(20) DEFAULT NULL, 31 | PRIMARY KEY (`id`) 32 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; 33 | 34 | insert into user (id,name,role_id) values (1,'Eric',1); 35 | insert into role (id,role_name) values (1,'admin'); 36 | insert into role (id,role_name) values (2,'operator'); 37 | insert into label (id,user_id,label) values (1,1,'a'); 38 | insert into label (id,user_id,label) values (2,1,'b'); 39 | commit; -------------------------------------------------------------------------------- /client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/es_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "_doc": { 3 | "properties": { 4 | "_name": { 5 | "type": "text" 6 | }, 7 | "_role_id": { 8 | "type": "long" 9 | }, 10 | "_role_name": { 11 | "type": "text" 12 | }, 13 | "_labels": { 14 | "type": "text" 15 | }, 16 | "_c_time": { 17 | "type": "date" 18 | } 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml_: -------------------------------------------------------------------------------- 1 | dataSourceKey: defaultDS 2 | destination: example 3 | esMapping: 4 | _index: mytest_user 5 | _type: _doc 6 | _id: _id 7 | sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, a.c_time as _c_time from user a" 8 | commitBatch: 3000 -------------------------------------------------------------------------------- /client-adapter/elasticsearch/src/test/resources/log4j2-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /client-adapter/elasticsearch/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /client-adapter/hbase/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter: -------------------------------------------------------------------------------- 1 | hbase=com.alibaba.otter.canal.client.adapter.hbase.HbaseAdapter -------------------------------------------------------------------------------- /client-adapter/launcher/src/main/bin/restart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sh stop.sh 4 | 5 | sh startup.sh 6 | -------------------------------------------------------------------------------- /client-adapter/launcher/src/main/bin/startup.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | @if not "%ECHO%" == "" echo %ECHO% 3 | @if "%OS%" == "Windows_NT" setlocal 4 | 5 | set ENV_PATH=.\ 6 | if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0% 7 | 8 | set conf_dir=%ENV_PATH%\..\conf 9 | 10 | set CLASSPATH=%conf_dir% 11 | set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% 12 | 13 | set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m 14 | set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 15 | set ADAPTER_OPTS= -DappName=canal-adapter 16 | 17 | set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %ADAPTER_OPTS% 18 | 19 | set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.adapter.launcher.CanalAdapterApplication 20 | echo start cmd : %CMD_STR% 21 | 22 | java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.adapter.launcher.CanalAdapterApplication -------------------------------------------------------------------------------- /client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/CanalAdapterApplication.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.adapter.launcher; 2 | 3 | import org.springframework.boot.Banner; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | 7 | /** 8 | * 启动入口 9 | * 10 | * @author rewerma @ 2018-10-20 11 | * @version 1.0.0 12 | */ 13 | @SpringBootApplication 14 | public class CanalAdapterApplication { 15 | 16 | public static void main(String[] args) { 17 | SpringApplication application = new SpringApplication(CanalAdapterApplication.class); 18 | application.setBannerMode(Banner.Mode.OFF); 19 | application.run(args); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/common/Mode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.adapter.launcher.common; 2 | 3 | public enum Mode { 4 | LOCAL, // 本地模式 5 | DISTRIBUTED // 分布式 6 | } 7 | -------------------------------------------------------------------------------- /client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/CuratorClient.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.adapter.launcher.config; 2 | 3 | import javax.annotation.PostConstruct; 4 | import javax.annotation.Resource; 5 | 6 | import org.apache.curator.framework.CuratorFramework; 7 | import org.apache.curator.framework.CuratorFrameworkFactory; 8 | import org.apache.curator.retry.ExponentialBackoffRetry; 9 | import org.springframework.stereotype.Component; 10 | 11 | /** 12 | * curator 配置类 13 | * 14 | * @author rewerma @ 2018-10-20 15 | * @version 1.0.0 16 | */ 17 | @Component 18 | public class CuratorClient { 19 | 20 | @Resource 21 | private AdapterCanalConfig adapterCanalConfig; 22 | 23 | private CuratorFramework curator = null; 24 | 25 | @PostConstruct 26 | public void init() { 27 | if (adapterCanalConfig.getZookeeperHosts() != null) { 28 | curator = CuratorFrameworkFactory.builder() 29 | .connectString(adapterCanalConfig.getZookeeperHosts()) 30 | .retryPolicy(new ExponentialBackoffRetry(1000, 3)) 31 | .sessionTimeoutMs(6000) 32 | .connectionTimeoutMs(3000) 33 | .namespace("canal-adapter") 34 | .build(); 35 | curator.start(); 36 | } 37 | } 38 | 39 | public CuratorFramework getCurator() { 40 | return curator; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/config/SpringContext.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.adapter.launcher.config; 2 | 3 | import org.springframework.beans.BeansException; 4 | import org.springframework.context.ApplicationContext; 5 | import org.springframework.context.ApplicationContextAware; 6 | import org.springframework.stereotype.Component; 7 | 8 | /** 9 | * spring util配置类 10 | * 11 | * @author rewerma @ 2018-10-20 12 | * @version 1.0.0 13 | */ 14 | @Component 15 | public class SpringContext implements ApplicationContextAware { 16 | 17 | private static ApplicationContext context; 18 | 19 | /* 20 | * 注入ApplicationContext 21 | */ 22 | public void setApplicationContext(final ApplicationContext context) throws BeansException { 23 | // 在加载Spring时自动获得context 24 | SpringContext.context = context; 25 | } 26 | 27 | public static Object getBean(final String beanName) { 28 | return SpringContext.context.getBean(beanName); 29 | } 30 | 31 | public static Object getBean(final Class clz) { 32 | return context.getBean(clz); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /client-adapter/launcher/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8081 3 | logging: 4 | level: 5 | ROOT: INFO 6 | org.springframework: WARN 7 | com.alibaba.otter.canal.client.adapter.hbase: DEBUG 8 | com.alibaba.otter.canal.client.adapter.es: trace 9 | com.alibaba.otter.canal.client.adapter.rdb: DEBUG 10 | spring: 11 | jackson: 12 | date-format: yyyy-MM-dd HH:mm:ss 13 | time-zone: GMT+8 14 | default-property-inclusion: non_null 15 | 16 | canal.conf: 17 | canalServerHost: 127.0.0.1:11111 18 | # zookeeperHosts: slave1:2181 19 | # mqServers: slave1:6667 #or rocketmq 20 | # flatMessage: true 21 | batchSize: 500 22 | syncBatchSize: 1000 23 | retries: 0 24 | timeout: 25 | accessKey: 26 | secretKey: 27 | mode: tcp # kafka rocketMQ 28 | srcDataSources: 29 | defaultDS: 30 | url: jdbc:mysql://192.168.2.172:3306/mdm_house?useUnicode=true 31 | username: root 32 | password: jnqn3@vyw 33 | localDS: 34 | url: jdbc:mysql://127.0.0.1:3306/test1?useUnicode=true 35 | username: root 36 | password: 123456 37 | canalAdapters: 38 | - instance: example # canal instance Name or mq topic name 39 | groups: 40 | - groupId: g1 41 | outerAdapters: 42 | - name: es 43 | hosts: 127.0.0.1:9300 44 | properties: 45 | cluster.name: myes 46 | client.transport.sniff: false -------------------------------------------------------------------------------- /client-adapter/launcher/src/main/resources/es/test1_user.yml: -------------------------------------------------------------------------------- 1 | dataSourceKey: localDS 2 | destination: example 3 | esMapping: 4 | _index: test1 5 | _type: user 6 | _id: id 7 | # pk: id 8 | sql: "select id,name,c_time,role_id from user" 9 | #objFields: 10 | #sql: "select id,name,c_time from user" 11 | enablefieldmap: true 12 | etlCondition: "" 13 | commitBatch: 3000 14 | -------------------------------------------------------------------------------- /client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.adapter.logger; 2 | 3 | import java.util.List; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import com.alibaba.fastjson.JSON; 9 | import com.alibaba.fastjson.serializer.SerializerFeature; 10 | import com.alibaba.otter.canal.client.adapter.OuterAdapter; 11 | import com.alibaba.otter.canal.client.adapter.support.Dml; 12 | import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; 13 | import com.alibaba.otter.canal.client.adapter.support.SPI; 14 | 15 | /** 16 | * 外部适配器示例 17 | * 18 | * @author machengyuan 2018-8-19 下午11:45:38 19 | * @version 1.0.0 20 | */ 21 | @SPI("logger") 22 | // logger参数对应CanalOuterAdapterConfiguration配置中的name 23 | public class LoggerAdapterExample implements OuterAdapter { 24 | 25 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 26 | 27 | @Override 28 | public void init(OuterAdapterConfig configuration) { 29 | 30 | } 31 | 32 | public void sync(List dmls) { 33 | for (Dml dml : dmls) { 34 | sync(dml); 35 | } 36 | } 37 | 38 | public void sync(Dml dml) { 39 | logger.info("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue)); 40 | } 41 | 42 | @Override 43 | public void destroy() { 44 | 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /client-adapter/logger/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter: -------------------------------------------------------------------------------- 1 | logger=com.alibaba.otter.canal.client.adapter.logger.LoggerAdapterExample -------------------------------------------------------------------------------- /client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.adapter.rdb.config; 2 | 3 | import java.util.LinkedHashMap; 4 | import java.util.Map; 5 | 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.yaml.snakeyaml.Yaml; 9 | 10 | import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; 11 | 12 | /** 13 | * RDB表映射配置加载器 14 | * 15 | * @author rewerma 2018-11-07 下午02:41:34 16 | * @version 1.0.0 17 | */ 18 | public class ConfigLoader { 19 | 20 | private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class); 21 | 22 | /** 23 | * 加载HBase表映射配置 24 | * 25 | * @return 配置名/配置文件名--对象 26 | */ 27 | public static Map load() { 28 | logger.info("## Start loading rdb mapping config ... "); 29 | 30 | Map result = new LinkedHashMap<>(); 31 | 32 | Map configContentMap = MappingConfigsLoader.loadConfigs("rdb"); 33 | configContentMap.forEach((fileName, content) -> { 34 | MappingConfig config = new Yaml().loadAs(content, MappingConfig.class); 35 | try { 36 | config.validate(); 37 | } catch (Exception e) { 38 | throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e); 39 | } 40 | result.put(fileName, config); 41 | }); 42 | 43 | logger.info("## Rdb mapping config loaded"); 44 | return result; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /client-adapter/rdb/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter: -------------------------------------------------------------------------------- 1 | rdb=com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter -------------------------------------------------------------------------------- /client-adapter/rdb/src/main/resources/rdb/mytest_user.yml: -------------------------------------------------------------------------------- 1 | dataSourceKey: defaultDS 2 | destination: example 3 | outerAdapterKey: oracle1 4 | concurrent: true 5 | dbMapping: 6 | database: mytest 7 | table: user 8 | targetTable: mytest.tb_user 9 | targetPk: 10 | id: id 11 | mapAll: true 12 | # targetColumns: 13 | # id: 14 | # name: 15 | # role_id: 16 | # c_time: 17 | # test1: -------------------------------------------------------------------------------- /client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.adapter.rdb.test; 2 | 3 | import java.util.Map; 4 | 5 | import org.junit.Assert; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | 9 | import com.alibaba.otter.canal.client.adapter.rdb.config.ConfigLoader; 10 | import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; 11 | import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; 12 | 13 | public class ConfigLoadTest { 14 | 15 | @Before 16 | public void before() { 17 | // 加载数据源连接池 18 | DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); 19 | } 20 | 21 | @Test 22 | public void testLoad() { 23 | Map configMap = ConfigLoader.load(); 24 | 25 | Assert.assertFalse(configMap.isEmpty()); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/TestConstant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.adapter.rdb.test; 2 | 3 | import java.sql.SQLException; 4 | 5 | import com.alibaba.druid.pool.DruidDataSource; 6 | 7 | public class TestConstant { 8 | 9 | public final static String jdbcUrl = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true"; 10 | public final static String jdbcUser = "root"; 11 | public final static String jdbcPassword = "121212"; 12 | 13 | public static DruidDataSource dataSource; 14 | 15 | static { 16 | dataSource = new DruidDataSource(); 17 | dataSource.setDriverClassName("com.mysql.jdbc.Driver"); 18 | dataSource.setUrl(jdbcUrl); 19 | dataSource.setUsername(jdbcUser); 20 | dataSource.setPassword(jdbcPassword); 21 | dataSource.setInitialSize(1); 22 | dataSource.setMinIdle(1); 23 | dataSource.setMaxActive(1); 24 | dataSource.setMaxWait(60000); 25 | dataSource.setTimeBetweenEvictionRunsMillis(60000); 26 | dataSource.setMinEvictableIdleTimeMillis(300000); 27 | dataSource.setPoolPreparedStatements(false); 28 | dataSource.setMaxPoolPreparedStatementPerConnectionSize(20); 29 | dataSource.setValidationQuery("select 1"); 30 | try { 31 | dataSource.init(); 32 | } catch (SQLException e) { 33 | e.printStackTrace(); 34 | } 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.adapter.rdb.test.sync; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter; 7 | import com.alibaba.otter.canal.client.adapter.rdb.test.TestConstant; 8 | import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; 9 | import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; 10 | 11 | public class Common { 12 | 13 | public static RdbAdapter init() { 14 | DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource); 15 | 16 | OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig(); 17 | outerAdapterConfig.setName("rdb"); 18 | outerAdapterConfig.setKey("oracle1"); 19 | Map properties = new HashMap<>(); 20 | properties.put("jdbc.driveClassName", "oracle.jdbc.OracleDriver"); 21 | properties.put("jdbc.url", "jdbc:oracle:thin:@127.0.0.1:49161:XE"); 22 | properties.put("jdbc.username", "mytest"); 23 | properties.put("jdbc.password", "m121212"); 24 | outerAdapterConfig.setProperties(properties); 25 | 26 | RdbAdapter adapter = new RdbAdapter(); 27 | adapter.init(outerAdapterConfig); 28 | return adapter; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /client-adapter/rdb/src/test/resources/log4j2-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /client-adapter/rdb/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /client-adapter/rdb/src/test/resources/rdb/mytest_user.yml: -------------------------------------------------------------------------------- 1 | dataSourceKey: defaultDS 2 | destination: example 3 | outerAdapterKey: oracle1 4 | concurrent: true 5 | dbMapping: 6 | database: mytest 7 | table: user 8 | targetTable: mytest.tb_user 9 | targetPk: 10 | id: id 11 | mapAll: true 12 | # targetColumns: 13 | # id: 14 | # name: 15 | # role_id: 16 | # c_time: 17 | # test1: -------------------------------------------------------------------------------- /client/src/main/java/com/alibaba/otter/canal/client/CanalNodeAccessStrategy.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client; 2 | 3 | import java.net.SocketAddress; 4 | 5 | /** 6 | * 集群节点访问控制接口 7 | * 8 | * @author jianghang 2012-10-29 下午07:55:41 9 | * @version 1.0.0 10 | */ 11 | public interface CanalNodeAccessStrategy { 12 | 13 | SocketAddress currentNode(); 14 | 15 | SocketAddress nextNode(); 16 | } 17 | -------------------------------------------------------------------------------- /client/src/main/java/com/alibaba/otter/canal/client/impl/ServerNotFoundException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.impl; 2 | 3 | import com.alibaba.otter.canal.protocol.exception.CanalClientException; 4 | 5 | public class ServerNotFoundException extends CanalClientException { 6 | 7 | private static final long serialVersionUID = -3471518241911601774L; 8 | 9 | public ServerNotFoundException(String errorCode, String errorDesc, Throwable cause){ 10 | super(errorCode, errorDesc, cause); 11 | } 12 | 13 | public ServerNotFoundException(String errorCode, String errorDesc){ 14 | super(errorCode, errorDesc); 15 | } 16 | 17 | public ServerNotFoundException(String errorCode, Throwable cause){ 18 | super(errorCode, cause); 19 | } 20 | 21 | public ServerNotFoundException(String errorCode){ 22 | super(errorCode); 23 | } 24 | 25 | public ServerNotFoundException(Throwable cause){ 26 | super(cause); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleNodeAccessStrategy.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.impl; 2 | 3 | import java.net.SocketAddress; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import com.alibaba.otter.canal.client.CanalNodeAccessStrategy; 8 | 9 | /** 10 | * 简单版本的node访问实现 11 | * 12 | * @author jianghang 2012-10-29 下午08:00:23 13 | * @version 1.0.0 14 | */ 15 | public class SimpleNodeAccessStrategy implements CanalNodeAccessStrategy { 16 | 17 | private List nodes = new ArrayList(); 18 | private int index = 0; 19 | 20 | public SimpleNodeAccessStrategy(List nodes){ 21 | if (nodes == null || nodes.size() < 1) { 22 | throw new IllegalArgumentException("at least 1 node required."); 23 | } 24 | this.nodes.addAll(nodes); 25 | } 26 | 27 | public SocketAddress nextNode() { 28 | try { 29 | return nodes.get(index); 30 | } finally { 31 | index = (index + 1) % nodes.size(); 32 | } 33 | } 34 | 35 | @Override 36 | public SocketAddress currentNode() { 37 | return nodes.get(index); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningData.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.impl.running; 2 | 3 | /** 4 | * client running状态信息 5 | * 6 | * @author jianghang 2012-11-22 下午03:41:50 7 | * @version 1.0.0 8 | */ 9 | public class ClientRunningData { 10 | 11 | private short clientId; 12 | private String address; 13 | private boolean active = true; 14 | 15 | public short getClientId() { 16 | return clientId; 17 | } 18 | 19 | public void setClientId(short clientId) { 20 | this.clientId = clientId; 21 | } 22 | 23 | public String getAddress() { 24 | return address; 25 | } 26 | 27 | public void setAddress(String address) { 28 | this.address = address; 29 | } 30 | 31 | public boolean isActive() { 32 | return active; 33 | } 34 | 35 | public void setActive(boolean active) { 36 | this.active = active; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningListener.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.impl.running; 2 | 3 | import java.net.InetSocketAddress; 4 | 5 | /** 6 | * 触发一下mainstem发生切换 7 | * 8 | * @author jianghang 2012-9-11 下午02:26:03 9 | * @version 1.0.0 10 | */ 11 | public interface ClientRunningListener { 12 | 13 | /** 14 | * 触发现在轮到自己做为active，需要载入上一个active的上下文数据 15 | */ 16 | public InetSocketAddress processActiveEnter(); 17 | 18 | /** 19 | * 触发一下当前active模式失败 20 | */ 21 | public void processActiveExit(); 22 | 23 | } 24 | -------------------------------------------------------------------------------- /client/src/main/java/com/alibaba/otter/canal/client/kafka/MessageDeserializer.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.kafka; 2 | 3 | import java.util.Map; 4 | 5 | import org.apache.kafka.common.serialization.Deserializer; 6 | 7 | import com.alibaba.otter.canal.client.CanalMessageDeserializer; 8 | import com.alibaba.otter.canal.protocol.Message; 9 | 10 | /** 11 | * Kafka Message类的反序列化 12 | * 13 | * @author machengyuan @ 2018-6-12 14 | * @version 1.0.0 15 | */ 16 | public class MessageDeserializer implements Deserializer { 17 | 18 | @Override 19 | public void configure(Map configs, boolean isKey) { 20 | } 21 | 22 | @Override 23 | public Message deserialize(String topic1, byte[] data) { 24 | return CanalMessageDeserializer.deserializer(data); 25 | } 26 | 27 | @Override 28 | public void close() { 29 | // nothing to do 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /client/src/main/java/com/alibaba/otter/canal/client/rocketmq/ConsumerBatchMessage.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.rocketmq; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.CountDownLatch; 5 | import java.util.concurrent.TimeUnit; 6 | 7 | public class ConsumerBatchMessage { 8 | 9 | private final List data; 10 | private CountDownLatch latch; 11 | private boolean hasFailure = false; 12 | 13 | public ConsumerBatchMessage(List data){ 14 | this.data = data; 15 | latch = new CountDownLatch(1); 16 | } 17 | 18 | public boolean waitFinish(long timeout) throws InterruptedException { 19 | return latch.await(timeout, TimeUnit.MILLISECONDS); 20 | } 21 | 22 | public boolean isSuccess() { 23 | return !hasFailure; 24 | } 25 | 26 | public List getData() { 27 | return data; 28 | } 29 | 30 | /** 31 | * Countdown if the sub message is successful. 32 | */ 33 | public void ack() { 34 | latch.countDown(); 35 | } 36 | 37 | /** 38 | * Countdown and fail-fast if the sub message is failed. 39 | */ 40 | public void fail() { 41 | hasFailure = true; 42 | // fail fast 43 | long count = latch.getCount(); 44 | for (int i = 0; i < count; i++) { 45 | latch.countDown(); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /client/src/test/java/com/alibaba/otter/canal/client/running/AbstractZkTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.running; 2 | 3 | import org.junit.Assert; 4 | 5 | public class AbstractZkTest { 6 | 7 | protected String destination = "ljhtest1"; 8 | protected String cluster1 = "127.0.0.1:2188"; 9 | protected String cluster2 = "127.0.0.1:2188,127.0.0.1:2188"; 10 | 11 | public void sleep(long time) { 12 | try { 13 | Thread.sleep(time); 14 | } catch (InterruptedException e) { 15 | Assert.fail(e.getMessage()); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /client/src/test/java/com/alibaba/otter/canal/client/running/kafka/AbstractKafkaTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.running.kafka; 2 | 3 | import org.junit.Assert; 4 | 5 | /** 6 | * Kafka 测试基类 7 | * 8 | * @author machengyuan @ 2018-6-12 9 | * @version 1.0.0 10 | */ 11 | public abstract class AbstractKafkaTest { 12 | 13 | public static String topic = "example"; 14 | public static Integer partition = null; 15 | public static String groupId = "g4"; 16 | public static String servers = "slave1:6667,slave2:6667,slave3:6667"; 17 | public static String zkServers = "slave1:2181,slave2:2181,slave3:2181"; 18 | 19 | public void sleep(long time) { 20 | try { 21 | Thread.sleep(time); 22 | } catch (InterruptedException e) { 23 | Assert.fail(e.getMessage()); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /client/src/test/java/com/alibaba/otter/canal/client/running/rocketmq/AbstractRocektMQTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.client.running.rocketmq; 2 | 3 | public class AbstractRocektMQTest { 4 | 5 | public static String topic = "example"; 6 | public static String groupId = "group"; 7 | public static String nameServers = "localhost:9876"; 8 | 9 | } 10 | -------------------------------------------------------------------------------- /client/src/test/java/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/otter/canal/common/AbstractCanalLifeCycle.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.common; 2 | 3 | /** 4 | * 基本实现 5 | * 6 | * @author jianghang 2012-7-12 上午10:11:07 7 | * @version 1.0.0 8 | */ 9 | public abstract class AbstractCanalLifeCycle implements CanalLifeCycle { 10 | 11 | protected volatile boolean running = false; // 是否处于运行中 12 | 13 | public boolean isStart() { 14 | return running; 15 | } 16 | 17 | public void start() { 18 | if (running) { 19 | throw new CanalException(this.getClass().getName() + " has startup , don't repeat start"); 20 | } 21 | 22 | running = true; 23 | } 24 | 25 | public void stop() { 26 | if (!running) { 27 | throw new CanalException(this.getClass().getName() + " isn't start , please check"); 28 | } 29 | 30 | running = false; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/otter/canal/common/CanalException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.common; 2 | 3 | import org.apache.commons.lang.exception.NestableRuntimeException; 4 | 5 | /** 6 | * @author jianghang 2012-7-12 上午10:10:31 7 | * @version 1.0.0 8 | */ 9 | public class CanalException extends NestableRuntimeException { 10 | 11 | private static final long serialVersionUID = -654893533794556357L; 12 | 13 | public CanalException(String errorCode){ 14 | super(errorCode); 15 | } 16 | 17 | public CanalException(String errorCode, Throwable cause){ 18 | super(errorCode, cause); 19 | } 20 | 21 | public CanalException(String errorCode, String errorDesc){ 22 | super(errorCode + ":" + errorDesc); 23 | } 24 | 25 | public CanalException(String errorCode, String errorDesc, Throwable cause){ 26 | super(errorCode + ":" + errorDesc, cause); 27 | } 28 | 29 | public CanalException(Throwable cause){ 30 | super(cause); 31 | } 32 | 33 | public Throwable fillInStackTrace() { 34 | return this; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/otter/canal/common/CanalLifeCycle.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.common; 2 | 3 | /** 4 | * @author jianghang 2012-7-12 上午09:39:33 5 | * @version 1.0.0 6 | */ 7 | public interface CanalLifeCycle { 8 | 9 | void start(); 10 | 11 | void stop(); 12 | 13 | boolean isStart(); 14 | } 15 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/otter/canal/common/alarm/CanalAlarmHandler.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.common.alarm; 2 | 3 | import com.alibaba.otter.canal.common.CanalLifeCycle; 4 | 5 | /** 6 | * canal报警处理机制 7 | * 8 | * @author jianghang 2012-8-22 下午10:08:56 9 | * @version 1.0.0 10 | */ 11 | public interface CanalAlarmHandler extends CanalLifeCycle { 12 | 13 | /** 14 | * 发送对应destination的报警 15 | * 16 | * @param destination 17 | * @param msg 18 | */ 19 | void sendAlarm(String destination, String msg); 20 | } 21 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/otter/canal/common/alarm/LogAlarmHandler.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.common.alarm; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; 7 | 8 | /** 9 | * 基于log的alarm机制实现 10 | * 11 | * @author jianghang 2012-8-22 下午10:12:35 12 | * @version 1.0.0 13 | */ 14 | public class LogAlarmHandler extends AbstractCanalLifeCycle implements CanalAlarmHandler { 15 | 16 | private static final Logger logger = LoggerFactory.getLogger(LogAlarmHandler.class); 17 | 18 | public void sendAlarm(String destination, String msg) { 19 | logger.error("destination:{}[{}]", new Object[] { destination, msg }); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ByteSerializer.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.common.zookeeper; 2 | 3 | import java.io.UnsupportedEncodingException; 4 | 5 | import org.I0Itec.zkclient.exception.ZkMarshallingError; 6 | import org.I0Itec.zkclient.serialize.ZkSerializer; 7 | 8 | /** 9 | * 基于string的序列化方式 10 | * 11 | * @author jianghang 2012-7-11 下午02:57:09 12 | * @version 1.0.0 13 | */ 14 | public class ByteSerializer implements ZkSerializer { 15 | 16 | public Object deserialize(final byte[] bytes) throws ZkMarshallingError { 17 | return bytes; 18 | } 19 | 20 | public byte[] serialize(final Object data) throws ZkMarshallingError { 21 | try { 22 | if (data instanceof byte[]) { 23 | return (byte[]) data; 24 | } else { 25 | return ((String) data).getBytes("utf-8"); 26 | } 27 | } catch (final UnsupportedEncodingException e) { 28 | throw new ZkMarshallingError(e); 29 | } 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/otter/canal/common/zookeeper/StringSerializer.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.common.zookeeper; 2 | 3 | import java.io.UnsupportedEncodingException; 4 | 5 | import org.I0Itec.zkclient.exception.ZkMarshallingError; 6 | import org.I0Itec.zkclient.serialize.ZkSerializer; 7 | 8 | /** 9 | * 基于string的序列化方式 10 | * 11 | * @author jianghang 2012-7-11 下午02:57:09 12 | * @version 1.0.0 13 | */ 14 | public class StringSerializer implements ZkSerializer { 15 | 16 | public Object deserialize(final byte[] bytes) throws ZkMarshallingError { 17 | try { 18 | return new String(bytes, "utf-8"); 19 | } catch (final UnsupportedEncodingException e) { 20 | throw new ZkMarshallingError(e); 21 | } 22 | } 23 | 24 | public byte[] serialize(final Object data) throws ZkMarshallingError { 25 | try { 26 | return ((String) data).getBytes("utf-8"); 27 | } catch (final UnsupportedEncodingException e) { 28 | throw new ZkMarshallingError(e); 29 | } 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningData.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.common.zookeeper.running; 2 | 3 | import java.io.Serializable; 4 | 5 | import org.apache.commons.lang.builder.ToStringBuilder; 6 | 7 | import com.alibaba.otter.canal.common.utils.CanalToStringStyle; 8 | 9 | /** 10 | * 服务端running状态信息 11 | * 12 | * @author jianghang 2012-11-22 下午03:11:30 13 | * @version 1.0.0 14 | */ 15 | public class ServerRunningData implements Serializable { 16 | 17 | private static final long serialVersionUID = 92260481691855281L; 18 | 19 | private Long cid; 20 | private String address; 21 | private boolean active = true; 22 | 23 | public ServerRunningData(){ 24 | } 25 | 26 | public ServerRunningData(Long cid, String address){ 27 | this.cid = cid; 28 | this.address = address; 29 | } 30 | 31 | public Long getCid() { 32 | return cid; 33 | } 34 | 35 | public void setCid(Long cid) { 36 | this.cid = cid; 37 | } 38 | 39 | public String getAddress() { 40 | return address; 41 | } 42 | 43 | public void setAddress(String address) { 44 | this.address = address; 45 | } 46 | 47 | public boolean isActive() { 48 | return active; 49 | } 50 | 51 | public void setActive(boolean active) { 52 | this.active = active; 53 | } 54 | 55 | public String toString() { 56 | return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningListener.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.common.zookeeper.running; 2 | 3 | /** 4 | * 触发一下mainstem发生切换 5 | * 6 | * @author jianghang 2012-9-11 下午02:26:03 7 | * @version 1.0.0 8 | */ 9 | public interface ServerRunningListener { 10 | 11 | /** 12 | * 启动时回调做点事情 13 | */ 14 | public void processStart(); 15 | 16 | /** 17 | * 关闭时回调做点事情 18 | */ 19 | public void processStop(); 20 | 21 | /** 22 | * 触发现在轮到自己做为active，需要载入上一个active的上下文数据 23 | */ 24 | public void processActiveEnter(); 25 | 26 | /** 27 | * 触发一下当前active模式失败 28 | */ 29 | public void processActiveExit(); 30 | 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitors.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.common.zookeeper.running; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * {@linkplain ServerRunningMonitor}管理容器，使用static进行数据全局共享 7 | * 8 | * @author jianghang 2012-12-3 下午09:32:06 9 | * @version 1.0.0 10 | */ 11 | public class ServerRunningMonitors { 12 | 13 | private static ServerRunningData serverData; 14 | private static Map runningMonitors; // 16 | 17 | public static ServerRunningData getServerData() { 18 | return serverData; 19 | } 20 | 21 | public static Map getRunningMonitors() { 22 | return runningMonitors; 23 | } 24 | 25 | public static ServerRunningMonitor getRunningMonitor(String destination) { 26 | return (ServerRunningMonitor) runningMonitors.get(destination); 27 | } 28 | 29 | public static void setServerData(ServerRunningData serverData) { 30 | ServerRunningMonitors.serverData = serverData; 31 | } 32 | 33 | public static void setRunningMonitors(Map runningMonitors) { 34 | ServerRunningMonitors.runningMonitors = runningMonitors; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/java/com/google/common/collect/MigrateMap.java: -------------------------------------------------------------------------------- 1 | package com.google.common.collect; 2 | 3 | import java.util.concurrent.ConcurrentMap; 4 | 5 | import com.google.common.base.Function; 6 | 7 | public class MigrateMap { 8 | 9 | @SuppressWarnings("deprecation") 10 | public static ConcurrentMap makeComputingMap(MapMaker maker, 11 | Function computingFunction) { 12 | return maker.makeComputingMap(computingFunction); 13 | } 14 | 15 | @SuppressWarnings("deprecation") 16 | public static ConcurrentMap makeComputingMap(Function computingFunction) { 17 | return new MapMaker().makeComputingMap(computingFunction); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /common/src/test/java/com/alibaba/otter/canal/common/AbstractZkTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.common; 2 | 3 | import org.junit.Assert; 4 | 5 | public class AbstractZkTest { 6 | 7 | protected String destination = "ljhtest1"; 8 | protected String cluster1 = "127.0.0.1:2188"; 9 | protected String cluster2 = "127.0.0.1:2188,127.0.0.1:2188"; 10 | 11 | public void sleep(long time) { 12 | try { 13 | Thread.sleep(time); 14 | } catch (InterruptedException e) { 15 | Assert.fail(e.getMessage()); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /common/src/test/java/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/AppendBlockLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.LogEvent; 5 | 6 | /** 7 | * Append_block_log_event. 8 | * 9 | * @author Changyuan.lh 10 | * @version 1.0 11 | */ 12 | public class AppendBlockLogEvent extends LogEvent { 13 | 14 | private final LogBuffer blockBuf; 15 | private final int blockLen; 16 | 17 | private final long fileId; 18 | 19 | /* AB = "Append Block" */ 20 | public static final int AB_FILE_ID_OFFSET = 0; 21 | 22 | public AppendBlockLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 23 | super(header); 24 | 25 | final int commonHeaderLen = descriptionEvent.commonHeaderLen; 26 | final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1]; 27 | final int totalHeaderLen = commonHeaderLen + postHeaderLen; 28 | 29 | buffer.position(commonHeaderLen + AB_FILE_ID_OFFSET); 30 | fileId = buffer.getUint32(); 31 | 32 | buffer.position(postHeaderLen); 33 | blockLen = buffer.limit() - totalHeaderLen; 34 | blockBuf = buffer.duplicate(blockLen); 35 | } 36 | 37 | public final long getFileId() { 38 | return fileId; 39 | } 40 | 41 | public final LogBuffer getBuffer() { 42 | return blockBuf; 43 | } 44 | 45 | public final byte[] getData() { 46 | return blockBuf.getData(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/BeginLoadQueryLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | 5 | /** 6 | * Event for the first block of file to be loaded, its only difference from 7 | * Append_block event is that this event creates or truncates existing file 8 | * before writing data. 9 | * 10 | * @author Changyuan.lh 11 | * @version 1.0 12 | */ 13 | public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent { 14 | 15 | public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 16 | super(header, buffer, descriptionEvent); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteFileLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.LogEvent; 5 | 6 | /** 7 | * Delete_file_log_event. 8 | * 9 | * @author Changyuan.lh 10 | * @version 1.0 11 | */ 12 | public final class DeleteFileLogEvent extends LogEvent { 13 | 14 | private final long fileId; 15 | 16 | /* DF = "Delete File" */ 17 | public static final int DF_FILE_ID_OFFSET = 0; 18 | 19 | public DeleteFileLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 20 | super(header); 21 | 22 | final int commonHeaderLen = descriptionEvent.commonHeaderLen; 23 | buffer.position(commonHeaderLen + DF_FILE_ID_OFFSET); 24 | fileId = buffer.getUint32(); // DF_FILE_ID_OFFSET 25 | } 26 | 27 | public final long getFileId() { 28 | return fileId; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | 5 | /** 6 | * Log row deletions. The event contain several delete rows for a table. Note 7 | * that each event contains only rows for one table. 8 | * 9 | * @author Changyuan.lh 10 | * @version 1.0 11 | */ 12 | public final class DeleteRowsLogEvent extends RowsLogEvent { 13 | 14 | public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 15 | super(header, buffer, descriptionEvent); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.LogEvent; 5 | 6 | /** 7 | * Execute_load_log_event. 8 | * 9 | * @author Changyuan.lh 10 | * @version 1.0 11 | */ 12 | public final class ExecuteLoadLogEvent extends LogEvent { 13 | 14 | private final long fileId; 15 | 16 | /* EL = "Execute Load" */ 17 | public static final int EL_FILE_ID_OFFSET = 0; 18 | 19 | public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 20 | super(header); 21 | 22 | final int commonHeaderLen = descriptionEvent.commonHeaderLen; 23 | buffer.position(commonHeaderLen + EL_FILE_ID_OFFSET); 24 | fileId = buffer.getUint32(); // EL_FILE_ID_OFFSET 25 | } 26 | 27 | public final long getFileId() { 28 | return fileId; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IgnorableLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.LogEvent; 5 | 6 | /** 7 | * 
 8 |  *   Base class for ignorable log events. Events deriving from
 9 |  *   this class can be safely ignored by slaves that cannot
10 |  *   recognize them. Newer slaves, will be able to read and
11 |  *   handle them. This has been designed to be an open-ended
12 |  *   architecture, so adding new derived events shall not harm
13 |  *   the old slaves that support ignorable log event mechanism
14 |  *   (they will just ignore unrecognized ignorable events).
15 |  * 
16 |  *   @note The only thing that makes an event ignorable is that it has
17 |  *   the LOG_EVENT_IGNORABLE_F flag set.  It is not strictly necessary
18 |  *   that ignorable event types derive from Ignorable_log_event; they may
19 |  *   just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
20 |  *   argument to the Log_event constructor.
21 |  *
22 | * 23 | * @author jianghang 2013-4-8 上午12:36:29 24 | * @version 1.0.3 25 | * @since mysql 5.6 26 | */ 27 | public class IgnorableLogEvent extends LogEvent { 28 | 29 | public IgnorableLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 30 | super(header); 31 | 32 | // do nothing , just ignore log event 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/PreviousGtidsLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.LogEvent; 5 | 6 | /** 7 | * @author jianghang 2013-4-8 上午12:36:29 8 | * @version 1.0.3 9 | * @since mysql 5.6 10 | */ 11 | public class PreviousGtidsLogEvent extends LogEvent { 12 | 13 | public PreviousGtidsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 14 | super(header); 15 | // do nothing , just for mysql gtid search function 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsQueryLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | 5 | /** 6 | * @author jianghang 2013-4-8 上午12:36:29 7 | * @version 1.0.3 8 | * @since mysql 5.6 9 | */ 10 | public class RowsQueryLogEvent extends IgnorableLogEvent { 11 | 12 | private String rowsQuery; 13 | 14 | public RowsQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 15 | super(header, buffer, descriptionEvent); 16 | 17 | final int commonHeaderLen = descriptionEvent.commonHeaderLen; 18 | final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1]; 19 | 20 | /* 21 | * m_rows_query length is stored using only one byte, but that length is 22 | * ignored and the complete query is read. 23 | */ 24 | int offset = commonHeaderLen + postHeaderLen + 1; 25 | int len = buffer.limit() - offset; 26 | rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1); 27 | } 28 | 29 | public String getRowsQuery() { 30 | return rowsQuery; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StopLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.LogEvent; 5 | 6 | /** 7 | * Stop_log_event. The Post-Header and Body for this event type are empty; it 8 | * only has the Common-Header. 9 | * 10 | * @author Changyuan.lh 11 | * @version 1.0 12 | */ 13 | public final class StopLogEvent extends LogEvent { 14 | 15 | public StopLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent description_event){ 16 | super(header); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TransactionContextLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.LogEvent; 5 | 6 | /** 7 | * @author agapple 2018年5月7日 下午7:05:39 8 | * @version 1.0.26 9 | * @since mysql 5.7 10 | */ 11 | public class TransactionContextLogEvent extends LogEvent { 12 | 13 | public TransactionContextLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 14 | super(header); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UnknownLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogEvent; 4 | 5 | /** 6 | * Unknown_log_event 7 | * 8 | * @author Changyuan.lh 9 | * @version 1.0 10 | */ 11 | public final class UnknownLogEvent extends LogEvent { 12 | 13 | public UnknownLogEvent(LogHeader header){ 14 | super(header); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | 5 | /** 6 | * Log row updates with a before image. The event contain several update rows 7 | * for a table. Note that each event contains only rows for one table. Also note 8 | * that the row data consists of pairs of row data: one row for the old data and 9 | * one row for the new data. 10 | * 11 | * @author Changyuan.lh 12 | * @version 1.0 13 | */ 14 | public final class UpdateRowsLogEvent extends RowsLogEvent { 15 | 16 | public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 17 | super(header, buffer, descriptionEvent, false); 18 | } 19 | 20 | public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent, 21 | boolean partial){ 22 | super(header, buffer, descriptionEvent, partial); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ViewChangeEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.LogEvent; 5 | 6 | /** 7 | * @author agapple 2018年5月7日 下午7:05:39 8 | * @version 1.0.26 9 | * @since mysql 5.7 10 | */ 11 | public class ViewChangeEvent extends LogEvent { 12 | 13 | public ViewChangeEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 14 | super(header); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/WriteRowsLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | 5 | /** 6 | * Log row insertions and updates. The event contain several insert/update rows 7 | * for a table. Note that each event contains only rows for one table. 8 | * 9 | * @author Changyuan.lh 10 | * @version 1.0 11 | */ 12 | public final class WriteRowsLogEvent extends RowsLogEvent { 13 | 14 | public WriteRowsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 15 | super(header, buffer, descriptionEvent); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/XidLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.LogEvent; 5 | 6 | /** 7 | * Logs xid of the transaction-to-be-committed in the 2pc protocol. Has no 8 | * meaning in replication, slaves ignore it. 9 | * 10 | * @author Changyuan.lh 11 | * @version 1.0 12 | */ 13 | public final class XidLogEvent extends LogEvent { 14 | 15 | private final long xid; 16 | 17 | public XidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 18 | super(header); 19 | 20 | /* The Post-Header is empty. The Variable Data part begins immediately. */ 21 | buffer.position(descriptionEvent.commonHeaderLen + descriptionEvent.postHeaderLen[XID_EVENT - 1]); 22 | xid = buffer.getLong64(); // !uint8korr 23 | } 24 | 25 | public final long getXid() { 26 | return xid; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/AnnotateRowsEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event.mariadb; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; 5 | import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent; 6 | import com.taobao.tddl.dbsync.binlog.event.LogHeader; 7 | 8 | /** 9 | * mariadb的ANNOTATE_ROWS_EVENT类型 10 | * 11 | * @author jianghang 2014-1-20 下午2:20:35 12 | * @since 1.0.17 13 | */ 14 | public class AnnotateRowsEvent extends IgnorableLogEvent { 15 | 16 | private String rowsQuery; 17 | 18 | public AnnotateRowsEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 19 | super(header, buffer, descriptionEvent); 20 | 21 | final int commonHeaderLen = descriptionEvent.getCommonHeaderLen(); 22 | final int postHeaderLen = descriptionEvent.getPostHeaderLen()[header.getType() - 1]; 23 | 24 | int offset = commonHeaderLen + postHeaderLen; 25 | int len = buffer.limit() - offset; 26 | rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1); 27 | } 28 | 29 | public String getRowsQuery() { 30 | return rowsQuery; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/BinlogCheckPointLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event.mariadb; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; 5 | import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent; 6 | import com.taobao.tddl.dbsync.binlog.event.LogHeader; 7 | 8 | /** 9 | * mariadb10的BINLOG_CHECKPOINT_EVENT类型 10 | * 11 | * @author jianghang 2014-1-20 下午2:22:04 12 | * @since 1.0.17 13 | */ 14 | public class BinlogCheckPointLogEvent extends IgnorableLogEvent { 15 | 16 | public BinlogCheckPointLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 17 | super(header, buffer, descriptionEvent); 18 | // do nothing , just mariadb binlog checkpoint 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidListLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event.mariadb; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; 5 | import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent; 6 | import com.taobao.tddl.dbsync.binlog.event.LogHeader; 7 | 8 | /** 9 | * mariadb的GTID_LIST_EVENT类型 10 | * 11 | * @author jianghang 2014-1-20 下午4:51:50 12 | * @since 1.0.17 13 | */ 14 | public class MariaGtidListLogEvent extends IgnorableLogEvent { 15 | 16 | public MariaGtidListLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 17 | super(header, buffer, descriptionEvent); 18 | // do nothing , just ignore log event 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event.mariadb; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; 5 | import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent; 6 | import com.taobao.tddl.dbsync.binlog.event.LogHeader; 7 | 8 | /** 9 | * mariadb的GTID_EVENT类型 10 | * 11 | * @author jianghang 2014-1-20 下午4:49:10 12 | * @since 1.0.17 13 | */ 14 | public class MariaGtidLogEvent extends IgnorableLogEvent { 15 | 16 | private long gtid; 17 | 18 | /** 19 | * 
20 |      * mariadb gtidlog event format
21 |      *     uint<8> GTID sequence
22 |      *     uint<4> Replication Domain ID
23 |      *     uint<1> Flags
24 |      * 
25 |      * 	if flag & FL_GROUP_COMMIT_ID
26 |      * 	    uint<8> commit_id
27 |      * 	else
28 |      * 	    uint<6> 0
29 |      *
30 | */ 31 | 32 | public MariaGtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 33 | super(header, buffer, descriptionEvent); 34 | gtid = buffer.getUlong64().longValue(); 35 | // do nothing , just ignore log event 36 | } 37 | 38 | public long getGtid() { 39 | return gtid; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/StartEncryptionLogEvent.java: -------------------------------------------------------------------------------- 1 | package com.taobao.tddl.dbsync.binlog.event.mariadb; 2 | 3 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 4 | import com.taobao.tddl.dbsync.binlog.LogEvent; 5 | import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; 6 | import com.taobao.tddl.dbsync.binlog.event.LogHeader; 7 | 8 | /** 9 | * mariadb的Start_encryption_log_event 10 | * 11 | * @author agapple 2018年5月7日 下午7:23:02 12 | * @version 1.0.26 13 | */ 14 | public class StartEncryptionLogEvent extends LogEvent { 15 | 16 | public StartEncryptionLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ 17 | super(header); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /dbsync/src/test/resources/binlog/mysql-bin.000001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/dbsync/src/test/resources/binlog/mysql-bin.000001 -------------------------------------------------------------------------------- /dbsync/src/test/resources/dummy.txt: -------------------------------------------------------------------------------- 1 | 本文件仅仅为定位绝对路径使用 -------------------------------------------------------------------------------- /deployer/src/main/bin/restart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sh stop.sh 4 | 5 | sh startup.sh 6 | -------------------------------------------------------------------------------- /deployer/src/main/bin/startup.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | @if not "%ECHO%" == "" echo %ECHO% 3 | @if "%OS%" == "Windows_NT" setlocal 4 | 5 | set ENV_PATH=.\ 6 | if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0% 7 | 8 | set conf_dir=%ENV_PATH%\..\conf 9 | set canal_conf=%conf_dir%\canal.properties 10 | set logback_configurationFile=%conf_dir%\logback.xml 11 | 12 | set CLASSPATH=%conf_dir% 13 | set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% 14 | 15 | set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m 16 | set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 17 | set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n 18 | set CANAL_OPTS= -DappName=otter-canal -Dlogback.configurationFile="%logback_configurationFile%" -Dcanal.conf="%canal_conf%" 19 | 20 | set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS% 21 | 22 | set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.deployer.CanalLauncher 23 | echo start cmd : %CMD_STR% 24 | 25 | java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.deployer.CanalLauncher -------------------------------------------------------------------------------- /deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceAction.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.deployer.monitor; 2 | 3 | /** 4 | * config配置变化 5 | * 6 | * @author jianghang 2013-2-18 下午01:19:29 7 | * @version 1.0.1 8 | */ 9 | public interface InstanceAction { 10 | 11 | /** 12 | * 启动destination 13 | */ 14 | void start(String destination); 15 | 16 | /** 17 | * 停止destination 18 | */ 19 | void stop(String destination); 20 | 21 | /** 22 | * 重载destination，可能需要stop,start操作，或者只是更新下内存配置 23 | */ 24 | void reload(String destination); 25 | } 26 | -------------------------------------------------------------------------------- /deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceConfigMonitor.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.deployer.monitor; 2 | 3 | import com.alibaba.otter.canal.common.CanalLifeCycle; 4 | 5 | /** 6 | * 监听instance file的文件变化，触发instance start/stop等操作 7 | * 8 | * @author jianghang 2013-2-6 下午06:19:56 9 | * @version 1.0.1 10 | */ 11 | public interface InstanceConfigMonitor extends CanalLifeCycle { 12 | 13 | void register(String destination, InstanceAction action); 14 | 15 | void unregister(String destination); 16 | } 17 | -------------------------------------------------------------------------------- /deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerInstanceConfigMonitor.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.deployer.monitor; 2 | 3 | import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; 4 | import com.alibaba.otter.canal.common.CanalLifeCycle; 5 | 6 | /** 7 | * @author jianghang 2013-2-18 下午03:19:06 8 | * @version 1.0.1 9 | */ 10 | public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle { 11 | 12 | public void register(String destination, InstanceAction action) { 13 | 14 | } 15 | 16 | public void unregister(String destination) { 17 | 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /deployer/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM canal/osbase:v1 2 | 3 | MAINTAINER agapple (jianghang115@gmail.com) 4 | 5 | # install canal 6 | COPY image/ /tmp/docker/ 7 | COPY canal.deployer-*.tar.gz /home/admin/ 8 | 9 | RUN \ 10 | cp -R /tmp/docker/alidata /alidata && \ 11 | chmod +x /alidata/bin/* && \ 12 | mkdir -p /home/admin && \ 13 | cp -R /tmp/docker/admin/* /home/admin/ && \ 14 | /bin/cp -f alidata/bin/lark-wait /usr/bin/lark-wait && \ 15 | 16 | mkdir -p /home/admin/canal-server && \ 17 | tar -xzvf /home/admin/canal.deployer-*.tar.gz -C /home/admin/canal-server && \ 18 | /bin/rm -f /home/admin/canal.deployer-*.tar.gz && \ 19 | 20 | mkdir -p home/admin/canal-server/logs && \ 21 | chmod +x /home/admin/*.sh && \ 22 | chmod +x /home/admin/bin/*.sh && \ 23 | chown admin: -R /home/admin && \ 24 | yum clean all && \ 25 | true 26 | 27 | # 2222 sys , 8000 debug , 11111 canal , 11112 metrics 28 | EXPOSE 2222 11111 8000 11112 29 | 30 | WORKDIR /home/admin 31 | 32 | ENTRYPOINT [ "/alidata/bin/main.sh" ] 33 | CMD [ "/home/admin/app.sh" ] 34 | -------------------------------------------------------------------------------- /docker/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | current_path=`pwd` 4 | case "`uname`" in 5 | Darwin) 6 | bin_abs_path=`cd $(dirname $0); pwd` 7 | ;; 8 | Linux) 9 | bin_abs_path=$(readlink -f $(dirname $0)) 10 | ;; 11 | *) 12 | bin_abs_path=`cd $(dirname $0); pwd` 13 | ;; 14 | esac 15 | BASE=${bin_abs_path} 16 | 17 | if [ "$1" == "base" ] ; then 18 | docker build --no-cache -t canal/osbase $BASE/base 19 | else 20 | rm -rf $BASE/canal.*.tar.gz ; 21 | cd $BASE/../ && mvn clean package -Dmaven.test.skip -Denv=release && cd $current_path ; 22 | cp $BASE/../target/canal.deployer-*.tar.gz $BASE/ 23 | docker build --no-cache -t canal/canal-server $BASE/ 24 | fi 25 | -------------------------------------------------------------------------------- /docker/image/admin/bin/clean_log: -------------------------------------------------------------------------------- 1 | # cron clean log once per minute 2 | */2 * * * * admin /home/admin/bin/clean_log.sh >>/tmp/clean_log.log 2>&1 3 | -------------------------------------------------------------------------------- /docker/image/admin/bin/clean_log.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Global Settings 4 | PATH="$HOME/bin:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/usr/X11R6/bin:/root/bin" 5 | export PATH 6 | 7 | CUTOFF="85" 8 | #获取磁盘使用率最高的分区 9 | USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) 10 | before=$USAGE 11 | 12 | baseClean(){ 13 | #删除tmp目录15天前的文件。 14 | #更新文档时间戳 15 | if [ -d /tmp/hsperfdata_admin ] 16 | then 17 | touch /tmp/hsperfdata_admin 18 | touch /tmp/hsperfdata_admin/* 19 | fi 20 | 21 | find /tmp/ -type f -mtime +15 | xargs -t rm -rf >/dev/null 2>&1 22 | 23 | 24 | now=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) 25 | echo "before:$before; now:$now" 26 | } 27 | 28 | CANAL_DIR="/home/admin/canal-server/logs" 29 | if [[ -d $CANAL_DIR ]]; then 30 | USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) 31 | if [[ $USAGE -ge 90 ]]; then 32 | find $CANAL_DIR -type f -mtime +7 | xargs rm -rf {} 33 | fi 34 | USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) 35 | if [[ $USAGE -ge 80 ]]; then 36 | find $CANAL_DIR -type f -mtime +3 | xargs rm -rf {} 37 | fi 38 | USAGE=$(df -h|awk 'NR>1 {gsub(/%$/,"",$5);print $5 }'|sort -nr|head -1) 39 | if [[ $USAGE -ge 80 ]]; then 40 | find $CANAL_DIR -type d -empty -mtime +3 | grep -v canal | xargs rm -rf {} 41 | find $CANAL_DIR -type f -iname '*.tmp' | xargs rm -rf {} 42 | fi 43 | baseClean 44 | exit 0 45 | fi -------------------------------------------------------------------------------- /docker/image/admin/health.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | metrics_port=`perl -le 'print $ENV{"canal.metrics.pull.port"}'` 3 | if [ "$metrics_port" == "" ]; then 4 | metrics_port="11112" 5 | fi 6 | 7 | CHECK_URL="http://127.0.0.1:$metrics_port/metrics" 8 | CHECK_POINT="canal" 9 | CHECK_COUNT=`curl -s --connect-timeout 7 --max-time 7 $CHECK_URL | grep -c $CHECK_POINT` 10 | if [ $CHECK_COUNT -eq 0 ]; then 11 | echo "[FAILED]" 12 | status=0 13 | error=1 14 | else 15 | echo "[ OK ]" 16 | status=1 17 | error=0 18 | fi -------------------------------------------------------------------------------- /docker/image/alidata/bin/exec_rc_local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "${SKIP_EXEC_RC_LOCAL}" = "YES" ] ; then 4 | echo "skip /etc/rc.local: SKIP_EXEC_RC_LOCAL=${SKIP_EXEC_RC_LOCAL}" 5 | exit 6 | fi 7 | 8 | if [ "${DOCKER_DEPLOY_TYPE}" = "HOST" ] ; then 9 | echo "skip /etc/rc.local: DOCKER_DEPLOY_TYPE=${DOCKER_DEPLOY_TYPE}" 10 | exit 11 | fi -------------------------------------------------------------------------------- /docker/image/alidata/bin/lark-wait: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | chown admin: -R /home/admin/ 5 | source /alidata/lib/proc.sh 6 | waitterm 7 | -------------------------------------------------------------------------------- /docker/image/alidata/bin/main.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -n "${DOCKER_DEPLOY_TYPE}" ] || DOCKER_DEPLOY_TYPE="VM" 4 | echo "DOCKER_DEPLOY_TYPE=${DOCKER_DEPLOY_TYPE}" 5 | 6 | # run init scripts 7 | for e in $(ls /alidata/init/*) ; do 8 | [ -x "${e}" ] || continue 9 | echo "==> INIT $e" 10 | $e 11 | echo "==> EXIT CODE: $?" 12 | done 13 | 14 | echo "==> INIT DEFAULT" 15 | service sshd start 16 | service crond start 17 | 18 | #echo "check hostname -i: `hostname -i`" 19 | #hti_num=`hostname -i|awk '{print NF}'` 20 | #if [ $hti_num -gt 1 ];then 21 | # echo "hostname -i result error:`hostname -i`" 22 | # exit 120 23 | #fi 24 | 25 | echo "==> INIT DONE" 26 | echo "==> RUN ${*}" 27 | exec "${@}" -------------------------------------------------------------------------------- /docker/image/alidata/init/02init-sshd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # set port 4 | if [ -z "${SSHD_PORT}" ] ; then 5 | SSHD_PORT=22 6 | [ "${DOCKER_DEPLOY_TYPE}" = "HOST" ] && SSHD_PORT=2222 7 | fi 8 | 9 | sed -r -i '/^OPTIONS=/ d' /etc/sysconfig/sshd 10 | echo 'OPTIONS="-p '"${SSHD_PORT}"'"' >> /etc/sysconfig/sshd 11 | 12 | # set admin ssh pulic key 13 | if [ "${USE_ADMIN_PASSAGE}" = "YES" ] ; then 14 | echo "set admin passage" 15 | mkdir -p /home/admin/.ssh 16 | chown admin:admin /home/admin/.ssh 17 | chown admin:admin /home/admin/.ssh/authorized_keys 18 | chmod 644 /home/admin/.ssh/authorized_keys 19 | fi 20 | -------------------------------------------------------------------------------- /docker/image/alidata/lib/proc.sh: -------------------------------------------------------------------------------- 1 | # waitterm 2 | # wait TERM/INT signal. 3 | # see: http://veithen.github.io/2014/11/16/sigterm-propagation.html 4 | waitterm() { 5 | local PID 6 | # any process to block 7 | tail -f /dev/null & 8 | PID="$!" 9 | # setup trap, could do nothing, or just kill the blocker 10 | trap "kill -TERM ${PID}" TERM INT 11 | # wait for signal, ignore wait exit code 12 | wait "${PID}" || true 13 | # clear trap 14 | trap - TERM INT 15 | # wait blocker, ignore blocker exit code 16 | wait "${PID}" 2>/dev/null || true 17 | } 18 | 19 | # waittermpid "${PIDFILE}". 20 | # monitor process by pidfile && wait TERM/INT signal. 21 | # if the process disappeared, return 1, means exit with ERROR. 22 | # if TERM or INT signal received, return 0, means OK to exit. 23 | waittermpid() { 24 | local PIDFILE PID do_run error 25 | PIDFILE="${1?}" 26 | do_run=true 27 | error=0 28 | trap "do_run=false" TERM INT 29 | while "${do_run}" ; do 30 | PID="$(cat "${PIDFILE}")" 31 | if ! ps -p "${PID}" >/dev/null 2>&1 ; then 32 | do_run=false 33 | error=1 34 | else 35 | sleep 1 36 | fi 37 | done 38 | trap - TERM INT 39 | return "${error}" 40 | } 41 | -------------------------------------------------------------------------------- /driver/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.alibaba.otter 5 | canal 6 | 1.1.3-SNAPSHOT 7 | ../pom.xml 8 | 9 | com.alibaba.otter 10 | canal.parse.driver 11 | jar 12 | canal driver module for otter ${project.version} 13 | 14 | 15 | com.alibaba.otter 16 | canal.common 17 | ${project.version} 18 | 19 | 20 | 21 | ch.qos.logback 22 | logback-core 23 | 24 | 25 | ch.qos.logback 26 | logback-classic 27 | 28 | 29 | org.slf4j 30 | jcl-over-slf4j 31 | 32 | 33 | org.slf4j 34 | slf4j-api 35 | 36 | 37 | 38 | junit 39 | junit 40 | test 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/CommandPacket.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.packets; 2 | 3 | import org.apache.commons.lang.builder.ToStringBuilder; 4 | 5 | import com.alibaba.otter.canal.common.utils.CanalToStringStyle; 6 | 7 | public abstract class CommandPacket implements IPacket { 8 | 9 | private byte command; 10 | 11 | // arg 12 | 13 | public void setCommand(byte command) { 14 | this.command = command; 15 | } 16 | 17 | public byte getCommand() { 18 | return command; 19 | } 20 | 21 | public String toString() { 22 | return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/GTIDSet.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.packets; 2 | 3 | import java.io.IOException; 4 | 5 | /** 6 | * Created by hiwjd on 2018/4/23. hiwjd0@gmail.com 7 | */ 8 | public interface GTIDSet { 9 | 10 | /** 11 | * 序列化成字节数组 12 | * 13 | * @return 14 | */ 15 | byte[] encode() throws IOException; 16 | 17 | /** 18 | * 更新当前实例 19 | * 20 | * @param str 21 | * @throws Exception 22 | */ 23 | void update(String str); 24 | } 25 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/IPacket.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.packets; 2 | 3 | import java.io.IOException; 4 | 5 | /** 6 | * Top Abstraction for network packet.
7 | * it exposes 2 behaviors for sub-class implementation which will be used to 8 | * marshal data into bytes before sending and to un-marshal data from data after 9 | * receiving.
10 | * 11 | * @author fujohnwang 12 | * @see 1.0 13 | */ 14 | public interface IPacket { 15 | 16 | /** 17 | * un-marshal raw bytes into {@link IPacket} state for application usage.
18 | * 19 | * @param data, the raw byte data received from networking 20 | */ 21 | void fromBytes(byte[] data) throws IOException; 22 | 23 | /** 24 | * marshal the {@link IPacket} state into raw bytes for sending out to 25 | * network.
26 | * 27 | * @return the bytes that's collected from {@link IPacket} state 28 | */ 29 | byte[] toBytes() throws IOException; 30 | } 31 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/PacketWithHeaderPacket.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.packets; 2 | 3 | import org.apache.commons.lang.builder.ToStringBuilder; 4 | 5 | import com.alibaba.otter.canal.common.utils.CanalToStringStyle; 6 | import com.google.common.base.Preconditions; 7 | 8 | public abstract class PacketWithHeaderPacket implements IPacket { 9 | 10 | protected HeaderPacket header; 11 | 12 | protected PacketWithHeaderPacket(){ 13 | } 14 | 15 | protected PacketWithHeaderPacket(HeaderPacket header){ 16 | setHeader(header); 17 | } 18 | 19 | public void setHeader(HeaderPacket header) { 20 | Preconditions.checkNotNull(header); 21 | this.header = header; 22 | } 23 | 24 | public HeaderPacket getHeader() { 25 | return header; 26 | } 27 | 28 | public String toString() { 29 | return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/QueryCommandPacket.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.packets.client; 2 | 3 | import java.io.ByteArrayOutputStream; 4 | import java.io.IOException; 5 | 6 | import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket; 7 | 8 | public class QueryCommandPacket extends CommandPacket { 9 | 10 | private String queryString; 11 | 12 | public QueryCommandPacket(){ 13 | setCommand((byte) 0x03); 14 | } 15 | 16 | public void fromBytes(byte[] data) throws IOException { 17 | } 18 | 19 | public byte[] toBytes() throws IOException { 20 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 21 | out.write(getCommand()); 22 | out.write(getQueryString().getBytes("UTF-8"));// 链接建立时默认指定编码为UTF-8 23 | return out.toByteArray(); 24 | } 25 | 26 | public void setQueryString(String queryString) { 27 | this.queryString = queryString; 28 | } 29 | 30 | public String getQueryString() { 31 | return queryString; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/QuitCommandPacket.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.packets.client; 2 | 3 | import java.io.ByteArrayOutputStream; 4 | import java.io.IOException; 5 | 6 | import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket; 7 | 8 | /** 9 | * quit cmd 10 | * 11 | * @author agapple 2016年3月1日 下午8:33:02 12 | * @since 1.0.22 13 | */ 14 | public class QuitCommandPacket extends CommandPacket { 15 | 16 | public static final byte[] QUIT = new byte[] { 1, 0, 0, 0, 1 }; 17 | 18 | public QuitCommandPacket(){ 19 | setCommand((byte) 0x01); 20 | } 21 | 22 | @Override 23 | public void fromBytes(byte[] data) throws IOException { 24 | 25 | } 26 | 27 | @Override 28 | public byte[] toBytes() throws IOException { 29 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 30 | out.write(getCommand()); 31 | out.write(QUIT); 32 | return out.toByteArray(); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/DataPacket.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.packets.server; 2 | 3 | import java.io.IOException; 4 | 5 | import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket; 6 | 7 | public class DataPacket extends CommandPacket { 8 | 9 | public void fromBytes(byte[] data) { 10 | 11 | } 12 | 13 | public byte[] toBytes() throws IOException { 14 | return null; 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/Reply323Packet.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.packets.server; 2 | 3 | import java.io.ByteArrayOutputStream; 4 | import java.io.IOException; 5 | 6 | import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket; 7 | import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; 8 | 9 | public class Reply323Packet extends PacketWithHeaderPacket { 10 | 11 | public byte[] seed; 12 | 13 | public void fromBytes(byte[] data) throws IOException { 14 | 15 | } 16 | 17 | public byte[] toBytes() throws IOException { 18 | if (seed == null) { 19 | return new byte[] { (byte) 0 }; 20 | } else { 21 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 22 | ByteHelper.writeNullTerminated(seed, out); 23 | return out.toByteArray(); 24 | } 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetPacket.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.packets.server; 2 | 3 | import java.net.SocketAddress; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | public class ResultSetPacket { 8 | 9 | private SocketAddress sourceAddress; 10 | private List fieldDescriptors = new ArrayList(); 11 | private List fieldValues = new ArrayList(); 12 | 13 | public void setFieldDescriptors(List fieldDescriptors) { 14 | this.fieldDescriptors = fieldDescriptors; 15 | } 16 | 17 | public List getFieldDescriptors() { 18 | return fieldDescriptors; 19 | } 20 | 21 | public void setFieldValues(List fieldValues) { 22 | this.fieldValues = fieldValues; 23 | } 24 | 25 | public List getFieldValues() { 26 | return fieldValues; 27 | } 28 | 29 | public void setSourceAddress(SocketAddress sourceAddress) { 30 | this.sourceAddress = sourceAddress; 31 | } 32 | 33 | public SocketAddress getSourceAddress() { 34 | return sourceAddress; 35 | } 36 | 37 | public String toString() { 38 | return "ResultSetPacket [fieldDescriptors=" + fieldDescriptors + ", fieldValues=" + fieldValues 39 | + ", sourceAddress=" + sourceAddress + "]"; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/RowDataPacket.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.packets.server; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket; 8 | import com.alibaba.otter.canal.parse.driver.mysql.utils.LengthCodedStringReader; 9 | 10 | public class RowDataPacket extends PacketWithHeaderPacket { 11 | 12 | private List columns = new ArrayList(); 13 | 14 | public void fromBytes(byte[] data) throws IOException { 15 | int index = 0; 16 | LengthCodedStringReader reader = new LengthCodedStringReader(null, index); 17 | do { 18 | getColumns().add(reader.readLengthCodedString(data)); 19 | } while (reader.getIndex() < data.length); 20 | } 21 | 22 | public byte[] toBytes() throws IOException { 23 | return null; 24 | } 25 | 26 | public void setColumns(List columns) { 27 | this.columns = columns; 28 | } 29 | 30 | public List getColumns() { 31 | return columns; 32 | } 33 | 34 | public String toString() { 35 | return "RowDataPacket [columns=" + columns + "]"; 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/BioSocketChannelPool.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.socket; 2 | 3 | import java.net.Socket; 4 | import java.net.SocketAddress; 5 | 6 | /** 7 | * @author luoyaogui 实现channel的管理（监听连接、读数据、回收） 2016-12-28 8 | * @author chuanyi 2018-3-3 保留open减少文件变更数量 9 | */ 10 | public abstract class BioSocketChannelPool { 11 | 12 | public static BioSocketChannel open(SocketAddress address) throws Exception { 13 | Socket socket = new Socket(); 14 | socket.setSoTimeout(BioSocketChannel.SO_TIMEOUT); 15 | socket.setTcpNoDelay(true); 16 | socket.setKeepAlive(true); 17 | socket.setReuseAddress(true); 18 | socket.connect(address, BioSocketChannel.DEFAULT_CONNECT_TIMEOUT); 19 | return new BioSocketChannel(socket); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.socket; 2 | 3 | import java.io.IOException; 4 | import java.net.SocketAddress; 5 | 6 | /** 7 | * @author agapple 2018年3月12日 下午10:36:44 8 | * @since 1.0.26 9 | */ 10 | public interface SocketChannel { 11 | 12 | public void write(byte[]... buf) throws IOException; 13 | 14 | public byte[] read(int readSize) throws IOException; 15 | 16 | public byte[] read(int readSize, int timeout) throws IOException; 17 | 18 | public void read(byte[] data, int off, int len, int timeout) throws IOException; 19 | 20 | public boolean isConnected(); 21 | 22 | public SocketAddress getRemoteSocketAddress(); 23 | 24 | public SocketAddress getLocalSocketAddress(); 25 | 26 | public void close(); 27 | } 28 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.socket; 2 | 3 | import java.net.SocketAddress; 4 | 5 | import org.apache.commons.lang.StringUtils; 6 | 7 | /** 8 | * @author agapple 2018年3月12日 下午10:46:22 9 | * @since 1.0.26 10 | */ 11 | public abstract class SocketChannelPool { 12 | 13 | public static SocketChannel open(SocketAddress address) throws Exception { 14 | String type = chooseSocketChannel(); 15 | if ("netty".equalsIgnoreCase(type)) { 16 | return NettySocketChannelPool.open(address); 17 | } else { 18 | return BioSocketChannelPool.open(address); 19 | } 20 | 21 | } 22 | 23 | private static String chooseSocketChannel() { 24 | String socketChannel = System.getenv("canal.socketChannel"); 25 | if (StringUtils.isEmpty(socketChannel)) { 26 | socketChannel = System.getProperty("canal.socketChannel"); 27 | } 28 | 29 | if (StringUtils.isEmpty(socketChannel)) { 30 | socketChannel = "bio"; // bio or netty 31 | } 32 | 33 | return socketChannel; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/LengthCodedStringReader.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.utils; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.commons.lang.ArrayUtils; 6 | 7 | public class LengthCodedStringReader { 8 | 9 | public static final String CODE_PAGE_1252 = "UTF-8"; 10 | 11 | private String encoding; 12 | private int index = 0; // 数组下标 13 | 14 | public LengthCodedStringReader(String encoding, int startIndex){ 15 | this.encoding = encoding; 16 | this.index = startIndex; 17 | } 18 | 19 | public String readLengthCodedString(byte[] data) throws IOException { 20 | byte[] lengthBytes = ByteHelper.readBinaryCodedLengthBytes(data, getIndex()); 21 | long length = ByteHelper.readLengthCodedBinary(data, getIndex()); 22 | setIndex(getIndex() + lengthBytes.length); 23 | if (ByteHelper.NULL_LENGTH == length) { 24 | return null; 25 | } 26 | 27 | try { 28 | return new String(ArrayUtils.subarray(data, getIndex(), (int) (getIndex() + length)), 29 | encoding == null ? CODE_PAGE_1252 : encoding); 30 | } finally { 31 | setIndex((int) (getIndex() + length)); 32 | } 33 | 34 | } 35 | 36 | public void setIndex(int index) { 37 | this.index = index; 38 | } 39 | 40 | public int getIndex() { 41 | return index; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/MSC.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql.utils; 2 | 3 | /** 4 | * MySQL Constants.
5 | * constants that is used in mysql server.
6 | * 7 | * @author fujohnwang 8 | */ 9 | public abstract class MSC { 10 | 11 | public static final int MAX_PACKET_LENGTH = (1 << 24); 12 | public static final int HEADER_PACKET_LENGTH_FIELD_LENGTH = 3; 13 | public static final int HEADER_PACKET_LENGTH_FIELD_OFFSET = 0; 14 | public static final int HEADER_PACKET_LENGTH = 4; 15 | public static final int HEADER_PACKET_NUMBER_FIELD_LENGTH = 1; 16 | 17 | public static final byte NULL_TERMINATED_STRING_DELIMITER = 0x00; 18 | public static final byte DEFAULT_PROTOCOL_VERSION = 0x0a; 19 | 20 | public static final int FIELD_COUNT_FIELD_LENGTH = 1; 21 | 22 | public static final int EVENT_TYPE_OFFSET = 4; 23 | public static final int EVENT_LEN_OFFSET = 9; 24 | 25 | public static final long DEFAULT_BINLOG_FILE_START_POSITION = 4; 26 | } 27 | -------------------------------------------------------------------------------- /driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/UUIDSetTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.driver.mysql; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | 8 | import org.junit.Test; 9 | 10 | import com.alibaba.otter.canal.parse.driver.mysql.packets.UUIDSet; 11 | 12 | /** 13 | * Created by hiwjd on 2018/4/26. hiwjd0@gmail.com 14 | */ 15 | public class UUIDSetTest { 16 | 17 | @Test 18 | public void testToString() { 19 | Map cases = new HashMap(4); 20 | cases.put("726757ad-4455-11e8-ae04-0242ac110002:1", "726757ad-4455-11e8-ae04-0242ac110002:1"); 21 | cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3", "726757ad-4455-11e8-ae04-0242ac110002:1-3"); 22 | cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3:4-6", "726757ad-4455-11e8-ae04-0242ac110002:1-6"); 23 | cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3:5-7", "726757ad-4455-11e8-ae04-0242ac110002:1-3:5-7"); 24 | 25 | for (Map.Entry entry : cases.entrySet()) { 26 | String expected = entry.getValue(); 27 | assertEquals(expected, UUIDSet.parse(entry.getKey()).toString()); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /example/src/main/bin/startup.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | @if not "%ECHO%" == "" echo %ECHO% 3 | @if "%OS%" == "Windows_NT" setlocal 4 | 5 | set ENV_PATH=.\ 6 | if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0% 7 | 8 | set conf_dir=%ENV_PATH%\..\conf 9 | set logback_configurationFile=%conf_dir%\logback.xml 10 | set client_mode=Simple 11 | if "%1%" != "" set client_mode=%1% 12 | 13 | set CLASSPATH=%conf_dir% 14 | set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% 15 | 16 | set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m 17 | set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 18 | set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9199,server=y,suspend=n 19 | set CANAL_OPTS= -DappName=otter-canal-example -Dlogback.configurationFile="%logback_configurationFile%" 20 | 21 | set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS% 22 | 23 | if "%client_mode%" == "Cluster" 24 | java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.ClusterCanalClientTest 25 | else 26 | java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.SimpleCanalClientTest 27 | -------------------------------------------------------------------------------- /example/src/main/bin/stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cygwin=false; 4 | case "`uname`" in 5 | CYGWIN*) 6 | cygwin=true 7 | ;; 8 | esac 9 | 10 | get_pid() { 11 | STR=$1 12 | PID=$2 13 | if $cygwin; then 14 | JAVA_CMD="$JAVA_HOME\bin\java" 15 | JAVA_CMD=`cygpath --path --unix $JAVA_CMD` 16 | JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'` 17 | else 18 | if [ ! -z "$PID" ]; then 19 | JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` 20 | else 21 | JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` 22 | fi 23 | fi 24 | echo $JAVA_PID; 25 | } 26 | 27 | base=`dirname $0`/.. 28 | pidfile=$base/bin/canal.pid 29 | if [ ! -f "$pidfile" ];then 30 | echo "canal is not running. exists" 31 | exit 32 | fi 33 | 34 | pid=`cat $pidfile` 35 | if [ "$pid" == "" ] ; then 36 | pid=`get_pid "appName=otter-canal-example"` 37 | fi 38 | 39 | echo -e "`hostname`: stopping canal $pid ... " 40 | kill $pid 41 | 42 | LOOPS=0 43 | while (true); 44 | do 45 | gpid=`get_pid "appName=otter-canal-example" "$pid"` 46 | if [ "$gpid" == "" ] ; then 47 | echo "Oook! cost:$LOOPS" 48 | `rm $pidfile` 49 | break; 50 | fi 51 | let LOOPS=LOOPS+1 52 | sleep 1 53 | done -------------------------------------------------------------------------------- /example/src/main/java/com/alibaba/otter/canal/example/db/MysqlLoadLauncher.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.example.db; 2 | 3 | import com.alibaba.otter.canal.example.db.mysql.MysqlClient; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | public class MysqlLoadLauncher { 8 | private static final Logger logger = LoggerFactory.getLogger(MysqlLoadLauncher.class); 9 | 10 | public static void main(String[] args) { 11 | try { 12 | logger.info("## start the canal mysql client."); 13 | final MysqlClient client = ServiceLocator.getMysqlClient(); 14 | logger.info("## the canal consumer is running now ......"); 15 | client.start(); 16 | Runtime.getRuntime().addShutdownHook(new Thread() { 17 | 18 | public void run() { 19 | try { 20 | logger.info("## stop the canal consumer"); 21 | client.stop(); 22 | } catch (Throwable e) { 23 | logger.warn("##something goes wrong when stopping canal consumer:\n{}", e); 24 | } finally { 25 | logger.info("## canal consumer is down."); 26 | } 27 | } 28 | 29 | }); 30 | } catch (Throwable e) { 31 | logger.error("## Something goes wrong when starting up the canal consumer:\n{}", e); 32 | System.exit(0); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /example/src/main/java/com/alibaba/otter/canal/example/db/ServiceLocator.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.example.db; 2 | 3 | import com.alibaba.otter.canal.example.db.mysql.MysqlClient; 4 | import org.springframework.beans.factory.DisposableBean; 5 | import org.springframework.context.ApplicationContext; 6 | import org.springframework.context.support.ClassPathXmlApplicationContext; 7 | import org.springframework.util.Assert; 8 | 9 | public class ServiceLocator implements DisposableBean { 10 | 11 | private static ApplicationContext applicationContext = null; 12 | 13 | static { 14 | try { 15 | applicationContext = new ClassPathXmlApplicationContext("classpath:client-spring.xml"); 16 | } catch (RuntimeException e) { 17 | throw e; 18 | } 19 | } 20 | 21 | private static T getBean(String name) { 22 | assertContextInjected(); 23 | return (T) applicationContext.getBean(name); 24 | } 25 | 26 | 27 | private static void clearHolder() { 28 | ServiceLocator.applicationContext = null; 29 | } 30 | 31 | @Override 32 | public void destroy() throws Exception { 33 | ServiceLocator.clearHolder(); 34 | } 35 | 36 | private static void assertContextInjected() { 37 | Assert.state(applicationContext != null, "ApplicationContext not set"); 38 | } 39 | 40 | 41 | public static MysqlClient getMysqlClient() { 42 | return getBean("mysqlClient"); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /example/src/main/java/com/alibaba/otter/canal/example/db/dialect/DbDialect.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.example.db.dialect; 2 | 3 | import org.apache.ddlutils.model.Table; 4 | import org.springframework.jdbc.core.JdbcTemplate; 5 | import org.springframework.jdbc.support.lob.LobHandler; 6 | import org.springframework.transaction.support.TransactionTemplate; 7 | 8 | public interface DbDialect { 9 | 10 | LobHandler getLobHandler(); 11 | 12 | JdbcTemplate getJdbcTemplate(); 13 | 14 | TransactionTemplate getTransactionTemplate(); 15 | 16 | Table findTable(String schema, String table); 17 | 18 | Table findTable(String schema, String table, boolean useCache); 19 | 20 | } 21 | -------------------------------------------------------------------------------- /example/src/main/java/com/alibaba/otter/canal/example/db/dialect/mysql/MysqlDialect.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2010-2101 Alibaba Group Holding Limited. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alibaba.otter.canal.example.db.dialect.mysql; 18 | 19 | import com.alibaba.otter.canal.example.db.dialect.AbstractDbDialect; 20 | import org.springframework.jdbc.core.JdbcTemplate; 21 | import org.springframework.jdbc.support.lob.LobHandler; 22 | 23 | public class MysqlDialect extends AbstractDbDialect { 24 | 25 | public MysqlDialect(JdbcTemplate jdbcTemplate, LobHandler lobHandler) { 26 | super(jdbcTemplate, lobHandler); 27 | } 28 | 29 | public boolean isEmptyStringNulled() { 30 | return false; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /example/src/main/java/com/alibaba/otter/canal/example/db/mysql/MysqlClient.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.example.db.mysql; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry; 4 | 5 | import java.util.List; 6 | 7 | public class MysqlClient extends AbstractMysqlClient { 8 | 9 | @Override 10 | public void insert(CanalEntry.Header header, List afterColumns) { 11 | execute(header, afterColumns); 12 | } 13 | 14 | @Override 15 | public void update(CanalEntry.Header header, List afterColumns) { 16 | execute(header, afterColumns); 17 | } 18 | 19 | @Override 20 | public void delete(CanalEntry.Header header, List beforeColumns) { 21 | execute(header, beforeColumns); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /example/src/main/resources/client.properties: -------------------------------------------------------------------------------- 1 | # client 配置 2 | zk.servers=127.0.0.1:2181 3 | # 5 * 1024 4 | client.batch.size=5120 5 | client.debug=false 6 | client.destination=example 7 | client.username=canal 8 | client.password=canal 9 | client.exceptionstrategy=1 10 | client.retrytimes=3 11 | client.filter=.*\\..* 12 | 13 | # 同步目标: mysql 配置 14 | target.mysql.url=jdbc:mysql://127.0.0.1:4306 15 | target.mysql.username=root 16 | target.mysql.password=123456 17 | -------------------------------------------------------------------------------- /example/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | %msg 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /filter/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.alibaba.otter 5 | canal 6 | 1.1.3-SNAPSHOT 7 | ../pom.xml 8 | 9 | com.alibaba.otter 10 | canal.filter 11 | jar 12 | canal sink module for otter ${project.version} 13 | 14 | 15 | com.alibaba.otter 16 | canal.common 17 | ${project.version} 18 | 19 | 20 | com.alibaba.otter 21 | canal.protocol 22 | ${project.version} 23 | 24 | 25 | com.googlecode.aviator 26 | aviator 27 | 28 | 29 | oro 30 | oro 31 | 32 | 33 | 34 | junit 35 | junit 36 | test 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /filter/src/main/java/com/alibaba/otter/canal/filter/CanalEventFilter.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.filter; 2 | 3 | import com.alibaba.otter.canal.filter.exception.CanalFilterException; 4 | 5 | /** 6 | * 数据过滤机制 7 | * 8 | * @author jianghang 2012-7-20 下午03:51:27 9 | */ 10 | public interface CanalEventFilter { 11 | 12 | boolean filter(T event) throws CanalFilterException; 13 | } 14 | -------------------------------------------------------------------------------- /filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterELFilter.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.filter.aviater; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.commons.lang.StringUtils; 7 | 8 | import com.alibaba.otter.canal.filter.CanalEventFilter; 9 | import com.alibaba.otter.canal.filter.exception.CanalFilterException; 10 | import com.alibaba.otter.canal.protocol.CanalEntry; 11 | import com.googlecode.aviator.AviatorEvaluator; 12 | 13 | /** 14 | * 基于aviater el表达式的匹配过滤 15 | * 16 | * @author jianghang 2012-7-23 上午10:46:32 17 | */ 18 | public class AviaterELFilter implements CanalEventFilter { 19 | 20 | public static final String ROOT_KEY = "entry"; 21 | private String expression; 22 | 23 | public AviaterELFilter(String expression){ 24 | this.expression = expression; 25 | } 26 | 27 | public boolean filter(CanalEntry.Entry entry) throws CanalFilterException { 28 | if (StringUtils.isEmpty(expression)) { 29 | return true; 30 | } 31 | 32 | Map env = new HashMap(); 33 | env.put(ROOT_KEY, entry); 34 | return (Boolean) AviatorEvaluator.execute(expression, env); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /filter/src/main/java/com/alibaba/otter/canal/filter/aviater/RegexFunction.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.filter.aviater; 2 | 3 | import java.util.Map; 4 | 5 | import org.apache.oro.text.regex.Perl5Matcher; 6 | 7 | import com.alibaba.otter.canal.filter.PatternUtils; 8 | import com.googlecode.aviator.runtime.function.AbstractFunction; 9 | import com.googlecode.aviator.runtime.function.FunctionUtils; 10 | import com.googlecode.aviator.runtime.type.AviatorBoolean; 11 | import com.googlecode.aviator.runtime.type.AviatorObject; 12 | 13 | /** 14 | * 提供aviator regex的代码扩展 15 | * 16 | * @author jianghang 2012-7-23 上午10:29:23 17 | */ 18 | public class RegexFunction extends AbstractFunction { 19 | 20 | public AviatorObject call(Map env, AviatorObject arg1, AviatorObject arg2) { 21 | String pattern = FunctionUtils.getStringValue(arg1, env); 22 | String text = FunctionUtils.getStringValue(arg2, env); 23 | Perl5Matcher matcher = new Perl5Matcher(); 24 | boolean isMatch = matcher.matches(text, PatternUtils.getPattern(pattern)); 25 | return AviatorBoolean.valueOf(isMatch); 26 | } 27 | 28 | public String getName() { 29 | return "regex"; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /filter/src/main/java/com/alibaba/otter/canal/filter/exception/CanalFilterException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.filter.exception; 2 | 3 | import com.alibaba.otter.canal.common.CanalException; 4 | 5 | /** 6 | * canal 异常定义 7 | * 8 | * @author jianghang 2012-6-15 下午04:57:35 9 | * @version 1.0.0 10 | */ 11 | public class CanalFilterException extends CanalException { 12 | 13 | private static final long serialVersionUID = -7288830284122672209L; 14 | 15 | public CanalFilterException(String errorCode){ 16 | super(errorCode); 17 | } 18 | 19 | public CanalFilterException(String errorCode, Throwable cause){ 20 | super(errorCode, cause); 21 | } 22 | 23 | public CanalFilterException(String errorCode, String errorDesc){ 24 | super(errorCode + ":" + errorDesc); 25 | } 26 | 27 | public CanalFilterException(String errorCode, String errorDesc, Throwable cause){ 28 | super(errorCode + ":" + errorDesc, cause); 29 | } 30 | 31 | public CanalFilterException(Throwable cause){ 32 | super(cause); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /images/QPS.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/QPS.PNG -------------------------------------------------------------------------------- /images/TPS.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/TPS.PNG -------------------------------------------------------------------------------- /images/basic.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/basic.PNG -------------------------------------------------------------------------------- /images/clients.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/clients.PNG -------------------------------------------------------------------------------- /images/delay.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/delay.PNG -------------------------------------------------------------------------------- /images/empty.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/empty.PNG -------------------------------------------------------------------------------- /images/idle.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/idle.PNG -------------------------------------------------------------------------------- /images/instance.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/instance.PNG -------------------------------------------------------------------------------- /images/latency.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/latency.PNG -------------------------------------------------------------------------------- /images/network.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/network.PNG -------------------------------------------------------------------------------- /images/overview.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/overview.PNG -------------------------------------------------------------------------------- /images/remain_events.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/remain_events.PNG -------------------------------------------------------------------------------- /images/remain_mem.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/remain_mem.PNG -------------------------------------------------------------------------------- /images/reqs.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/reqs.PNG -------------------------------------------------------------------------------- /images/rows.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/rows.PNG -------------------------------------------------------------------------------- /images/store.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/store.PNG -------------------------------------------------------------------------------- /images/throughput.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/throughput.PNG -------------------------------------------------------------------------------- /images/transactions.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/images/transactions.PNG -------------------------------------------------------------------------------- /instance/core/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.alibaba.otter 5 | canal 6 | 1.1.3-SNAPSHOT 7 | ../../pom.xml 8 | 9 | canal.instance.core 10 | jar 11 | canal instance core module for otter ${project.version} 12 | 13 | 14 | com.alibaba.otter 15 | canal.common 16 | ${project.version} 17 | 18 | 19 | com.alibaba.otter 20 | canal.store 21 | ${project.version} 22 | 23 | 24 | com.alibaba.otter 25 | canal.meta 26 | ${project.version} 27 | 28 | 29 | com.alibaba.otter 30 | canal.parse 31 | ${project.version} 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstance.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.instance.core; 2 | 3 | import com.alibaba.otter.canal.common.CanalLifeCycle; 4 | import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler; 5 | import com.alibaba.otter.canal.meta.CanalMetaManager; 6 | import com.alibaba.otter.canal.parse.CanalEventParser; 7 | import com.alibaba.otter.canal.protocol.ClientIdentity; 8 | import com.alibaba.otter.canal.sink.CanalEventSink; 9 | import com.alibaba.otter.canal.store.CanalEventStore; 10 | 11 | /** 12 | * 代表单个canal实例，比如一个destination会独立一个实例 13 | * 14 | * @author jianghang 2012-7-12 下午12:04:58 15 | * @version 1.0.0 16 | */ 17 | public interface CanalInstance extends CanalLifeCycle { 18 | 19 | String getDestination(); 20 | 21 | CanalEventParser getEventParser(); 22 | 23 | CanalEventSink getEventSink(); 24 | 25 | CanalEventStore getEventStore(); 26 | 27 | CanalMetaManager getMetaManager(); 28 | 29 | CanalAlarmHandler getAlarmHandler(); 30 | 31 | /** 32 | * 客户端发生订阅/取消订阅行为 33 | */ 34 | boolean subscribeChange(ClientIdentity identity); 35 | 36 | CanalMQConfig getMqConfig(); 37 | } 38 | -------------------------------------------------------------------------------- /instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstanceGenerator.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.instance.core; 2 | 3 | /** 4 | * @author zebin.xuzb @ 2012-7-12 5 | * @version 1.0.0 6 | */ 7 | public interface CanalInstanceGenerator { 8 | 9 | /** 10 | * 通过 destination 产生特定的 {@link CanalInstance} 11 | * 12 | * @param destination 13 | * @return 14 | */ 15 | CanalInstance generate(String destination); 16 | } 17 | -------------------------------------------------------------------------------- /instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalMQConfig.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.instance.core; 2 | 3 | public class CanalMQConfig { 4 | 5 | private String topic; 6 | private Integer partition; 7 | private Integer partitionsNum; 8 | private String partitionHash; 9 | 10 | public String getTopic() { 11 | return topic; 12 | } 13 | 14 | public void setTopic(String topic) { 15 | this.topic = topic; 16 | } 17 | 18 | public Integer getPartition() { 19 | return partition; 20 | } 21 | 22 | public void setPartition(Integer partition) { 23 | this.partition = partition; 24 | } 25 | 26 | public Integer getPartitionsNum() { 27 | return partitionsNum; 28 | } 29 | 30 | public void setPartitionsNum(Integer partitionsNum) { 31 | this.partitionsNum = partitionsNum; 32 | } 33 | 34 | public String getPartitionHash() { 35 | return partitionHash; 36 | } 37 | 38 | public void setPartitionHash(String partitionHash) { 39 | this.partitionHash = partitionHash; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /instance/manager/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.alibaba.otter 5 | canal 6 | 1.1.3-SNAPSHOT 7 | ../../pom.xml 8 | 9 | com.alibaba.otter 10 | canal.instance.manager 11 | jar 12 | canal instance manager module for otter ${project.version} 13 | 14 | 15 | com.alibaba.otter 16 | canal.instance.core 17 | ${project.version} 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalConfigClient.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.instance.manager; 2 | 3 | import com.alibaba.otter.canal.instance.manager.model.Canal; 4 | 5 | /** 6 | * 对应canal的配置 7 | * 8 | * @author jianghang 2012-7-4 下午03:09:17 9 | * @version 1.0.0 10 | */ 11 | public class CanalConfigClient { 12 | 13 | /** 14 | * 根据对应的destinantion查询Canal信息 15 | */ 16 | public Canal findCanal(String destination) { 17 | // TODO 根据自己的业务实现 18 | throw new UnsupportedOperationException(); 19 | } 20 | 21 | /** 22 | * 根据对应的destinantion查询filter信息 23 | */ 24 | public String findFilter(String destination) { 25 | // TODO 根据自己的业务实现 26 | throw new UnsupportedOperationException(); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/ManagerCanalInstanceGenerator.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.instance.manager; 2 | 3 | import com.alibaba.otter.canal.instance.core.CanalInstance; 4 | import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator; 5 | import com.alibaba.otter.canal.instance.manager.model.Canal; 6 | 7 | /** 8 | * 基于manager生成对应的{@linkplain CanalInstance} 9 | * 10 | * @author jianghang 2012-7-12 下午05:37:09 11 | * @version 1.0.0 12 | */ 13 | public class ManagerCanalInstanceGenerator implements CanalInstanceGenerator { 14 | 15 | private CanalConfigClient canalConfigClient; 16 | 17 | public CanalInstance generate(String destination) { 18 | Canal canal = canalConfigClient.findCanal(destination); 19 | String filter = canalConfigClient.findFilter(destination); 20 | return new CanalInstanceWithManager(canal, filter); 21 | } 22 | 23 | // ================ setter / getter ================ 24 | 25 | public void setCanalConfigClient(CanalConfigClient canalConfigClient) { 26 | this.canalConfigClient = canalConfigClient; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalStatus.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.instance.manager.model; 2 | 3 | /** 4 | * 运行状态 5 | * 6 | * @author jianghang 2012-7-13 下午12:54:13 7 | * @version 1.0.0 8 | */ 9 | public enum CanalStatus { 10 | /** 启动 */ 11 | START, 12 | /** 停止 */ 13 | STOP; 14 | 15 | public boolean isStart() { 16 | return this.equals(CanalStatus.START); 17 | } 18 | 19 | public boolean isStop() { 20 | return this.equals(CanalStatus.STOP); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /instance/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.alibaba.otter 5 | canal 6 | 1.1.3-SNAPSHOT 7 | ../pom.xml 8 | 9 | com.alibaba.otter 10 | canal.instance 11 | pom 12 | canal instance module for otter ${project.version} 13 | 14 | 15 | core 16 | manager 17 | spring 18 | 19 | 20 | -------------------------------------------------------------------------------- /instance/spring/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.alibaba.otter 5 | canal 6 | 1.1.3-SNAPSHOT 7 | ../../pom.xml 8 | 9 | com.alibaba.otter 10 | canal.instance.spring 11 | jar 12 | canal instance spring module for otter ${project.version} 13 | 14 | 15 | com.alibaba.otter 16 | canal.instance.core 17 | ${project.version} 18 | 19 | 20 | 21 | junit 22 | junit 23 | test 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/SpringCanalInstanceGenerator.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.instance.spring; 2 | 3 | import org.springframework.beans.BeansException; 4 | import org.springframework.beans.factory.BeanFactory; 5 | import org.springframework.beans.factory.BeanFactoryAware; 6 | 7 | import com.alibaba.otter.canal.instance.core.CanalInstance; 8 | import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator; 9 | 10 | /** 11 | * @author zebin.xuzb @ 2012-7-12 12 | * @version 1.0.0 13 | */ 14 | public class SpringCanalInstanceGenerator implements CanalInstanceGenerator, BeanFactoryAware { 15 | 16 | private String defaultName = "instance"; 17 | private BeanFactory beanFactory; 18 | 19 | public CanalInstance generate(String destination) { 20 | String beanName = destination; 21 | if (!beanFactory.containsBean(beanName)) { 22 | beanName = defaultName; 23 | } 24 | 25 | return (CanalInstance) beanFactory.getBean(beanName); 26 | } 27 | 28 | public void setBeanFactory(BeanFactory beanFactory) throws BeansException { 29 | this.beanFactory = beanFactory; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/SocketAddressEditor.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.instance.spring.support; 2 | 3 | import java.beans.PropertyEditorSupport; 4 | import java.net.InetSocketAddress; 5 | 6 | import org.apache.commons.lang.StringUtils; 7 | import org.springframework.beans.PropertyEditorRegistrar; 8 | import org.springframework.beans.PropertyEditorRegistry; 9 | 10 | public class SocketAddressEditor extends PropertyEditorSupport implements PropertyEditorRegistrar { 11 | 12 | public void registerCustomEditors(PropertyEditorRegistry registry) { 13 | registry.registerCustomEditor(InetSocketAddress.class, this); 14 | } 15 | 16 | public void setAsText(String text) throws IllegalArgumentException { 17 | String[] addresses = StringUtils.split(text, ":"); 18 | if (addresses.length > 0) { 19 | if (addresses.length != 2) { 20 | throw new RuntimeException("address[" + text + "] is illegal, eg.127.0.0.1:3306"); 21 | } else { 22 | setValue(new InetSocketAddress(addresses[0], Integer.valueOf(addresses[1]))); 23 | } 24 | } else { 25 | setValue(null); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /instance/spring/src/test/resources/retl/instance.properties: -------------------------------------------------------------------------------- 1 | canal.instance.detecting.enable = true 2 | 3 | ################################################# 4 | ## mysql serverId 5 | canal.instance.mysql.slaveId = 1234 6 | 7 | # position info 8 | canal.instance.master.address = 127.0.0.1:3306 9 | canal.instance.master.journal.name = 10 | canal.instance.master.position = 11 | canal.instance.master.timestamp = 12 | 13 | canal.instance.master1.address = 127.0.0.1:3306 14 | canal.instance.master1.journal.name = 15 | canal.instance.master1.position = 16 | canal.instance.master1.timestamp = 17 | 18 | canal.instance.master2.address = 127.0.0.1:3306 19 | canal.instance.master2.journal.name = 20 | canal.instance.master2.position = 21 | canal.instance.master2.timestamp = 22 | 23 | #canal.instance.standby.address = 24 | #canal.instance.standby.journal.name = 25 | #canal.instance.standby.position = 26 | #canal.instance.standby.timestamp = 27 | 28 | # username/password 29 | canal.instance.dbUsername = xxxxx 30 | canal.instance.dbPassword=cZozNf1mzW6EQLGO2q9u99619xbZLO0fbua3EX08r4BWNXb8lAt1aHrTEOBttd6UY8Vnuc0easlVXZDdLtt8BQ== 31 | canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ== 32 | canal.instance.defaultDatabaseName = 33 | canal.instance.connectionCharset = UTF-8 34 | # enable druid Decrypt database password 35 | canal.instance.enableDruid=true 36 | 37 | # table regex 38 | canal.instance.filter.regex = .*\\..* 39 | 40 | ################################################# -------------------------------------------------------------------------------- /meta/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.alibaba.otter 5 | canal 6 | 1.1.3-SNAPSHOT 7 | ../pom.xml 8 | 9 | com.alibaba.otter 10 | canal.meta 11 | jar 12 | canal meta module for otter ${project.version} 13 | 14 | 15 | com.alibaba.otter 16 | canal.common 17 | ${project.version} 18 | 19 | 20 | com.alibaba.otter 21 | canal.protocol 22 | ${project.version} 23 | 24 | 25 | 26 | junit 27 | junit 28 | test 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /meta/src/main/java/com/alibaba/otter/canal/meta/exception/CanalMetaManagerException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.meta.exception; 2 | 3 | import com.alibaba.otter.canal.common.CanalException; 4 | 5 | /** 6 | * @author zebin.xuzb @ 2012-6-21 7 | * @version 1.0.0 8 | */ 9 | public class CanalMetaManagerException extends CanalException { 10 | 11 | private static final long serialVersionUID = -654893533794556357L; 12 | 13 | public CanalMetaManagerException(String errorCode){ 14 | super(errorCode); 15 | } 16 | 17 | public CanalMetaManagerException(String errorCode, Throwable cause){ 18 | super(errorCode, cause); 19 | } 20 | 21 | public CanalMetaManagerException(String errorCode, String errorDesc){ 22 | super(errorCode + ":" + errorDesc); 23 | } 24 | 25 | public CanalMetaManagerException(String errorCode, String errorDesc, Throwable cause){ 26 | super(errorCode + ":" + errorDesc, cause); 27 | } 28 | 29 | public CanalMetaManagerException(Throwable cause){ 30 | super(cause); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /meta/src/test/java/com/alibaba/otter/canal/meta/AbstractZkTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.meta; 2 | 3 | import org.junit.Assert; 4 | 5 | public class AbstractZkTest { 6 | 7 | protected String destination = "ljhtest1"; 8 | protected String cluster1 = "127.0.0.1:2188"; 9 | protected String cluster2 = "127.0.0.1:2188,127.0.0.1:2188"; 10 | 11 | public void sleep(long time) { 12 | try { 13 | Thread.sleep(time); 14 | } catch (InterruptedException e) { 15 | Assert.fail(e.getMessage()); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /meta/src/test/java/com/alibaba/otter/canal/meta/MemoryMetaManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.meta; 2 | 3 | import java.util.Map; 4 | 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import com.alibaba.otter.canal.protocol.position.PositionRange; 9 | 10 | public class MemoryMetaManagerTest extends AbstractMetaManagerTest { 11 | 12 | @Test 13 | public void testSubscribeAll() { 14 | MemoryMetaManager metaManager = new MemoryMetaManager(); 15 | metaManager.start(); 16 | doSubscribeTest(metaManager); 17 | metaManager.stop(); 18 | } 19 | 20 | @Test 21 | public void testBatchAll() { 22 | MemoryMetaManager metaManager = new MemoryMetaManager(); 23 | metaManager.start(); 24 | doBatchTest(metaManager); 25 | 26 | metaManager.clearAllBatchs(clientIdentity); 27 | Map ranges = metaManager.listAllBatchs(clientIdentity); 28 | Assert.assertEquals(0, ranges.size()); 29 | metaManager.stop(); 30 | } 31 | 32 | @Test 33 | public void testCursorAll() { 34 | MemoryMetaManager metaManager = new MemoryMetaManager(); 35 | metaManager.start(); 36 | doCursorTest(metaManager); 37 | metaManager.stop(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/CanalEventParser.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse; 2 | 3 | import com.alibaba.otter.canal.common.CanalLifeCycle; 4 | 5 | /** 6 | * 数据复制控制器 7 | * 8 | * @author jianghang 2012-6-21 下午04:03:25 9 | * @version 1.0.0 10 | */ 11 | public interface CanalEventParser extends CanalLifeCycle { 12 | 13 | } 14 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/CanalHASwitchable.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse; 2 | 3 | import com.alibaba.otter.canal.parse.support.AuthenticationInfo; 4 | 5 | /** 6 | * 支持可切换的数据复制控制器 7 | * 8 | * @author jianghang 2012-6-26 下午05:41:43 9 | * @version 1.0.0 10 | */ 11 | public interface CanalHASwitchable { 12 | 13 | public void doSwitch(); 14 | 15 | public void doSwitch(AuthenticationInfo newAuthenticationInfo); 16 | } 17 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalHAException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.exception; 2 | 3 | import com.alibaba.otter.canal.common.CanalException; 4 | 5 | /** 6 | * canal 异常定义 7 | * 8 | * @author jianghang 2012-6-15 下午04:57:35 9 | * @version 1.0.0 10 | */ 11 | public class CanalHAException extends CanalException { 12 | 13 | private static final long serialVersionUID = -7288830284122672209L; 14 | 15 | public CanalHAException(String errorCode){ 16 | super(errorCode); 17 | } 18 | 19 | public CanalHAException(String errorCode, Throwable cause){ 20 | super(errorCode, cause); 21 | } 22 | 23 | public CanalHAException(String errorCode, String errorDesc){ 24 | super(errorCode + ":" + errorDesc); 25 | } 26 | 27 | public CanalHAException(String errorCode, String errorDesc, Throwable cause){ 28 | super(errorCode + ":" + errorDesc, cause); 29 | } 30 | 31 | public CanalHAException(Throwable cause){ 32 | super(cause); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalParseException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.exception; 2 | 3 | import com.alibaba.otter.canal.common.CanalException; 4 | 5 | /** 6 | * canal 异常定义 7 | * 8 | * @author jianghang 2012-6-15 下午04:57:35 9 | * @version 1.0.0 10 | */ 11 | public class CanalParseException extends CanalException { 12 | 13 | private static final long serialVersionUID = -7288830284122672209L; 14 | 15 | public CanalParseException(String errorCode){ 16 | super(errorCode); 17 | } 18 | 19 | public CanalParseException(String errorCode, Throwable cause){ 20 | super(errorCode, cause); 21 | } 22 | 23 | public CanalParseException(String errorCode, String errorDesc){ 24 | super(errorCode + ":" + errorDesc); 25 | } 26 | 27 | public CanalParseException(String errorCode, String errorDesc, Throwable cause){ 28 | super(errorCode + ":" + errorDesc, cause); 29 | } 30 | 31 | public CanalParseException(Throwable cause){ 32 | super(cause); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/exception/PositionNotFoundException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.exception; 2 | 3 | /** 4 | * @author chengjin.lyf on 2018/7/20 下午2:54 5 | * @since 1.0.25 6 | */ 7 | public class PositionNotFoundException extends CanalParseException { 8 | 9 | private static final long serialVersionUID = -7382448928116244017L; 10 | 11 | public PositionNotFoundException(String errorCode){ 12 | super(errorCode); 13 | } 14 | 15 | public PositionNotFoundException(String errorCode, Throwable cause){ 16 | super(errorCode, cause); 17 | } 18 | 19 | public PositionNotFoundException(String errorCode, String errorDesc){ 20 | super(errorCode, errorDesc); 21 | } 22 | 23 | public PositionNotFoundException(String errorCode, String errorDesc, Throwable cause){ 24 | super(errorCode, errorDesc, cause); 25 | } 26 | 27 | public PositionNotFoundException(Throwable cause){ 28 | super(cause); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/exception/ServerIdNotMatchException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.exception; 2 | 3 | import com.alibaba.otter.canal.common.CanalException; 4 | 5 | /** 6 | * @author chengjin.lyf on 2018/8/8 下午1:07 7 | * @since 1.0.25 8 | */ 9 | public class ServerIdNotMatchException extends CanalException { 10 | 11 | private static final long serialVersionUID = -6124989280379293953L; 12 | 13 | public ServerIdNotMatchException(String errorCode){ 14 | super(errorCode); 15 | } 16 | 17 | public ServerIdNotMatchException(String errorCode, Throwable cause){ 18 | super(errorCode, cause); 19 | } 20 | 21 | public ServerIdNotMatchException(String errorCode, String errorDesc){ 22 | super(errorCode, errorDesc); 23 | } 24 | 25 | public ServerIdNotMatchException(String errorCode, String errorDesc, Throwable cause){ 26 | super(errorCode, errorDesc, cause); 27 | } 28 | 29 | public ServerIdNotMatchException(Throwable cause){ 30 | super(cause); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/exception/TableIdNotFoundException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.exception; 2 | 3 | import com.alibaba.otter.canal.common.CanalException; 4 | 5 | public class TableIdNotFoundException extends CanalException { 6 | 7 | private static final long serialVersionUID = -7288830284122672209L; 8 | 9 | public TableIdNotFoundException(String errorCode){ 10 | super(errorCode); 11 | } 12 | 13 | public TableIdNotFoundException(String errorCode, Throwable cause){ 14 | super(errorCode, cause); 15 | } 16 | 17 | public TableIdNotFoundException(String errorCode, String errorDesc){ 18 | super(errorCode + ":" + errorDesc); 19 | } 20 | 21 | public TableIdNotFoundException(String errorCode, String errorDesc, Throwable cause){ 22 | super(errorCode + ":" + errorDesc, cause); 23 | } 24 | 25 | public TableIdNotFoundException(Throwable cause){ 26 | super(cause); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/ha/CanalHAController.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.ha; 2 | 3 | import com.alibaba.otter.canal.common.CanalLifeCycle; 4 | import com.alibaba.otter.canal.parse.exception.CanalHAException; 5 | 6 | /** 7 | * HA 控制器实现 8 | * 9 | * @author jianghang 2012-6-26 下午05:21:07 10 | * @version 1.0.0 11 | */ 12 | public interface CanalHAController extends CanalLifeCycle { 13 | 14 | public void start() throws CanalHAException; 15 | 16 | public void stop() throws CanalHAException; 17 | } 18 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractBinlogParser.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound; 2 | 3 | import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; 4 | import com.alibaba.otter.canal.parse.exception.CanalParseException; 5 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 6 | 7 | public abstract class AbstractBinlogParser extends AbstractCanalLifeCycle implements BinlogParser { 8 | 9 | public void reset() { 10 | } 11 | 12 | public Entry parse(T event, TableMeta tableMeta) throws CanalParseException { 13 | return null; 14 | } 15 | 16 | public Entry parse(T event) throws CanalParseException { 17 | return null; 18 | } 19 | 20 | public void stop() { 21 | reset(); 22 | super.stop(); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound; 2 | 3 | import com.alibaba.otter.canal.common.CanalLifeCycle; 4 | import com.alibaba.otter.canal.parse.exception.CanalParseException; 5 | import com.alibaba.otter.canal.protocol.CanalEntry; 6 | 7 | /** 8 | * 解析binlog的接口 9 | * 10 | * @author: yuanzu Date: 12-9-20 Time: 下午8:46 11 | */ 12 | public interface BinlogParser extends CanalLifeCycle { 13 | 14 | CanalEntry.Entry parse(T event, boolean isSeek) throws CanalParseException; 15 | 16 | void reset(); 17 | } 18 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound; 2 | 3 | import java.io.IOException; 4 | 5 | import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet; 6 | 7 | /** 8 | * 通用的Erosa的链接接口, 用于一般化处理mysql/oracle的解析过程 9 | * 10 | * @author: yuanzu Date: 12-9-20 Time: 下午2:47 11 | */ 12 | public interface ErosaConnection { 13 | 14 | public void connect() throws IOException; 15 | 16 | public void reconnect() throws IOException; 17 | 18 | public void disconnect() throws IOException; 19 | 20 | /** 21 | * 用于快速数据查找,和dump的区别在于，seek会只给出部分的数据 22 | */ 23 | public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException; 24 | 25 | public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException; 26 | 27 | public void dump(long timestamp, SinkFunction func) throws IOException; 28 | 29 | /** 30 | * 通过GTID同步binlog 31 | */ 32 | public void dump(GTIDSet gtidSet, SinkFunction func) throws IOException; 33 | 34 | // ------------- 35 | 36 | public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException; 37 | 38 | public void dump(long timestamp, MultiStageCoprocessor coprocessor) throws IOException; 39 | 40 | public void dump(GTIDSet gtidSet, MultiStageCoprocessor coprocessor) throws IOException; 41 | 42 | ErosaConnection fork(); 43 | 44 | public long queryServerId() throws IOException; 45 | } 46 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/HeartBeatCallback.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound; 2 | 3 | /** 4 | * 提供mysql heartBeat心跳数据的callback机制 5 | * 6 | * @author jianghang 2012-6-26 下午04:49:56 7 | * @version 1.0.0 8 | */ 9 | public interface HeartBeatCallback { 10 | 11 | /** 12 | * 心跳发送成功 13 | */ 14 | public void onSuccess(long costTime); 15 | 16 | /** 17 | * 心跳发送失败 18 | */ 19 | public void onFailed(Throwable e); 20 | 21 | } 22 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/MultiStageCoprocessor.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound; 2 | 3 | import com.alibaba.otter.canal.common.CanalLifeCycle; 4 | import com.taobao.tddl.dbsync.binlog.LogBuffer; 5 | import com.taobao.tddl.dbsync.binlog.LogEvent; 6 | 7 | /** 8 | * 针对解析器提供一个多阶段协同的处理 9 | * 10 | * 
11 |  * 1. 网络接收 (单线程)
12 |  * 2. 事件基本解析 (单线程，事件类型、DDL解析构造TableMeta、维护位点信息)
13 |  * 3. 事件深度解析 (多线程, DML事件数据的完整解析)
14 |  * 4. 投递到store (单线程)
15 |  *
16 | * 17 | * @author agapple 2018年7月3日 下午4:54:17 18 | * @since 1.0.26 19 | */ 20 | public interface MultiStageCoprocessor extends CanalLifeCycle { 21 | 22 | /** 23 | * 网络数据投递 24 | */ 25 | public boolean publish(LogBuffer buffer); 26 | 27 | public boolean publish(LogEvent event); 28 | } 29 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ParserExceptionHandler.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound; 2 | 3 | /** 4 | * @author chengjin.lyf on 2018/7/20 下午3:55 5 | * @since 1.0.25 6 | */ 7 | public interface ParserExceptionHandler { 8 | 9 | void handle(Throwable e); 10 | } 11 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/SinkFunction.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound; 2 | 3 | /** 4 | * receive parsed bytes , 用于处理要解析的数据块 5 | * 6 | * @author: yuanzu Date: 12-9-20 Time: 下午2:50 7 | */ 8 | 9 | public interface SinkFunction { 10 | 11 | public boolean sink(EVENT event); 12 | } 13 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/DbsyncMysqlEventParser.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound.mysql; 2 | 3 | public class DbsyncMysqlEventParser { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/SlaveEntryPosition.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound.mysql; 2 | 3 | import com.alibaba.otter.canal.protocol.position.EntryPosition; 4 | 5 | /** 6 | * slave status状态的信息 7 | * 8 | * @author jianghang 2013-1-23 下午09:42:18 9 | * @version 1.0.0 10 | */ 11 | public class SlaveEntryPosition extends EntryPosition { 12 | 13 | private static final long serialVersionUID = 5271424551446372093L; 14 | private final String masterHost; 15 | private final String masterPort; 16 | 17 | public SlaveEntryPosition(String fileName, long position, String masterHost, String masterPort){ 18 | super(fileName, position); 19 | 20 | this.masterHost = masterHost; 21 | this.masterPort = masterPort; 22 | } 23 | 24 | public String getMasterHost() { 25 | return masterHost; 26 | } 27 | 28 | public String getMasterPort() { 29 | return masterPort; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/data/RdsItem.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound.mysql.rds.data; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * @author chengjin.lyf on 2018/8/7 下午2:26 7 | * @since 1.0.25 8 | */ 9 | public class RdsItem { 10 | 11 | private List BinLogFile; 12 | 13 | public List getBinLogFile() { 14 | return BinLogFile; 15 | } 16 | 17 | public void setBinLogFile(List binLogFile) { 18 | BinLogFile = binLogFile; 19 | } 20 | 21 | @Override 22 | public String toString() { 23 | return "RdsItem [BinLogFile=" + BinLogFile + "]"; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DefaultTableMetaTSDBFactory.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; 2 | 3 | /** 4 | * @author agapple 2017年10月11日 下午8:45:40 5 | * @since 1.0.25 6 | */ 7 | public class DefaultTableMetaTSDBFactory implements TableMetaTSDBFactory { 8 | 9 | /** 10 | * 代理一下tableMetaTSDB的获取,使用隔离的spring定义 11 | */ 12 | public TableMetaTSDB build(String destination, String springXml) { 13 | return TableMetaTSDBBuilder.build(destination, springXml); 14 | } 15 | 16 | public void destory(String destination) { 17 | TableMetaTSDBBuilder.destory(destination); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; 2 | 3 | import java.util.Map; 4 | 5 | import com.alibaba.otter.canal.parse.inbound.TableMeta; 6 | import com.alibaba.otter.canal.protocol.position.EntryPosition; 7 | 8 | /** 9 | * 表结构的时间序列存储 10 | * 11 | * @author agapple 2017年7月27日 下午4:06:30 12 | * @since 1.0.25 13 | */ 14 | public interface TableMetaTSDB { 15 | 16 | /** 17 | * 初始化 18 | */ 19 | public boolean init(String destination); 20 | 21 | /** 22 | * 销毁资源 23 | */ 24 | public void destory(); 25 | 26 | /** 27 | * 获取当前的表结构 28 | */ 29 | public TableMeta find(String schema, String table); 30 | 31 | /** 32 | * 添加ddl到时间序列库中 33 | */ 34 | public boolean apply(EntryPosition position, String schema, String ddl, String extra); 35 | 36 | /** 37 | * 回滚到指定位点的表结构 38 | */ 39 | public boolean rollback(EntryPosition position); 40 | 41 | /** 42 | * 生成快照内容 43 | */ 44 | public Map snapshot(); 45 | 46 | } 47 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBFactory.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; 2 | 3 | /** 4 | * tableMeta构造器,允许重载实现 5 | * 6 | * @author agapple 2018年8月8日 上午11:01:08 7 | * @since 1.0.26 8 | */ 9 | 10 | public interface TableMetaTSDBFactory { 11 | 12 | /** 13 | * 代理一下tableMetaTSDB的获取,使用隔离的spring定义 14 | */ 15 | public TableMetaTSDB build(String destination, String springXml); 16 | 17 | public void destory(String destination); 18 | } 19 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/index/AbstractLogPositionManager.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.index; 2 | 3 | import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; 4 | 5 | /** 6 | * Created by yinxiu on 17/3/17. Email: marklin.hz@gmail.com 7 | */ 8 | public abstract class AbstractLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager { 9 | } 10 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/index/CanalLogPositionManager.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.index; 2 | 3 | import com.alibaba.otter.canal.common.CanalLifeCycle; 4 | import com.alibaba.otter.canal.parse.exception.CanalParseException; 5 | import com.alibaba.otter.canal.protocol.position.LogPosition; 6 | 7 | /** 8 | * Created by yinxiu on 17/3/17. Email: marklin.hz@gmail.com 9 | */ 10 | public interface CanalLogPositionManager extends CanalLifeCycle { 11 | 12 | LogPosition getLatestIndexBy(String destination); 13 | 14 | void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException; 15 | 16 | } 17 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManager.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.index; 2 | 3 | import java.util.Map; 4 | import java.util.Set; 5 | 6 | import com.alibaba.otter.canal.parse.exception.CanalParseException; 7 | import com.alibaba.otter.canal.protocol.position.LogPosition; 8 | import com.google.common.collect.MapMaker; 9 | 10 | /** 11 | * Created by yinxiu on 17/3/17. Email: marklin.hz@gmail.com 12 | */ 13 | public class MemoryLogPositionManager extends AbstractLogPositionManager { 14 | 15 | private Map positions; 16 | 17 | @Override 18 | public void start() { 19 | super.start(); 20 | positions = new MapMaker().makeMap(); 21 | } 22 | 23 | @Override 24 | public void stop() { 25 | super.stop(); 26 | positions.clear(); 27 | } 28 | 29 | @Override 30 | public LogPosition getLatestIndexBy(String destination) { 31 | return positions.get(destination); 32 | } 33 | 34 | @Override 35 | public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException { 36 | positions.put(destination, logPosition); 37 | } 38 | 39 | public Set destinations() { 40 | return positions.keySet(); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /parse/src/main/java/com/alibaba/otter/canal/parse/support/HaAuthenticationInfo.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.support; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collection; 5 | import java.util.List; 6 | 7 | /** 8 | * @author zebin.xuzb 2012-9-26 下午3:08:11 9 | * @version 1.0.0 10 | */ 11 | public class HaAuthenticationInfo { 12 | 13 | private AuthenticationInfo master; 14 | private List slavers = new ArrayList(); 15 | 16 | public AuthenticationInfo getMaster() { 17 | return master; 18 | } 19 | 20 | public void setMaster(AuthenticationInfo master) { 21 | this.master = master; 22 | } 23 | 24 | public List getSlavers() { 25 | return slavers; 26 | } 27 | 28 | public void addSlaver(AuthenticationInfo slaver) { 29 | this.slavers.add(slaver); 30 | } 31 | 32 | public void addSlavers(Collection slavers) { 33 | this.slavers.addAll(slavers); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /parse/src/main/resources/ddl/derby/meta_history.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE meta_history ( 2 | id bigint GENERATED ALWAYS AS IDENTITY NOT NULL, 3 | gmt_create timestamp NOT NULL, 4 | gmt_modified timestamp NOT NULL, 5 | destination varchar(128) DEFAULT NULL, 6 | binlog_file varchar(64) DEFAULT NULL, 7 | binlog_offest bigint DEFAULT NULL, 8 | binlog_master_id varchar(64) DEFAULT NULL, 9 | binlog_timestamp bigint DEFAULT NULL, 10 | use_schema varchar(1024) DEFAULT NULL, 11 | sql_schema varchar(1024) DEFAULT NULL, 12 | sql_table varchar(1024) DEFAULT NULL, 13 | sql_text clob(16 M) DEFAULT NULL, 14 | sql_type varchar(1024) DEFAULT NULL, 15 | extra varchar(512) DEFAULT NULL, 16 | PRIMARY KEY (id), 17 | CONSTRAINT meta_history_binlog_file_offest UNIQUE (destination,binlog_master_id,binlog_file,binlog_offest) 18 | ); 19 | 20 | create index meta_history_destination on meta_history(destination); 21 | create index meta_history_destination_timestamp on meta_history(destination,binlog_timestamp); 22 | create index meta_history_gmt_modified on meta_history(gmt_modified); -------------------------------------------------------------------------------- /parse/src/main/resources/ddl/derby/meta_snapshot.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE meta_snapshot ( 2 | id bigint GENERATED ALWAYS AS IDENTITY NOT NULL, 3 | gmt_create timestamp NOT NULL, 4 | gmt_modified timestamp NOT NULL, 5 | destination varchar(128) DEFAULT NULL, 6 | binlog_file varchar(64) DEFAULT NULL, 7 | binlog_offest bigint DEFAULT NULL, 8 | binlog_master_id varchar(64) DEFAULT NULL, 9 | binlog_timestamp bigint DEFAULT NULL, 10 | data clob(16 M) DEFAULT NULL, 11 | extra varchar(512) DEFAULT NULL, 12 | PRIMARY KEY (id), 13 | CONSTRAINT meta_snapshot_binlog_file_offest UNIQUE (destination,binlog_master_id,binlog_file,binlog_offest) 14 | ); 15 | 16 | create index meta_snapshot_destination on meta_snapshot(destination); 17 | create index meta_snapshot_destination_timestamp on meta_snapshot(destination,binlog_timestamp); 18 | create index meta_snapshot_gmt_modified on meta_snapshot(gmt_modified); -------------------------------------------------------------------------------- /parse/src/main/resources/ddl/h2/meta_history.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS `meta_history` ( 2 | `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', 3 | `gmt_create` datetime NOT NULL COMMENT '创建时间', 4 | `gmt_modified` datetime NOT NULL COMMENT '修改时间', 5 | `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', 6 | `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', 7 | `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', 8 | `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', 9 | `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', 10 | `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema', 11 | `sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema', 12 | `sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table', 13 | `sql_text` longtext DEFAULT NULL COMMENT '执行的sql', 14 | `sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型', 15 | `extra` text DEFAULT NULL COMMENT '额外的扩展信息', 16 | PRIMARY KEY (`id`), 17 | UNIQUE KEY meta_history_binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), 18 | KEY `meta_history_destination` (`destination`), 19 | KEY `meta_history_destination_timestamp` (`destination`,`binlog_timestamp`), 20 | KEY `meta_history_gmt_modified` (`gmt_modified`) 21 | ); -------------------------------------------------------------------------------- /parse/src/main/resources/ddl/h2/meta_snapshot.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS `meta_snapshot` ( 2 | `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', 3 | `gmt_create` datetime NOT NULL COMMENT '创建时间', 4 | `gmt_modified` datetime NOT NULL COMMENT '修改时间', 5 | `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', 6 | `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', 7 | `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', 8 | `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', 9 | `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', 10 | `data` longtext DEFAULT NULL COMMENT '表结构数据', 11 | `extra` text DEFAULT NULL COMMENT '额外的扩展信息', 12 | PRIMARY KEY (`id`), 13 | UNIQUE KEY meta_snapshot_binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), 14 | KEY `meta_snapshot_destination` (`destination`), 15 | KEY `meta_snapshot_destination_timestamp` (`destination`,`binlog_timestamp`), 16 | KEY `meta_snapshot_gmt_modified` (`gmt_modified`) 17 | ); -------------------------------------------------------------------------------- /parse/src/main/resources/ddl/mysql/meta_history.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS `meta_history` ( 2 | `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', 3 | `gmt_create` datetime NOT NULL COMMENT '创建时间', 4 | `gmt_modified` datetime NOT NULL COMMENT '修改时间', 5 | `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', 6 | `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', 7 | `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', 8 | `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', 9 | `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', 10 | `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema', 11 | `sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema', 12 | `sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table', 13 | `sql_text` longtext DEFAULT NULL COMMENT '执行的sql', 14 | `sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型', 15 | `extra` text DEFAULT NULL COMMENT '额外的扩展信息', 16 | PRIMARY KEY (`id`), 17 | UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), 18 | KEY `destination` (`destination`), 19 | KEY `destination_timestamp` (`destination`,`binlog_timestamp`), 20 | KEY `gmt_modified` (`gmt_modified`) 21 | ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构变化明细表'; -------------------------------------------------------------------------------- /parse/src/main/resources/ddl/mysql/meta_snapshot.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS `meta_snapshot` ( 2 | `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', 3 | `gmt_create` datetime NOT NULL COMMENT '创建时间', 4 | `gmt_modified` datetime NOT NULL COMMENT '修改时间', 5 | `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', 6 | `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', 7 | `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', 8 | `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', 9 | `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', 10 | `data` longtext DEFAULT NULL COMMENT '表结构数据', 11 | `extra` text DEFAULT NULL COMMENT '额外的扩展信息', 12 | PRIMARY KEY (`id`), 13 | UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), 14 | KEY `destination` (`destination`), 15 | KEY `destination_timestamp` (`destination`,`binlog_timestamp`), 16 | KEY `gmt_modified` (`gmt_modified`) 17 | ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构记录表快照表'; -------------------------------------------------------------------------------- /parse/src/test/java/com/alibaba/otter/canal/parse/helper/TimeoutChecker.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.helper; 2 | 3 | /** 4 | * 用于检查超时, 主要用于启动服务以后，如果在指定的时间内没有响应，则自动退出 5 | * 6 | * @author: yuanzu Date: 12-9-26 Time: 上午10:55 7 | */ 8 | public class TimeoutChecker { 9 | 10 | /** 11 | * 最后一次动作的时间 12 | */ 13 | private long lastTouch; 14 | /** 15 | * 超时时间 16 | */ 17 | private long timeoutMillis; 18 | 19 | /** 20 | * 运行标志 21 | */ 22 | private boolean running = true; 23 | 24 | /** 25 | * default 3s 26 | */ 27 | private static final long DEFAULT_TIMEOUT_MILLIS = 3 * 1000; 28 | 29 | public TimeoutChecker(long timeoutMillis){ 30 | this.timeoutMillis = timeoutMillis; 31 | touch(); 32 | } 33 | 34 | public TimeoutChecker(){ 35 | this(DEFAULT_TIMEOUT_MILLIS); 36 | } 37 | 38 | /** 39 | * 更新 40 | */ 41 | public void touch() { 42 | this.lastTouch = System.currentTimeMillis(); 43 | } 44 | 45 | /** 46 | * 等待空闲 47 | * 48 | * @throws InterruptedException 49 | */ 50 | public void waitForIdle() throws InterruptedException { 51 | while (this.running && (System.currentTimeMillis() - this.lastTouch) < this.timeoutMillis) { 52 | Thread.sleep(50); 53 | } 54 | } 55 | 56 | public void stop() { 57 | this.running = false; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogOpenApiTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound.mysql; 2 | 3 | import java.util.Date; 4 | import java.util.List; 5 | 6 | import org.apache.commons.lang.time.DateUtils; 7 | import org.junit.Test; 8 | 9 | import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogOpenApi; 10 | import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.BinlogFile; 11 | import com.alibaba.otter.canal.parse.inbound.mysql.rds.data.RdsBackupPolicy; 12 | 13 | /** 14 | * @author agapple 2017年10月15日 下午2:14:34 15 | * @since 1.0.25 16 | */ 17 | public class RdsBinlogOpenApiTest { 18 | 19 | @Test 20 | public void testSimple() throws Throwable { 21 | Date startTime = DateUtils.parseDate("2018-08-10 12:00:00", new String[] { "yyyy-MM-dd HH:mm:ss" }); 22 | Date endTime = DateUtils.parseDate("2018-08-11 12:00:00", new String[] { "yyyy-MM-dd HH:mm:ss" }); 23 | String url = "https://rds.aliyuncs.com/"; 24 | String ak = ""; 25 | String sk = ""; 26 | String dbInstanceId = ""; 27 | 28 | RdsBackupPolicy backupPolicy = RdsBinlogOpenApi.queryBinlogBackupPolicy(url, ak, sk, dbInstanceId); 29 | System.out.println(backupPolicy); 30 | 31 | List binlogFiles = RdsBinlogOpenApi.listBinlogFiles(url, ak, sk, dbInstanceId, startTime, endTime); 32 | System.out.println(binlogFiles); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; 2 | 3 | import java.io.File; 4 | import java.io.FileInputStream; 5 | import java.net.URL; 6 | 7 | import org.apache.commons.io.IOUtils; 8 | import org.apache.commons.lang.StringUtils; 9 | import org.junit.Assert; 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | import org.springframework.test.context.ContextConfiguration; 13 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 14 | 15 | import com.alibaba.otter.canal.parse.inbound.TableMeta; 16 | 17 | /** 18 | * @author agapple 2017年8月1日 下午7:15:54 19 | */ 20 | @RunWith(SpringJUnit4ClassRunner.class) 21 | @ContextConfiguration(locations = { "/tsdb/mysql-tsdb.xml" }) 22 | public class MemoryTableMetaTest { 23 | 24 | @Test 25 | public void testSimple() throws Throwable { 26 | MemoryTableMeta memoryTableMeta = new MemoryTableMeta(); 27 | URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt"); 28 | File dummyFile = new File(url.getFile()); 29 | File create = new File(dummyFile.getParent() + "/ddl", "create.sql"); 30 | String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n"); 31 | memoryTableMeta.apply(null, "test", sql, null); 32 | 33 | TableMeta meta = memoryTableMeta.find("test", "test"); 34 | System.out.println(meta); 35 | Assert.assertTrue(meta.getFieldMetaByName("ID").isKey()); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerBuilderTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; 2 | 3 | import org.junit.Test; 4 | import org.springframework.util.Assert; 5 | 6 | /** 7 | * @author agapple 2017年10月12日 上午10:50:00 8 | * @since 1.0.25 9 | */ 10 | public class TableMetaManagerBuilderTest { 11 | 12 | @Test 13 | public void testSimple() { 14 | TableMetaTSDB tableMetaTSDB = TableMetaTSDBBuilder.build("test", "classpath:tsdb/mysql-tsdb.xml"); 15 | Assert.notNull(tableMetaTSDB); 16 | TableMetaTSDBBuilder.destory("test"); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractZkTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.index; 2 | 3 | import org.junit.Assert; 4 | 5 | public class AbstractZkTest { 6 | 7 | protected String destination = "ljhtest1"; 8 | protected String cluster1 = "127.0.0.1:2188"; 9 | protected String cluster2 = "127.0.0.1:2188,127.0.0.1:2188"; 10 | 11 | public void sleep(long time) { 12 | try { 13 | Thread.sleep(time); 14 | } catch (InterruptedException e) { 15 | Assert.fail(e.getMessage()); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /parse/src/test/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.index; 2 | 3 | import org.junit.Test; 4 | 5 | public class MemoryLogPositionManagerTest extends AbstractLogPositionManagerTest { 6 | 7 | @Test 8 | public void testAll() { 9 | MemoryLogPositionManager logPositionManager = new MemoryLogPositionManager(); 10 | logPositionManager.start(); 11 | doTest(logPositionManager); 12 | logPositionManager.stop(); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /parse/src/test/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManagerTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.index; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | 7 | import com.alibaba.otter.canal.common.zookeeper.ZkClientx; 8 | import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; 9 | 10 | public class ZooKeeperLogPositionManagerTest extends AbstractLogPositionManagerTest { 11 | 12 | private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2); 13 | 14 | @Before 15 | public void setUp() { 16 | String path = ZookeeperPathUtils.getDestinationPath(destination); 17 | zkclientx.deleteRecursive(path); 18 | } 19 | 20 | @After 21 | public void tearDown() { 22 | String path = ZookeeperPathUtils.getDestinationPath(destination); 23 | zkclientx.deleteRecursive(path); 24 | } 25 | 26 | @Test 27 | public void testAll() { 28 | ZooKeeperLogPositionManager logPositionManager = new ZooKeeperLogPositionManager(zkclientx); 29 | logPositionManager.start(); 30 | 31 | doTest(logPositionManager); 32 | logPositionManager.stop(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalEventSinkTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.parse.stub; 2 | 3 | import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; 4 | import com.alibaba.otter.canal.sink.CanalEventSink; 5 | 6 | public abstract class AbstractCanalEventSinkTest extends AbstractCanalLifeCycle implements CanalEventSink { 7 | 8 | public void interrupt() { 9 | // do nothing 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /parse/src/test/resources/binlog/mysql-bin.000001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/parse/src/test/resources/binlog/mysql-bin.000001 -------------------------------------------------------------------------------- /parse/src/test/resources/binlog/mysql-bin.000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/parse/src/test/resources/binlog/mysql-bin.000002 -------------------------------------------------------------------------------- /parse/src/test/resources/binlog/tsdb/mysql-bin.000001: -------------------------------------------------------------------------------- 1 | �bine��Yw{5.7.19-loge��Y8 2 | _ 3 | 4 | 5 | **4m��;f��Y#��Ŗ�( -------------------------------------------------------------------------------- /parse/src/test/resources/binlog/tsdb/mysql-bin.000002: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/parse/src/test/resources/binlog/tsdb/mysql-bin.000002 -------------------------------------------------------------------------------- /parse/src/test/resources/binlog/tsdb/mysql-bin.000003: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/itmifen/canal/034ad622e05e888fd5b0825b25ddb3c8ba372e39/parse/src/test/resources/binlog/tsdb/mysql-bin.000003 -------------------------------------------------------------------------------- /parse/src/test/resources/ddl/ddl_test1.sql: -------------------------------------------------------------------------------- 1 | create table yushitai_test.card_record ( 2 | id bigint auto_increment, 3 | last_update timestamp not null DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 4 | hint varchar(64) charset ascii not null, 5 | value varchar(255) charset ascii not null, 6 | primary key(id), 7 | unique key hint_uidx(hint) 8 | ) auto_increment=256 ; 9 | 10 | DROP TABLE IF EXISTS _card_record_gho /* generated by server */ ; 11 | DROP TABLE IF EXISTS _card_record_del /* generated by server */ ; 12 | 13 | create /* gh-ost */ table yushitai_test._card_record_gho like yushitai_test.card_record ; 14 | alter /* gh-ost */ table yushitai_test._card_record_gho add column customization_id bigint unsigned NOT NULL COMMENT 'TEST' ; 15 | 16 | create /* gh-ost */ table yushitai_test._card_record_del ( 17 | id int auto_increment primary key 18 | ) engine=InnoDB comment='ghost-cut-over-sentry' ; 19 | 20 | DROP TABLE IF EXISTS _card_record_del /* generated by server */ ; 21 | rename /* gh-ost */ table yushitai_test.card_record to yushitai_test._card_record_del, yushitai_test._card_record_gho to yushitai_test.card_record; -------------------------------------------------------------------------------- /parse/src/test/resources/ddl/ddl_test2.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE yushitai_test.card_record ( 2 | id bigint AUTO_INCREMENT, 3 | name varchar(32) DEFAULT NULL, 4 | alias varchar(32) DEFAULT NULL, 5 | INDEX index_name(name), 6 | CONSTRAINT pk_id PRIMARY KEY (id), 7 | UNIQUE uk_name (name,alias) 8 | ) AUTO_INCREMENT = 256 9 | -------------------------------------------------------------------------------- /parse/src/test/resources/dummy.txt: -------------------------------------------------------------------------------- 1 | 本文件仅仅为定位绝对路径使用 -------------------------------------------------------------------------------- /parse/src/test/resources/tsdb/sql-map/sqlmap-config.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /prometheus/src/main/java/com/alibaba/otter/canal/prometheus/InstanceRegistry.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.prometheus; 2 | 3 | import com.alibaba.otter.canal.instance.core.CanalInstance; 4 | 5 | /** 6 | * @author Chuanyi Li 7 | */ 8 | public interface InstanceRegistry { 9 | 10 | void register(CanalInstance instance); 11 | 12 | void unregister(CanalInstance instance); 13 | 14 | } 15 | -------------------------------------------------------------------------------- /prometheus/src/main/java/com/alibaba/otter/canal/prometheus/PrometheusProvider.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.prometheus; 2 | 3 | import com.alibaba.otter.canal.spi.CanalMetricsProvider; 4 | import com.alibaba.otter.canal.spi.CanalMetricsService; 5 | 6 | /** 7 | * @author Chuanyi Li 8 | */ 9 | public class PrometheusProvider implements CanalMetricsProvider { 10 | 11 | @Override 12 | public CanalMetricsService getService() { 13 | return PrometheusService.getInstance(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /prometheus/src/main/resources/META-INF/services/com.alibaba.otter.canal.spi.CanalMetricsProvider: -------------------------------------------------------------------------------- 1 | com.alibaba.otter.canal.prometheus.PrometheusProvider 2 | -------------------------------------------------------------------------------- /protocol/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.alibaba.otter 5 | canal 6 | 1.1.3-SNAPSHOT 7 | ../pom.xml 8 | 9 | com.alibaba.otter 10 | canal.protocol 11 | jar 12 | canal protocol module for otter ${project.version} 13 | http://b2b-doc.alibaba-inc.com/display/opentech/Otter 14 | 15 | 16 | 17 | com.alibaba.otter 18 | canal.common 19 | ${project.version} 20 | 21 | 22 | com.google.protobuf 23 | protobuf-java 24 | 25 | 26 | commons-lang 27 | commons-lang 28 | 29 | 30 | oro 31 | oro 32 | 33 | 34 | com.googlecode.aviator 35 | aviator 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /protocol/src/main/java/com/alibaba/otter/canal/protocol/exception/CanalClientException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.protocol.exception; 2 | 3 | import org.apache.commons.lang.exception.NestableRuntimeException; 4 | 5 | /** 6 | * @author zebin.xuzb @ 2012-6-20 7 | * @version 1.0.0 8 | */ 9 | public class CanalClientException extends NestableRuntimeException { 10 | 11 | private static final long serialVersionUID = -7545341502620139031L; 12 | 13 | public CanalClientException(String errorCode){ 14 | super(errorCode); 15 | } 16 | 17 | public CanalClientException(String errorCode, Throwable cause){ 18 | super(errorCode, cause); 19 | } 20 | 21 | public CanalClientException(String errorCode, String errorDesc){ 22 | super(errorCode + ":" + errorDesc); 23 | } 24 | 25 | public CanalClientException(String errorCode, String errorDesc, Throwable cause){ 26 | super(errorCode + ":" + errorDesc, cause); 27 | } 28 | 29 | public CanalClientException(Throwable cause){ 30 | super(cause); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /protocol/src/main/java/com/alibaba/otter/canal/protocol/position/MetaqPosition.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.protocol.position; 2 | 3 | /** 4 | * @author zebin.xuzb 2012-11-3 上午12:23:01 5 | * @since 1.0.0 6 | */ 7 | public class MetaqPosition extends Position { 8 | 9 | private static final long serialVersionUID = -8673508769040569273L; 10 | 11 | private String topic; 12 | private String msgNewId; 13 | private long offset; 14 | 15 | public MetaqPosition(String topic, String msgNewId, long offset){ 16 | super(); 17 | this.topic = topic; 18 | this.msgNewId = msgNewId; 19 | this.offset = offset; 20 | } 21 | 22 | public String getTopic() { 23 | return topic; 24 | } 25 | 26 | public String getMsgNewId() { 27 | return msgNewId; 28 | } 29 | 30 | public void setTopic(String topic) { 31 | this.topic = topic; 32 | } 33 | 34 | public void setMsgNewId(String msgNewId) { 35 | this.msgNewId = msgNewId; 36 | } 37 | 38 | public long getOffset() { 39 | return offset; 40 | } 41 | 42 | public void setOffset(long offset) { 43 | this.offset = offset; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /protocol/src/main/java/com/alibaba/otter/canal/protocol/position/Position.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.protocol.position; 2 | 3 | import java.io.Serializable; 4 | 5 | import org.apache.commons.lang.builder.ToStringBuilder; 6 | 7 | import com.alibaba.otter.canal.common.utils.CanalToStringStyle; 8 | 9 | /** 10 | * 事件唯一标示 11 | */ 12 | public abstract class Position implements Serializable { 13 | 14 | private static final long serialVersionUID = 2332798099928474975L; 15 | 16 | public String toString() { 17 | return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/kafka/MessageSerializer.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.kafka; 2 | 3 | import java.util.Map; 4 | 5 | import org.apache.commons.lang.BooleanUtils; 6 | import org.apache.kafka.common.serialization.Serializer; 7 | 8 | import com.alibaba.otter.canal.common.CanalMessageSerializer; 9 | import com.alibaba.otter.canal.protocol.Message; 10 | 11 | /** 12 | * Kafka Message类的序列化 13 | * 14 | * @author machengyuan 2018-6-11 下午05:30:49 15 | * @version 1.0.0 16 | */ 17 | public class MessageSerializer implements Serializer { 18 | 19 | private boolean filterTransactionEntry = false; 20 | 21 | public MessageSerializer(){ 22 | this.filterTransactionEntry = BooleanUtils.toBoolean(System.getProperty("canal.instance.filter.transaction.entry", 23 | "false")); 24 | } 25 | 26 | @Override 27 | public void configure(Map configs, boolean isKey) { 28 | } 29 | 30 | @Override 31 | public byte[] serialize(String topic, Message data) { 32 | return CanalMessageSerializer.serializer(data, filterTransactionEntry); 33 | } 34 | 35 | @Override 36 | public void close() { 37 | // nothing to do 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/server/CanalServer.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.server; 2 | 3 | import com.alibaba.otter.canal.common.CanalLifeCycle; 4 | import com.alibaba.otter.canal.server.exception.CanalServerException; 5 | 6 | /** 7 | * 对应canal整个服务实例，一个jvm实例只有一份server 8 | * 9 | * @author jianghang 2012-7-12 下午01:32:29 10 | * @version 1.0.0 11 | */ 12 | public interface CanalServer extends CanalLifeCycle { 13 | 14 | void start() throws CanalServerException; 15 | 16 | void stop() throws CanalServerException; 17 | } 18 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/server/CanalServerStarter.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.server; 2 | 3 | /** 4 | * 外部服务如Kafka, RocketMQ启动接口 5 | * 6 | * @author machengyuan 2018-8-23 下午05:20:29 7 | * @version 1.0.0 8 | */ 9 | public interface CanalServerStarter { 10 | 11 | void init(); 12 | } 13 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/server/CanalService.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.server; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | 5 | import com.alibaba.otter.canal.protocol.ClientIdentity; 6 | import com.alibaba.otter.canal.protocol.Message; 7 | import com.alibaba.otter.canal.server.exception.CanalServerException; 8 | 9 | public interface CanalService { 10 | 11 | void subscribe(ClientIdentity clientIdentity) throws CanalServerException; 12 | 13 | void unsubscribe(ClientIdentity clientIdentity) throws CanalServerException; 14 | 15 | Message get(ClientIdentity clientIdentity, int batchSize) throws CanalServerException; 16 | 17 | Message get(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit) throws CanalServerException; 18 | 19 | Message getWithoutAck(ClientIdentity clientIdentity, int batchSize) throws CanalServerException; 20 | 21 | Message getWithoutAck(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit) 22 | throws CanalServerException; 23 | 24 | void ack(ClientIdentity clientIdentity, long batchId) throws CanalServerException; 25 | 26 | void rollback(ClientIdentity clientIdentity) throws CanalServerException; 27 | 28 | void rollback(ClientIdentity clientIdentity, Long batchId) throws CanalServerException; 29 | } 30 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/server/exception/CanalServerException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.server.exception; 2 | 3 | import com.alibaba.otter.canal.common.CanalException; 4 | 5 | /** 6 | * canal 异常定义 7 | * 8 | * @author jianghang 2012-6-15 下午04:57:35 9 | * @version 1.0.0 10 | */ 11 | public class CanalServerException extends CanalException { 12 | 13 | private static final long serialVersionUID = -7288830284122672209L; 14 | 15 | public CanalServerException(String errorCode){ 16 | super(errorCode); 17 | } 18 | 19 | public CanalServerException(String errorCode, Throwable cause){ 20 | super(errorCode, cause); 21 | } 22 | 23 | public CanalServerException(String errorCode, String errorDesc){ 24 | super(errorCode + ":" + errorDesc); 25 | } 26 | 27 | public CanalServerException(String errorCode, String errorDesc, Throwable cause){ 28 | super(errorCode + ":" + errorDesc, cause); 29 | } 30 | 31 | public CanalServerException(Throwable cause){ 32 | super(cause); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNettyProfiler.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.server.netty; 2 | 3 | import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; 4 | import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator.ClientRequestResult; 5 | 6 | /** 7 | * @author Chuanyi Li 8 | */ 9 | public class CanalServerWithNettyProfiler { 10 | 11 | public static final ClientInstanceProfiler NOP = new DefaultClientInstanceProfiler(); 12 | private ClientInstanceProfiler instanceProfiler; 13 | 14 | private static class SingletonHolder { 15 | private static CanalServerWithNettyProfiler SINGLETON = new CanalServerWithNettyProfiler(); 16 | } 17 | 18 | private CanalServerWithNettyProfiler() { 19 | this.instanceProfiler = NOP; 20 | } 21 | 22 | public static CanalServerWithNettyProfiler profiler() { 23 | return SingletonHolder.SINGLETON; 24 | } 25 | 26 | public void profiling(ClientRequestResult result) { 27 | instanceProfiler.profiling(result); 28 | } 29 | 30 | public void setInstanceProfiler(ClientInstanceProfiler instanceProfiler) { 31 | this.instanceProfiler = instanceProfiler; 32 | } 33 | 34 | private static class DefaultClientInstanceProfiler extends AbstractCanalLifeCycle implements ClientInstanceProfiler { 35 | @Override 36 | public void profiling(ClientRequestResult result) {} 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/server/netty/ClientInstanceProfiler.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.server.netty; 2 | 3 | import com.alibaba.otter.canal.common.CanalLifeCycle; 4 | import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator.ClientRequestResult; 5 | 6 | /** 7 | * @author Chuanyi Li 8 | */ 9 | public interface ClientInstanceProfiler extends CanalLifeCycle { 10 | 11 | void profiling(ClientRequestResult result); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/server/netty/handler/FixedHeaderFrameDecoder.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.server.netty.handler; 2 | 3 | import org.jboss.netty.buffer.ChannelBuffer; 4 | import org.jboss.netty.channel.Channel; 5 | import org.jboss.netty.channel.ChannelHandlerContext; 6 | import org.jboss.netty.handler.codec.replay.ReplayingDecoder; 7 | import org.jboss.netty.handler.codec.replay.VoidEnum; 8 | 9 | /** 10 | * 解析对应的header信息 11 | * 12 | * @author jianghang 2012-10-24 上午11:31:39 13 | * @version 1.0.0 14 | */ 15 | public class FixedHeaderFrameDecoder extends ReplayingDecoder { 16 | 17 | protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer, VoidEnum state) 18 | throws Exception { 19 | return buffer.readBytes(buffer.readInt()); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.spi; 2 | 3 | import java.io.IOException; 4 | 5 | import com.alibaba.otter.canal.common.MQProperties; 6 | import com.alibaba.otter.canal.protocol.Message; 7 | 8 | public interface CanalMQProducer { 9 | 10 | /** 11 | * Init producer. 12 | * 13 | * @param mqProperties MQ config 14 | */ 15 | void init(MQProperties mqProperties); 16 | 17 | /** 18 | * Send canal message to related topic 19 | * 20 | * @param canalDestination canal mq destination 21 | * @param message canal message 22 | * @throws IOException 23 | */ 24 | void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) throws IOException; 25 | 26 | /** 27 | * Stop MQ producer service 28 | */ 29 | void stop(); 30 | 31 | interface Callback { 32 | 33 | void commit(); 34 | 35 | void rollback(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsProvider.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.spi; 2 | 3 | /** 4 | * Use java service provider mechanism to provide {@link CanalMetricsService}. 5 | * 
 6 |  * Example:
 7 |  * {@code
 8 |  *     ServiceLoader providers = ServiceLoader.load(CanalMetricsProvider.class);
 9 |  *     List list = new ArrayList();
10 |  *     for (CanalMetricsProvider provider : providers) {
11 |  *         list.add(provider);
12 |  *     }
13 |  * }
14 |  *
15 | * @author Chuanyi Li 16 | */ 17 | public interface CanalMetricsProvider { 18 | 19 | /** 20 | * @return Impl of {@link CanalMetricsService} 21 | */ 22 | CanalMetricsService getService(); 23 | 24 | } 25 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/spi/CanalMetricsService.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.spi; 2 | 3 | import com.alibaba.otter.canal.instance.core.CanalInstance; 4 | 5 | /** 6 | * Canal server/instance metrics for export. 7 | * 8 | * Designed to be created by service provider. 9 | * 10 | * @see CanalMetricsProvider 11 | * @author Chuanyi Li 12 | */ 13 | public interface CanalMetricsService { 14 | 15 | /** 16 | * Initialization on canal server startup. 17 | */ 18 | void initialize(); 19 | 20 | /** 21 | * Clean-up at canal server stop phase. 22 | */ 23 | void terminate(); 24 | 25 | /** 26 | * @return {@code true} if the metrics service is running, otherwise {@code false}. 27 | */ 28 | boolean isRunning(); 29 | 30 | /** 31 | * Register instance level metrics for specified instance. 32 | * @param instance {@link CanalInstance} 33 | */ 34 | void register(CanalInstance instance); 35 | 36 | /** 37 | * Unregister instance level metrics for specified instance. 38 | * @param instance {@link CanalInstance} 39 | */ 40 | void unregister(CanalInstance instance); 41 | 42 | /** 43 | * @param port server port for pull 44 | */ 45 | void setServerPort(int port); 46 | 47 | } 48 | -------------------------------------------------------------------------------- /server/src/main/java/com/alibaba/otter/canal/spi/NopCanalMetricsService.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.spi; 2 | 3 | import com.alibaba.otter.canal.instance.core.CanalInstance; 4 | 5 | /** 6 | * @author Chuanyi Li 7 | */ 8 | public class NopCanalMetricsService implements CanalMetricsService { 9 | 10 | public static final NopCanalMetricsService NOP = new NopCanalMetricsService(); 11 | 12 | private NopCanalMetricsService() {} 13 | 14 | @Override 15 | public void initialize() { 16 | 17 | } 18 | 19 | @Override 20 | public void terminate() { 21 | 22 | } 23 | 24 | @Override 25 | public boolean isRunning() { 26 | return false; 27 | } 28 | 29 | @Override 30 | public void register(CanalInstance instance) { 31 | 32 | } 33 | 34 | @Override 35 | public void unregister(CanalInstance instance) { 36 | 37 | } 38 | 39 | @Override 40 | public void setServerPort(int port) { 41 | 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /sink/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.alibaba.otter 5 | canal 6 | 1.1.3-SNAPSHOT 7 | ../pom.xml 8 | 9 | com.alibaba.otter 10 | canal.sink 11 | jar 12 | canal sink module for otter ${project.version} 13 | 14 | 15 | com.alibaba.otter 16 | canal.common 17 | ${project.version} 18 | 19 | 20 | com.alibaba.otter 21 | canal.protocol 22 | ${project.version} 23 | 24 | 25 | com.alibaba.otter 26 | canal.filter 27 | ${project.version} 28 | 29 | 30 | com.alibaba.otter 31 | canal.store 32 | ${project.version} 33 | 34 | 35 | 36 | junit 37 | junit 38 | test 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /sink/src/main/java/com/alibaba/otter/canal/sink/AbstractCanalEventDownStreamHandler.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.sink; 2 | 3 | import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; 4 | 5 | /** 6 | * 默认的实现 7 | * 8 | * @author jianghang 2013-10-8 下午8:35:29 9 | * @since 1.0.12 10 | */ 11 | public class AbstractCanalEventDownStreamHandler extends AbstractCanalLifeCycle implements CanalEventDownStreamHandler { 12 | 13 | public T before(T events) { 14 | return events; 15 | } 16 | 17 | public T retry(T events) { 18 | return events; 19 | } 20 | 21 | public T after(T events) { 22 | return events; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /sink/src/main/java/com/alibaba/otter/canal/sink/CanalEventDownStreamHandler.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.sink; 2 | 3 | import com.alibaba.otter.canal.common.CanalLifeCycle; 4 | 5 | /** 6 | * 处理下sink时的数据流 7 | * 8 | * @author jianghang 2012-7-31 下午03:06:26 9 | * @version 1.0.0 10 | */ 11 | public interface CanalEventDownStreamHandler extends CanalLifeCycle { 12 | 13 | /** 14 | * 提交到store之前做一下处理，允许替换Event 15 | */ 16 | public T before(T events); 17 | 18 | /** 19 | * store处于full后，retry时处理做一下处理 20 | */ 21 | public T retry(T events); 22 | 23 | /** 24 | * 提交store成功后做一下处理 25 | */ 26 | public T after(T events); 27 | } 28 | -------------------------------------------------------------------------------- /sink/src/main/java/com/alibaba/otter/canal/sink/CanalEventSink.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.sink; 2 | 3 | import java.net.InetSocketAddress; 4 | 5 | import com.alibaba.otter.canal.common.CanalLifeCycle; 6 | import com.alibaba.otter.canal.sink.entry.group.GroupEventSink; 7 | import com.alibaba.otter.canal.sink.exception.CanalSinkException; 8 | 9 | /** 10 | * event事件消费者 11 | * 12 | * 
13 |  * 1. 剥离filter/sink为独立的两个动作，方便在快速判断数据是否有效
14 |  *
15 | * 16 | * @author jianghang 2012-6-21 下午05:03:40 17 | * @version 1.0.0 18 | */ 19 | public interface CanalEventSink extends CanalLifeCycle { 20 | 21 | /** 22 | * 提交数据 23 | * 24 | * @param event 25 | * @param remoteAddress 26 | * @param destination 27 | * @throws CanalSinkException 28 | * @throws InterruptedException 29 | */ 30 | boolean sink(T event, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, 31 | InterruptedException; 32 | 33 | /** 34 | * 中断消费，比如解析模块发生了切换，想临时中断当前的merge请求，清理对应的上下文状态，可见{@linkplain GroupEventSink} 35 | */ 36 | void interrupt(); 37 | 38 | } 39 | -------------------------------------------------------------------------------- /sink/src/main/java/com/alibaba/otter/canal/sink/entry/HeartBeatEntryEventHandler.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.sink.entry; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; 7 | import com.alibaba.otter.canal.sink.AbstractCanalEventDownStreamHandler; 8 | import com.alibaba.otter.canal.store.model.Event; 9 | 10 | /** 11 | * 处理一下一下heartbeat数据 12 | * 13 | * @author jianghang 2013-10-8 下午6:03:53 14 | * @since 1.0.12 15 | */ 16 | public class HeartBeatEntryEventHandler extends AbstractCanalEventDownStreamHandler> { 17 | 18 | public List before(List events) { 19 | boolean existHeartBeat = false; 20 | for (Event event : events) { 21 | if (event.getEntryType() == EntryType.HEARTBEAT) { 22 | existHeartBeat = true; 23 | } 24 | } 25 | 26 | if (!existHeartBeat) { 27 | return events; 28 | } else { 29 | // 目前heartbeat和其他事件是分离的，保险一点还是做一下检查处理 30 | List result = new ArrayList(); 31 | for (Event event : events) { 32 | if (event.getEntryType() != EntryType.HEARTBEAT) { 33 | result.add(event); 34 | } 35 | } 36 | 37 | return result; 38 | } 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/GroupBarrier.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.sink.entry.group; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | import java.util.concurrent.TimeoutException; 5 | 6 | /** 7 | * 针对group合并的barrier接口，控制多个sink操作的合并处理 8 | * 9 | * @author jianghang 2012-10-18 下午05:07:35 10 | * @version 1.0.0 11 | */ 12 | public interface GroupBarrier { 13 | 14 | /** 15 | * 判断当前的数据对象是否允许通过 16 | * 17 | * @param event 18 | * @throws InterruptedException 19 | */ 20 | public void await(T event) throws InterruptedException; 21 | 22 | /** 23 | * 判断当前的数据对象是否允许通过，带超时控制 24 | * 25 | * @param event 26 | * @param timeout 27 | * @param unit 28 | * @throws InterruptedException 29 | * @throws TimeoutException 30 | */ 31 | public void await(T event, long timeout, TimeUnit unit) throws InterruptedException, TimeoutException; 32 | 33 | /** 34 | * sink成功，清理对应barrier的状态 35 | */ 36 | public void clear(T event); 37 | 38 | /** 39 | * 出现切换，发起interrupt，清理对应的上下文 40 | */ 41 | public void interrupt(); 42 | } 43 | -------------------------------------------------------------------------------- /sink/src/main/java/com/alibaba/otter/canal/sink/exception/CanalSinkException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.sink.exception; 2 | 3 | import com.alibaba.otter.canal.common.CanalException; 4 | 5 | /** 6 | * canal 异常定义 7 | * 8 | * @author jianghang 2012-6-15 下午04:57:35 9 | * @version 1.0.0 10 | */ 11 | public class CanalSinkException extends CanalException { 12 | 13 | private static final long serialVersionUID = -7288830284122672209L; 14 | 15 | public CanalSinkException(String errorCode){ 16 | super(errorCode); 17 | } 18 | 19 | public CanalSinkException(String errorCode, Throwable cause){ 20 | super(errorCode, cause); 21 | } 22 | 23 | public CanalSinkException(String errorCode, String errorDesc){ 24 | super(errorCode + ":" + errorDesc); 25 | } 26 | 27 | public CanalSinkException(String errorCode, String errorDesc, Throwable cause){ 28 | super(errorCode + ":" + errorDesc, cause); 29 | } 30 | 31 | public CanalSinkException(Throwable cause){ 32 | super(cause); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /store/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.alibaba.otter 5 | canal 6 | 1.1.3-SNAPSHOT 7 | ../pom.xml 8 | 9 | com.alibaba.otter 10 | canal.store 11 | jar 12 | canal store module for otter ${project.version} 13 | 14 | 15 | com.alibaba.otter 16 | canal.common 17 | ${project.version} 18 | 19 | 20 | com.alibaba.otter 21 | canal.protocol 22 | ${project.version} 23 | 24 | 25 | com.alibaba.otter 26 | canal.meta 27 | ${project.version} 28 | 29 | 30 | 31 | junit 32 | junit 33 | test 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /store/src/main/java/com/alibaba/otter/canal/store/AbstractCanalGroupStore.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.store; 2 | 3 | import java.util.Map; 4 | 5 | import org.springframework.util.Assert; 6 | 7 | import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; 8 | import com.google.common.collect.MapMaker; 9 | 10 | /** 11 | * @author zebin.xuzb 2012-10-30 下午3:45:17 12 | * @since 1.0.0 13 | */ 14 | public abstract class AbstractCanalGroupStore extends AbstractCanalLifeCycle implements CanalGroupEventStore { 15 | 16 | protected Map stores = new MapMaker().makeMap(); 17 | 18 | @Override 19 | public void addStoreInfo(StoreInfo info) { 20 | checkInfo(info); 21 | stores.put(info.getStoreName(), info); 22 | } 23 | 24 | protected void checkInfo(StoreInfo info) { 25 | Assert.notNull(info); 26 | Assert.hasText(info.getStoreName()); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /store/src/main/java/com/alibaba/otter/canal/store/CanalGroupEventStore.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.store; 2 | 3 | /** 4 | * 提供给上层统一的 store 视图，内部则支持多种store混合，并且维持着多个store供上层进行路由 5 | * 6 | * @author zebin.xuzb 2012-10-30 下午12:17:26 7 | * @since 1.0.0 8 | */ 9 | public interface CanalGroupEventStore extends CanalEventStore { 10 | 11 | void addStoreInfo(StoreInfo info); 12 | } 13 | -------------------------------------------------------------------------------- /store/src/main/java/com/alibaba/otter/canal/store/CanalStoreConstants.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.store; 2 | 3 | /** 4 | * 常量值 5 | * 6 | * @author jianghang 2012-6-14 下午09:40:33 7 | * @version 1.0.0 8 | */ 9 | public interface CanalStoreConstants { 10 | 11 | public static final String CODE_POSITION_NOT_FOUND = "position:%s not found"; 12 | 13 | public static final String CODE_POSITION_NOT_IN_ORDER = "position:%s not in order"; 14 | 15 | public static final String ENCODING = "utf8"; 16 | 17 | public static final int MAX_STORECOUNT = 100; 18 | 19 | public static final int ROLLOVERCOUNT = 100; 20 | 21 | } 22 | -------------------------------------------------------------------------------- /store/src/main/java/com/alibaba/otter/canal/store/CanalStoreException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.store; 2 | 3 | import com.alibaba.otter.canal.common.CanalException; 4 | 5 | /** 6 | * canal 异常定义 7 | * 8 | * @author jianghang 2012-6-15 下午04:57:35 9 | * @version 1.0.0 10 | */ 11 | public class CanalStoreException extends CanalException { 12 | 13 | private static final long serialVersionUID = -7288830284122672209L; 14 | 15 | public CanalStoreException(String errorCode){ 16 | super(errorCode); 17 | } 18 | 19 | public CanalStoreException(String errorCode, Throwable cause){ 20 | super(errorCode, cause); 21 | } 22 | 23 | public CanalStoreException(String errorCode, String errorDesc){ 24 | super(errorCode + ":" + errorDesc); 25 | } 26 | 27 | public CanalStoreException(String errorCode, String errorDesc, Throwable cause){ 28 | super(errorCode + ":" + errorDesc, cause); 29 | } 30 | 31 | public CanalStoreException(Throwable cause){ 32 | super(cause); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /store/src/main/java/com/alibaba/otter/canal/store/CanalStoreScavenge.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.store; 2 | 3 | import com.alibaba.otter.canal.protocol.position.Position; 4 | 5 | /** 6 | * store空间回收机制，信息采集以及控制何时调用{@linkplain CanalEventStore}.cleanUtil()接口 7 | * 8 | * @author jianghang 2012-8-8 上午11:57:42 9 | * @version 1.0.0 10 | */ 11 | public interface CanalStoreScavenge { 12 | 13 | /** 14 | * 清理position之前的数据 15 | */ 16 | void cleanUntil(Position position) throws CanalStoreException; 17 | 18 | /** 19 | * 删除所有的数据 20 | */ 21 | void cleanAll() throws CanalStoreException; 22 | } 23 | -------------------------------------------------------------------------------- /store/src/main/java/com/alibaba/otter/canal/store/StoreInfo.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.store; 2 | 3 | /** 4 | * @author zebin.xuzb 2012-10-30 下午1:05:13 5 | * @since 1.0.0 6 | */ 7 | public class StoreInfo { 8 | 9 | private String storeName; 10 | private String filter; 11 | 12 | public String getStoreName() { 13 | return storeName; 14 | } 15 | 16 | public String getFilter() { 17 | return filter; 18 | } 19 | 20 | public void setStoreName(String storeName) { 21 | this.storeName = storeName; 22 | } 23 | 24 | public void setFilter(String filter) { 25 | this.filter = filter; 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /store/src/main/java/com/alibaba/otter/canal/store/model/BatchMode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.store.model; 2 | 3 | /** 4 | * 批处理模式 5 | * 6 | * @author jianghang 2013-3-18 上午11:51:15 7 | * @version 1.0.3 8 | */ 9 | public enum BatchMode { 10 | 11 | /** 对象数量 */ 12 | ITEMSIZE, 13 | 14 | /** 内存大小 */ 15 | MEMSIZE; 16 | 17 | public boolean isItemSize() { 18 | return this == BatchMode.ITEMSIZE; 19 | } 20 | 21 | public boolean isMemSize() { 22 | return this == BatchMode.MEMSIZE; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /store/src/main/java/com/alibaba/otter/canal/store/model/Events.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.otter.canal.store.model; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import org.apache.commons.lang.builder.ToStringBuilder; 8 | 9 | import com.alibaba.otter.canal.common.utils.CanalToStringStyle; 10 | import com.alibaba.otter.canal.protocol.position.PositionRange; 11 | 12 | /** 13 | * 代表一组数据对象的集合 14 | * 15 | * @author jianghang 2012-6-14 下午09:07:41 16 | * @version 1.0.0 17 | */ 18 | public class Events implements Serializable { 19 | 20 | private static final long serialVersionUID = -7337454954300706044L; 21 | 22 | private PositionRange positionRange = new PositionRange(); 23 | private List events = new ArrayList(); 24 | 25 | public List getEvents() { 26 | return events; 27 | } 28 | 29 | public void setEvents(List events) { 30 | this.events = events; 31 | } 32 | 33 | public PositionRange getPositionRange() { 34 | return positionRange; 35 | } 36 | 37 | public void setPositionRange(PositionRange positionRange) { 38 | this.positionRange = positionRange; 39 | } 40 | 41 | public String toString() { 42 | return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); 43 | } 44 | } 45 | --------------------------------------------------------------------------------