├── .gitignore ├── .idea ├── .name ├── compiler.xml ├── copyright │ └── profiles_settings.xml ├── dictionaries │ └── plin.xml ├── encodings.xml ├── libraries │ ├── Maven__ch_qos_logback_logback_classic_1_1_2.xml │ ├── Maven__ch_qos_logback_logback_core_1_1_2.xml │ ├── Maven__com_alibaba_druid_1_0_11.xml │ ├── Maven__com_alibaba_fastjson_1_1_35.xml │ ├── Maven__com_alibaba_otter_canal_client_1_0_20.xml │ ├── Maven__com_alibaba_otter_canal_common_1_0_20.xml │ ├── Maven__com_alibaba_otter_canal_protocol_1_0_20.xml │ ├── Maven__com_github_sgroschupf_zkclient_0_1.xml │ ├── Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml │ ├── Maven__com_google_code_findbugs_jsr305_1_3_9.xml │ ├── Maven__com_google_guava_guava_12_0_1.xml │ ├── Maven__com_google_guava_guava_18_0.xml │ ├── Maven__com_google_protobuf_protobuf_java_2_4_1.xml │ ├── Maven__com_google_protobuf_protobuf_java_2_5_0.xml │ ├── Maven__com_jcraft_jsch_0_1_42.xml │ ├── Maven__com_thoughtworks_paranamer_paranamer_2_3.xml │ ├── Maven__com_yammer_metrics_metrics_core_2_2_0.xml │ ├── Maven__commons_beanutils_commons_beanutils_1_7_0.xml │ ├── Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml │ ├── Maven__commons_cli_commons_cli_1_2.xml │ ├── Maven__commons_codec_commons_codec_1_9.xml │ ├── Maven__commons_collections_commons_collections_3_2_2.xml │ ├── Maven__commons_configuration_commons_configuration_1_6.xml │ ├── Maven__commons_digester_commons_digester_1_8.xml │ ├── Maven__commons_el_commons_el_1_0.xml │ ├── Maven__commons_httpclient_commons_httpclient_3_1.xml │ ├── Maven__commons_io_commons_io_2_4.xml │ ├── Maven__commons_lang_commons_lang_2_6.xml │ ├── Maven__commons_logging_commons_logging_1_2.xml │ ├── Maven__commons_net_commons_net_3_1.xml │ ├── Maven__io_netty_netty_3_6_2_Final.xml │ ├── Maven__io_netty_netty_all_4_0_23_Final.xml │ ├── Maven__javax_activation_activation_1_1.xml │ ├── Maven__javax_xml_bind_jaxb_api_2_2_2.xml │ ├── Maven__javax_xml_stream_stax_api_1_0_2.xml │ ├── Maven__junit_junit_4_12.xml │ ├── Maven__log4j_log4j_1_2_17.xml │ ├── Maven__net_jpountz_lz4_lz4_1_2_0.xml │ ├── Maven__net_sf_jopt_simple_jopt_simple_3_2.xml │ ├── Maven__org_apache_avro_avro_1_7_4.xml │ ├── Maven__org_apache_commons_commons_compress_1_4_1.xml │ ├── Maven__org_apache_commons_commons_lang3_3_3_2.xml │ ├── Maven__org_apache_commons_commons_math3_3_1_1.xml │ ├── Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml │ ├── Maven__org_apache_directory_api_api_util_1_0_0_M20.xml │ ├── Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml │ ├── Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml │ ├── Maven__org_apache_hadoop_hadoop_annotations_2_5_1.xml │ ├── Maven__org_apache_hadoop_hadoop_auth_2_5_1.xml │ ├── Maven__org_apache_hadoop_hadoop_common_2_5_1.xml │ ├── Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_5_1.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_api_2_5_1.xml │ ├── Maven__org_apache_hadoop_hadoop_yarn_common_2_5_1.xml │ ├── Maven__org_apache_hbase_hbase_annotations_1_2_3.xml │ ├── Maven__org_apache_hbase_hbase_client_1_2_3.xml │ ├── Maven__org_apache_hbase_hbase_common_1_2_3.xml │ ├── Maven__org_apache_hbase_hbase_protocol_1_2_3.xml │ ├── Maven__org_apache_htrace_htrace_core_3_1_0_incubating.xml │ ├── Maven__org_apache_httpcomponents_httpclient_4_2_5.xml │ ├── Maven__org_apache_httpcomponents_httpcore_4_2_4.xml │ ├── Maven__org_apache_kafka_kafka_2_10_0_8_2_0.xml │ ├── Maven__org_apache_kafka_kafka_clients_0_8_2_0.xml │ ├── Maven__org_apache_zookeeper_zookeeper_3_4_5.xml │ ├── Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml │ ├── Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml │ ├── Maven__org_hamcrest_hamcrest_core_1_3.xml │ ├── Maven__org_jboss_netty_netty_3_2_2_Final.xml │ ├── Maven__org_jruby_jcodings_jcodings_1_0_8.xml │ ├── Maven__org_jruby_joni_joni_2_1_2.xml │ ├── Maven__org_mortbay_jetty_jetty_util_6_1_26.xml │ ├── Maven__org_scala_lang_scala_library_2_10_4.xml │ ├── Maven__org_slf4j_jcl_over_slf4j_1_7_12.xml │ ├── Maven__org_slf4j_log4j_over_slf4j_1_7_7.xml │ ├── Maven__org_slf4j_slf4j_api_1_7_7.xml │ ├── Maven__org_springframework_spring_2_5_6.xml │ ├── Maven__org_tukaani_xz_1_0.xml │ ├── Maven__org_xerial_snappy_snappy_java_1_1_1_6.xml │ └── Maven__xmlenc_xmlenc_0_52.xml ├── misc.xml ├── modules.xml ├── thriftCompiler.xml ├── uiDesigner.xml └── workspace.xml ├── .travis.yml ├── README.md ├── build ├── carrygo.build.iml ├── pom.xml └── src │ └── main │ ├── assembly │ ├── dev.xml │ └── prod.xml │ ├── bin │ ├── start.sh │ └── stop.sh │ └── conf │ ├── dev │ ├── carrygo-conf.xml │ ├── carrygo.properties │ ├── env.sh │ └── logback.xml │ └── prod │ ├── carrygo-conf.xml │ ├── carrygo.properties │ ├── env.sh │ └── logback.xml ├── carrygo.iml ├── common ├── carrygo.common.iml ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── abc │ │ └── carrygo │ │ └── common │ │ ├── Constants.java │ │ ├── conf │ │ ├── Configured.java │ │ └── SpringConf.java │ │ ├── factory │ │ └── ListeningExecutorFactory.java │ │ ├── hbase │ │ ├── Constant.java │ │ ├── HBaseCell.java │ │ ├── HBaseClient.java │ │ ├── HBaseConnector.java │ │ └── HBaseTableAdmin.java │ │ ├── kafka │ │ ├── KafkaAcceptor.java │ │ ├── KafkaAddress.java │ │ ├── KafkaConnectors.java │ │ ├── KafkaConsumer.java │ │ ├── KafkaMessage.java │ │ ├── KafkaPublisher.java │ │ └── RecordPartitioner.java │ │ ├── serde │ │ └── ProtobufSerde.java │ │ ├── utils │ │ ├── ConcurrentUtil.java │ │ ├── ConfigLoader.java │ │ └── ReflectionUtil.java │ │ ├── writer │ │ └── AbstractWriter.java │ │ └── zookeeper │ │ └── ZkAddress.java │ └── test │ └── java │ └── com │ └── abc │ └── carrgo │ └── common │ └── utils │ └── TestUtils.java ├── install_protobuf.sh ├── parser ├── carrygo.parser.iml ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── abc │ │ └── carrygo │ │ └── parser │ │ ├── AbstractEventFilter.java │ │ ├── EventService.java │ │ ├── common │ │ ├── EventFeedManager.java │ │ ├── entity │ │ │ ├── Event.java │ │ │ └── EventFeed.java │ │ └── util │ │ │ ├── DataLock.java │ │ │ ├── EntryParser.java │ │ │ └── ParserUtil.java │ │ ├── kafka │ │ ├── PublishCount.java │ │ ├── feed │ │ │ ├── KafkaFeed.java │ │ │ └── KafkaFeedTask.java │ │ ├── manager │ │ │ └── KafkaFeedManager.java │ │ ├── mysql │ │ │ ├── MySQLDDLEvent.java │ │ │ └── MySQLDMLEvent.java │ │ ├── parse │ │ │ ├── KafkaEntryParser.java │ │ │ └── KafkaEventFilter.java │ │ └── service │ │ │ ├── KafkaEventHandler.java │ │ │ ├── KafkaEventService.java │ │ │ ├── KafkaFeedProcessor.java │ │ │ └── KafkaFeedProcessorBuilder.java │ │ └── server │ │ └── CarryServer.java │ └── resources │ ├── carrygo-conf.xml │ ├── carrygo.properties │ └── logback.xml ├── pom.xml ├── protocol ├── carrgo.protocol.iml ├── carrygo.protocol.iml ├── pom.xml └── src │ └── main │ └── proto │ └── SQLProtocol.proto └── store ├── carrygo.store.iml ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── abc │ │ └── carrygo │ │ └── store │ │ ├── AbstractWriterProcessor.java │ │ ├── hbase │ │ ├── HBaseWriter.java │ │ ├── HBaseWriterProcessor.java │ │ ├── HBaseWriterProcessorBuilder.java │ │ ├── MessageReceived.java │ │ └── service │ │ │ └── HBaseWriterService.java │ │ └── server │ │ └── WriterServer.java └── resources │ ├── carrygo-conf.xml │ ├── carrygo.properties │ └── logback.xml └── test └── java └── com └── abc └── carrygo └── store └── hbase ├── HBaseWriterBase.java ├── HBaseWriterMultiThreadTest.java └── HBaseWriterTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | logs 3 | 4 | # eclipse 5 | .classpath 6 | .project 7 | .settings 8 | 9 | # idea 10 | .idea 11 | *.iml 12 | *.ipr 13 | *.iws 14 | 15 | # maven 16 | *.versionsBackup 17 | 18 | # OS X 19 | .DS_Store 20 | 21 | # log 22 | *.log 23 | log.* 24 | log 25 | 26 | -------------------------------------------------------------------------------- /.idea/.name: -------------------------------------------------------------------------------- 1 | carrygo -------------------------------------------------------------------------------- /.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 65 | -------------------------------------------------------------------------------- /.idea/copyright/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /.idea/dictionaries/plin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__ch_qos_logback_logback_classic_1_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__ch_qos_logback_logback_core_1_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_alibaba_druid_1_0_11.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_alibaba_fastjson_1_1_35.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_alibaba_otter_canal_client_1_0_20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_alibaba_otter_canal_common_1_0_20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_alibaba_otter_canal_protocol_1_0_20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_github_sgroschupf_zkclient_0_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_github_stephenc_findbugs_findbugs_annotations_1_3_9_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_code_findbugs_jsr305_1_3_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_guava_guava_12_0_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_guava_guava_18_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_4_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_google_protobuf_protobuf_java_2_5_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_jcraft_jsch_0_1_42.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_thoughtworks_paranamer_paranamer_2_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__com_yammer_metrics_metrics_core_2_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_beanutils_commons_beanutils_1_7_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_beanutils_commons_beanutils_core_1_8_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_cli_commons_cli_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_codec_commons_codec_1_9.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_collections_commons_collections_3_2_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_configuration_commons_configuration_1_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_digester_commons_digester_1_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_el_commons_el_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_httpclient_commons_httpclient_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_io_commons_io_2_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__commons_net_commons_net_3_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_netty_netty_3_6_2_Final.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__io_netty_netty_all_4_0_23_Final.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_activation_activation_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_xml_bind_jaxb_api_2_2_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__junit_junit_4_12.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__log4j_log4j_1_2_17.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_jpountz_lz4_lz4_1_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__net_sf_jopt_simple_jopt_simple_3_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_avro_avro_1_7_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_compress_1_4_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_lang3_3_3_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_commons_commons_math3_3_1_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_api_api_asn1_api_1_0_0_M20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_api_api_util_1_0_0_M20.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_server_apacheds_i18n_2_0_0_M15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_directory_server_apacheds_kerberos_codec_2_0_0_M15.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_annotations_2_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_auth_2_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_common_2_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_mapreduce_client_core_2_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_api_2_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hadoop_hadoop_yarn_common_2_5_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hbase_hbase_annotations_1_2_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hbase_hbase_client_1_2_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hbase_hbase_common_1_2_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_hbase_hbase_protocol_1_2_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_htrace_htrace_core_3_1_0_incubating.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_2_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_2_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_kafka_kafka_2_10_0_8_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_kafka_kafka_clients_0_8_2_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_apache_zookeeper_zookeeper_3_4_5.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_core_asl_1_9_13.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_codehaus_jackson_jackson_mapper_asl_1_9_13.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_3.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_jboss_netty_netty_3_2_2_Final.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_jruby_jcodings_jcodings_1_0_8.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_jruby_joni_joni_2_1_2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_mortbay_jetty_jetty_util_6_1_26.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_scala_lang_scala_library_2_10_4.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_jcl_over_slf4j_1_7_12.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_log4j_over_slf4j_1_7_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_springframework_spring_2_5_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_tukaani_xz_1_0.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__org_xerial_snappy_snappy_java_1_1_1_6.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/libraries/Maven__xmlenc_xmlenc_0_52.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | Android Lint 22 | 23 | 24 | Data flow issuesJava 25 | 26 | 27 | Groovy 28 | 29 | 30 | Initialization issuesJava 31 | 32 | 33 | J2ME issuesJava 34 | 35 | 36 | Java 37 | 38 | 39 | JavaScript 40 | 41 | 42 | Memory issuesJava 43 | 44 | 45 | Performance issuesJava 46 | 47 | 48 | Potentially confusing code constructsJavaScript 49 | 50 | 51 | Threading issuesGroovy 52 | 53 | 54 | Threading issuesJava 55 | 56 | 57 | 58 | 59 | Android 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 1.7 83 | 84 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 100 | 101 | 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/thriftCompiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/uiDesigner.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | sudo: required 3 | language: java 4 | jdk: openjdk7 5 | cache: 6 | directories: 7 | - $HOME/.m2 8 | before_install: 9 | - ./install_protobuf.sh 10 | install: 11 | - mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V 12 | group: stable 13 | os: linux 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | carrygo 2 | =========== 3 | 4 | [![Build Status](https://travis-ci.org/linpelvis/carrygo.svg?branch=master)](https://travis-ci.org/linpelvis/carrygo) 5 | [![Code Health](https://landscape.io/github/linpelvis/carrygo/master/landscape.svg?style=flat)](https://landscape.io/github/linpelvis/carrygo/master) 6 | [![Requirements Status](https://requires.io/github/linpelvis/carrygo/requirements.svg?branch=master)](https://requires.io/github/linpelvis/carrygo/requirements/?branch=master) 7 | 8 | 9 | 主要作用是同步mysql数据库至hbase，kudu等存储系统。 10 | 11 | 12 | 13 | # 功能 # 14 | 15 | - canal日志解析，kafka持久化消息 16 | - 消费kafka消息同步至hbase等存储引擎 17 | - mysql的数据同步支持DDL、DML至hbase 18 | 19 | 20 | # 原理 # 21 | 22 | carrygo解析及存储服务主要有两个模块，分别是parser和store。 23 | 24 | ### CarryServer ### 25 | 1. 基于canal解析mysql的binlog，批量获取message的entry 26 | 2. 过滤出需同步的db，并解析数据流，同步到kafka中 27 | 28 | ### WriterServer ### 29 | 1. 基于topic接受kafka的消息 30 | 2. HBaseWriter解析数据并存储 31 | 32 | # 相关资源 # 33 | canal：https://github.com/alibaba/canal 34 | 35 | # TODO # 36 | 1. CarryServer服务不可用导致queue数据丢失，存储batchId至zookeeper并记录消费的batchId，丢失就rollback。 37 | 2. 添加CarrygoManager，CarrgoMonitor服务。 38 | 3. 添加管理控制台。 39 | 40 | 41 | # 问题反馈 # 42 | 1. 邮箱：linpelvis@gmail.com 43 | 2. issue：https://github.com/linpelvis/carrygo/issues 44 | 45 | -------------------------------------------------------------------------------- /build/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | carrygo 7 | com.abc.carrygo 8 | 1.0-SNAPSHOT 9 | ../pom.xml 10 | 11 | 4.0.0 12 | 13 | carrygo.build 14 | 15 | 16 | 17 | 18 | com.abc.carrygo 19 | carrygo.parser 20 | 1.0-SNAPSHOT 21 | 22 | 23 | 24 | com.abc.carrygo 25 | carrygo.store 26 | 1.0-SNAPSHOT 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | dev 35 | 36 | true 37 | 38 | 39 | carrygo 40 | dev.xml 41 | 42 | 43 | 44 | prod 45 | 46 | carrygo 47 | prod.xml 48 | 49 | 50 | 51 | 52 | 53 | ${project.finalName} 54 | 55 | 56 | maven-assembly-plugin 57 | 58 | 59 | false 60 | 61 | src/main/assembly/${assembly.descriptor.file} 62 | 63 | 64 | 65 | 66 | make-assembly 67 | package 68 | 69 | single 70 | 71 | 72 | 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /build/src/main/assembly/dev.xml: -------------------------------------------------------------------------------- 1 | 4 | dist 5 | 6 | tar.gz 7 | 8 | true 9 | 10 | 11 | . 12 | / 13 | 14 | README* 15 | 16 | 17 | 18 | ./src/main/bin 19 | bin 20 | 21 | **/* 22 | 23 | 0755 24 | 25 | 26 | ./src/main/conf/dev 27 | /conf 28 | 29 | **/* 30 | 31 | 32 | 33 | ./src/main/resources 34 | /conf 35 | 36 | **/* 37 | 38 | 39 | 40 | 41 | 42 | lib 43 | runtime 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /build/src/main/assembly/prod.xml: -------------------------------------------------------------------------------- 1 | 4 | dist 5 | 6 | tar.gz 7 | 8 | true 9 | 10 | 11 | . 12 | / 13 | 14 | README* 15 | 16 | 17 | 18 | ./src/main/bin 19 | bin 20 | 21 | **/* 22 | 23 | 0755 24 | 25 | 26 | ./src/main/conf/prod 27 | /conf 28 | 29 | **/* 30 | 31 | 32 | 33 | ./src/main/resources 34 | /conf 35 | 36 | **/* 37 | 38 | 39 | 40 | 41 | 42 | lib 43 | runtime 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /build/src/main/bin/start.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linpelvis/carrygo/deedd9d4841cc3edede2ec6c4f7cc60bc335c2d8/build/src/main/bin/start.sh -------------------------------------------------------------------------------- /build/src/main/bin/stop.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linpelvis/carrygo/deedd9d4841cc3edede2ec6c4f7cc60bc335c2d8/build/src/main/bin/stop.sh -------------------------------------------------------------------------------- /build/src/main/conf/dev/carrygo-conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | classpath:carrygo.properties 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /build/src/main/conf/dev/carrygo.properties: -------------------------------------------------------------------------------- 1 | [kafka] 2 | kafka.ip = 127.0.0.1 3 | kafka.port = 9092 4 | kafka.topic = test 5 | 6 | [zookeeper] 7 | zookeeper.destination = example 8 | zookeeper.ip = 127.0.0.1 9 | zookeeper.port = 2181 10 | 11 | -------------------------------------------------------------------------------- /build/src/main/conf/dev/env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | SERVER_LIST="carry writer" 4 | 5 | CARRY_MAIN_CLASS="com.abc.carrygo.parser.server.CarryServer" 6 | WRITER_MAIN_CLASS="com.abc.carrygo.store.server.WriterServer" 7 | 8 | 9 | JAVA_OPTS="-server -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+HeapDumpOnOutOfMemoryError -Xloggc:gc.log -XX:+UseConcMarkSweepGC" 10 | JAVA_OPTS="$JAVA_OPTS -XX:MaxPermSize=512m" 11 | -------------------------------------------------------------------------------- /build/src/main/conf/dev/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | log 6 | 7 | 9 | log.%d{yyyy-MM-dd}.%i 10 | 11 | 12 | 500MB 13 | 14 | 15 | 3 16 | 17 | 18 | 19 | UTF-8 20 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /build/src/main/conf/prod/carrygo-conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | classpath:carrygo.properties 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /build/src/main/conf/prod/carrygo.properties: -------------------------------------------------------------------------------- 1 | [kafka] 2 | kafka.ip = 127.0.0.1 3 | kafka.port = 9092 4 | kafka.topic = test 5 | 6 | [zookeeper] 7 | zookeeper.destination = example 8 | zookeeper.ip = 127.0.0.1 9 | zookeeper.port = 2181 10 | 11 | -------------------------------------------------------------------------------- /build/src/main/conf/prod/env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | SERVER_LIST="carry writer" 4 | 5 | CARRY_MAIN_CLASS="com.abc.carrygo.parser.server.CarryServer" 6 | WRITER_MAIN_CLASS="com.abc.carrygo.store.server.WriterServer" 7 | 8 | 9 | JAVA_OPTS="-server -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+HeapDumpOnOutOfMemoryError -Xloggc:gc.log -XX:+UseConcMarkSweepGC" 10 | JAVA_OPTS="$JAVA_OPTS -XX:MaxPermSize=512m" 11 | -------------------------------------------------------------------------------- /build/src/main/conf/prod/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | log 6 | 7 | 9 | log.%d{yyyy-MM-dd}.%i 10 | 11 | 12 | 500MB 13 | 14 | 15 | 3 16 | 17 | 18 | 19 | UTF-8 20 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /carrygo.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | carrygo 7 | com.abc.carrygo 8 | 1.0-SNAPSHOT 9 | ../pom.xml 10 | 11 | 4.0.0 12 | 13 | carrygo.common 14 | jar 15 | 16 | 17 | 18 | 19 | com.abc.carrygo 20 | carrygo.protocol 21 | 1.0-SNAPSHOT 22 | 23 | 24 | 25 | org.apache.hbase 26 | hbase-client 27 | 28 | 29 | 30 | org.apache.kafka 31 | kafka-clients 32 | 33 | 34 | 35 | org.apache.kafka 36 | kafka_2.10 37 | 38 | 39 | 40 | com.alibaba 41 | druid 42 | 43 | 44 | 45 | org.apache.commons 46 | commons-lang3 47 | 48 | 49 | 50 | com.alibaba.otter 51 | canal.client 52 | 53 | 54 | 55 | ch.qos.logback 56 | logback-classic 57 | runtime 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/Constants.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common; 2 | 3 | /** 4 | * Created by plin on 10/12/16. 5 | */ 6 | public final class Constants { 7 | public static final int CAPACITY = 10000; 8 | public static final int THREAD_RETRY_TIMES = 3; 9 | public static final int THREAD_RETRY_INTERVAL = 1000; 10 | 11 | public static final String KEY_ZK_PARENT = "zookeeper.znode.parent"; 12 | public static final String KEY_ZK_QUORUM = "hbase.zookeeper.quorum"; 13 | public static final String KEY_ZK_CLIENTPORT = "hbase.zookeeper.clientPort"; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/conf/Configured.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.conf; 2 | 3 | import com.abc.carrygo.common.kafka.KafkaAddress; 4 | import com.abc.carrygo.common.utils.ConfigLoader; 5 | import com.abc.carrygo.common.zookeeper.ZkAddress; 6 | 7 | /** 8 | * Created by plin on 10/8/16. 9 | */ 10 | public class Configured { 11 | public static ZkAddress zkAddress; 12 | public static KafkaAddress kafkaAddress; 13 | 14 | public static String kafkaTopic; 15 | 16 | static { 17 | ConfigLoader configLoader = new ConfigLoader("carrygo.properties"); 18 | 19 | zkAddress = new ZkAddress( 20 | configLoader.getValue("zookeeper.destination"), 21 | configLoader.getValue("zookeeper.ip"), 22 | configLoader.getValue("zookeeper.port")); 23 | 24 | kafkaAddress = new KafkaAddress( 25 | configLoader.getValue("kafka.ip"), 26 | configLoader.getValue("kafka.port")); 27 | 28 | kafkaTopic = configLoader.getValue("kafka.topic"); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/conf/SpringConf.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.conf; 2 | 3 | import com.abc.carrygo.common.kafka.KafkaAddress; 4 | import com.abc.carrygo.common.zookeeper.ZkAddress; 5 | import org.springframework.context.ApplicationContext; 6 | import org.springframework.context.support.FileSystemXmlApplicationContext; 7 | 8 | /** 9 | * Created by plin on 10/8/16. 10 | */ 11 | public class SpringConf { 12 | private static final String path = "server/src/main/resources/carrygo-conf.xml"; 13 | 14 | public static ZkAddress zkAddress; 15 | public static KafkaAddress kafkaAddress; 16 | public static ApplicationContext context; 17 | 18 | static { 19 | context = new FileSystemXmlApplicationContext(path); 20 | zkAddress = (ZkAddress) context.getBean("zkAddress"); 21 | kafkaAddress = (KafkaAddress) context.getBean("kafkaAddress"); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/factory/ListeningExecutorFactory.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.factory; 2 | 3 | import com.abc.carrygo.common.utils.ConcurrentUtil; 4 | import com.google.common.util.concurrent.ForwardingListeningExecutorService; 5 | import com.google.common.util.concurrent.ListeningExecutorService; 6 | import org.apache.commons.lang3.StringUtils; 7 | 8 | import java.util.concurrent.ExecutorService; 9 | import java.util.concurrent.Executors; 10 | import java.util.concurrent.ThreadFactory; 11 | 12 | public class ListeningExecutorFactory extends ForwardingListeningExecutorService { 13 | 14 | private static final String DEFAULT_THREAD_FACTORY_NAME = "carrygo"; 15 | private static final int DEFAULT_THREADS; 16 | 17 | static { 18 | DEFAULT_THREADS = Math.max(1, 19 | Runtime.getRuntime().availableProcessors() * 2); 20 | } 21 | 22 | private ListeningExecutorService listeningExecutorService; 23 | 24 | public ListeningExecutorFactory() { 25 | this(DEFAULT_THREADS); 26 | } 27 | 28 | public ListeningExecutorFactory(int nThreads, String name) { 29 | this(nThreads > 0 ? nThreads : DEFAULT_THREADS, 30 | ConcurrentUtil.getNamedThreadFactory(StringUtils.isNotEmpty(name) ? name : DEFAULT_THREAD_FACTORY_NAME)); 31 | } 32 | 33 | public ListeningExecutorFactory(int nThreads) { 34 | this(nThreads, DEFAULT_THREAD_FACTORY_NAME); 35 | } 36 | 37 | public ListeningExecutorFactory(int nThreads, ThreadFactory factory) { 38 | this(Executors.newFixedThreadPool(nThreads, factory)); 39 | } 40 | 41 | public ListeningExecutorFactory(ExecutorService executorService) { 42 | this.listeningExecutorService = ConcurrentUtil.getListeningExecutorService(executorService); 43 | } 44 | 45 | @Override 46 | protected ListeningExecutorService delegate() { 47 | return this.listeningExecutorService; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/hbase/Constant.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.hbase; 2 | 3 | 4 | public class Constant { 5 | public static final String DEFAULT_NULLMODE = "EMPTY_BYTES"; 6 | public static final String DEFAULT_ENCODING = "utf-8"; 7 | public static final int DEFAULT_BATCHROWS = 100; 8 | public static final int MAX_BATCHROWS = 1000; 9 | public static final String DEFAULT_COLUMN_FAMILY = "cf"; 10 | public static final String DEFAULT_QUALIFIER = "unknown"; 11 | } 12 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/hbase/HBaseCell.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.hbase; 2 | 3 | public class HBaseCell { 4 | private String rowKey; 5 | private String colf; 6 | private String col; 7 | private String value; 8 | 9 | public String getRowKey() { 10 | return rowKey; 11 | } 12 | 13 | public void setRowKey(String rowKey) { 14 | this.rowKey = rowKey; 15 | } 16 | 17 | public String getColf() { 18 | return colf; 19 | } 20 | 21 | public void setColf(String colf) { 22 | this.colf = colf; 23 | } 24 | 25 | public String getCol() { 26 | return col; 27 | } 28 | 29 | public void setCol(String col) { 30 | this.col = col; 31 | } 32 | 33 | public String getValue() { 34 | return value; 35 | } 36 | 37 | public void setValue(String value) { 38 | this.value = value; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/hbase/HBaseClient.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.hbase; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import org.apache.hadoop.hbase.client.Delete; 5 | import org.apache.hadoop.hbase.client.Put; 6 | import org.apache.hadoop.hbase.client.Table; 7 | import org.apache.hadoop.hbase.util.Bytes; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.io.IOException; 12 | import java.util.ArrayList; 13 | import java.util.Arrays; 14 | import java.util.List; 15 | 16 | public class HBaseClient { 17 | private final Logger log = LoggerFactory.getLogger(this.getClass()); 18 | 19 | public void put(String tableName, HBaseCell cell) throws Exception { 20 | log.info("put cell, tableName:{}, cell:{}", tableName, JSONObject.toJSON(cell)); 21 | 22 | Table table = HBaseConnector.getTable(tableName); 23 | 24 | Put p = new Put(Bytes.toBytes(cell.getRowKey())); 25 | p.addColumn(Bytes.toBytes(cell.getColf()), 26 | Bytes.toBytes(cell.getCol()), 27 | Bytes.toBytes(cell.getValue())); 28 | 29 | put(Arrays.asList(p), table); 30 | } 31 | 32 | public void put(String tableName, List cells) throws Exception { 33 | log.info("put cells, tableName:{}, cells:{}", tableName, JSONObject.toJSON(cells)); 34 | 35 | Table table = HBaseConnector.getTable(tableName); 36 | 37 | List puts = new ArrayList<>(); 38 | if (cells != null && cells.size() > 0) { 39 | 40 | for (HBaseCell cell : cells) { 41 | 42 | Put p = new Put(Bytes.toBytes(cell.getRowKey())); 43 | p.addColumn(Bytes.toBytes(cell.getColf()), 44 | Bytes.toBytes(cell.getCol() == null ? Constant.DEFAULT_QUALIFIER : cell.getCol()), 45 | Bytes.toBytes(cell.getValue() == null ? "" : cell.getValue())); 46 | puts.add(p); 47 | } 48 | 49 | put(puts, table); 50 | } //if(cells != null && cells.size() > 0) 51 | } 52 | 53 | 54 | public void delete(String tableName, String rowKey) throws Exception { 55 | log.info("delete rowKey, tableName:{}, rowKey:{}", tableName, rowKey); 56 | 57 | Table table = HBaseConnector.getTable(tableName); 58 | 59 | Delete delete = new Delete(rowKey.getBytes()); 60 | 61 | try { 62 | table.delete(delete); 63 | } catch (Exception e) { 64 | log.error("delete rowKey " + delete.toString() + " failed.", e); 65 | throw e; 66 | } finally { 67 | if (table != null) { 68 | try { 69 | table.close(); 70 | } catch (Exception e) { 71 | log.error("close table" + table.getName() + " failed.", e); 72 | throw e; 73 | } 74 | } 75 | } 76 | } 77 | 78 | public void delete(String tableName, List rowKeys) throws Exception { 79 | log.info("delete rowKeys, tableName:{}, rowKeys:{}", tableName, rowKeys); 80 | 81 | Table table = HBaseConnector.getTable(tableName); 82 | 83 | List deletes = new ArrayList<>(); 84 | 85 | for (String rowKey : rowKeys) { 86 | Delete delete = new Delete(rowKey.getBytes()); 87 | deletes.add(delete); 88 | } 89 | 90 | delete(deletes, table); 91 | } 92 | 93 | private void delete(List deletes, Table table) throws IOException { 94 | try { 95 | table.delete(deletes); 96 | } catch (Exception e) { 97 | log.error("delete rowKeys " + deletes.toString() + " failed.", e); 98 | throw e; 99 | } finally { 100 | if (table != null) { 101 | try { 102 | table.close(); 103 | } catch (Exception e) { 104 | log.error("close table" + table.getName() + " failed.", e); 105 | throw e; 106 | } 107 | } 108 | } 109 | } 110 | 111 | private void put(List puts, Table table) throws IOException { 112 | try { 113 | table.put(puts); 114 | } catch (Exception e) { 115 | log.error("put table " + table.getName() + " failed.", e); 116 | throw e; 117 | } finally { 118 | if (table != null) { 119 | try { 120 | table.close(); 121 | } catch (Exception e) { 122 | log.error("close table " + table.getName() + " failed.", e); 123 | throw e; 124 | } 125 | } 126 | } 127 | } 128 | 129 | public void close() throws Exception { 130 | HBaseConnector.close(); 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/hbase/HBaseConnector.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.hbase; 2 | 3 | import com.abc.carrygo.common.zookeeper.ZkAddress; 4 | import org.apache.hadoop.conf.Configuration; 5 | import org.apache.hadoop.hbase.HBaseConfiguration; 6 | import org.apache.hadoop.hbase.TableName; 7 | import org.apache.hadoop.hbase.client.Admin; 8 | import org.apache.hadoop.hbase.client.Connection; 9 | import org.apache.hadoop.hbase.client.ConnectionFactory; 10 | import org.apache.hadoop.hbase.client.Table; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.io.IOException; 15 | 16 | /** 17 | * Created by plin on 10/13/16. 18 | */ 19 | public class HBaseConnector { 20 | private static final Logger log = LoggerFactory.getLogger(HBaseConnector.class); 21 | 22 | private static Connection connection; 23 | 24 | private static Admin admin; 25 | 26 | public static void init(ZkAddress zkAddress) throws IOException { 27 | //configuration.set("hbase.zookeeper.quorum","10.10.3.181,10.10.3.182,10.10.3.183"); 28 | //configuration.set("hbase.zookeeper.property.clientPort","2181"); 29 | System.setProperty("hadoop.home.dir", "/"); 30 | 31 | Configuration configuration = HBaseConfiguration.create(); 32 | configuration.set("hbase.zookeeper.quorum", zkAddress.getZkIp()); 33 | configuration.set("hbase.zookeeper.property.clientPort", zkAddress.getZkPort()); 34 | configuration.set("zookeeper.znode.parent", "/hbase"); 35 | 36 | configuration.set("hbase.client.retries.number", "3"); // default 35 37 | configuration.set("hbase.rpc.timeout", "10000"); // default 60 secs 38 | configuration.set("hbase.rpc.shortoperation.timeout", "5000"); // default 10 secs 39 | 40 | 41 | // if (confs != null && confs.size() > 0) { 42 | // for (String key : confs.keySet()) { 43 | // configuration.set(key, confs.get(key)); 44 | // } 45 | // } 46 | 47 | try { 48 | //hbase-client中protobuf版本与hbase不兼容会导致无法连接的问题 49 | connection = ConnectionFactory.createConnection(configuration); 50 | admin = connection.getAdmin(); 51 | } catch (Exception e) { 52 | log.error("hbase init exception occured.", e); 53 | throw e; 54 | } 55 | } 56 | 57 | public static void close() throws Exception { 58 | try { 59 | if (connection != null) { 60 | connection.close(); 61 | } 62 | } catch (Exception e) { 63 | log.error("hbase connection close exception occured.", e); 64 | throw e; 65 | } 66 | } 67 | 68 | public static Admin getAdmin() { 69 | return admin; 70 | } 71 | 72 | public static Connection getConnection() { 73 | return connection; 74 | } 75 | 76 | public static Table getTable(String tableName) throws Exception { 77 | Table table = null; 78 | try { 79 | if (connection != null) { 80 | table = connection.getTable(TableName.valueOf(tableName)); 81 | } 82 | } catch (Exception e) { 83 | log.error("get table " + tableName + " failed.", e); 84 | throw e; 85 | } 86 | return table; 87 | } 88 | } 89 | 90 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/kafka/KafkaAcceptor.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.kafka; 2 | 3 | import com.abc.carrygo.common.factory.ListeningExecutorFactory; 4 | import com.abc.carrygo.common.utils.ReflectionUtil; 5 | import com.abc.carrygo.common.writer.AbstractWriter; 6 | import com.abc.carrygo.common.zookeeper.ZkAddress; 7 | import kafka.consumer.KafkaStream; 8 | import kafka.javaapi.consumer.ConsumerConnector; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.concurrent.TimeUnit; 16 | 17 | /** 18 | * Created by plin on 9/26/16. 19 | */ 20 | public class KafkaAcceptor { 21 | private static final Logger log = LoggerFactory.getLogger(KafkaAcceptor.class); 22 | private static final String DEFAULT_EXECUTOR_NAME = "kafka-acceptor"; 23 | private static final int numThreads = 1; 24 | private ConsumerConnector consumerConnector; 25 | private ListeningExecutorFactory executor; 26 | private String topic; 27 | 28 | public KafkaAcceptor(ZkAddress zkAddress, String topic) { 29 | this(zkAddress, topic, zkAddress.getZkDestination()); 30 | } 31 | 32 | public KafkaAcceptor(ZkAddress zkAddress, String topic, String groupId) { 33 | this.consumerConnector = KafkaConnectors.newConsumer(zkAddress, groupId); 34 | this.topic = topic; 35 | } 36 | 37 | 38 | public void run(int nThreads, AbstractWriter writer, String theClass) throws Exception { 39 | Class aClass = Class.forName(theClass); 40 | 41 | Map topicCountMap = new HashMap<>(); 42 | 43 | // one topic one thread 44 | topicCountMap.put(topic, numThreads); 45 | 46 | try { 47 | Map>> consumerMap = consumerConnector.createMessageStreams(topicCountMap); 48 | 49 | for (String topic : consumerMap.keySet()) { 50 | log.info("start acceptor topic:{}", topic); 51 | 52 | List> streams = consumerMap.get(topic); 53 | 54 | executor = new ListeningExecutorFactory(nThreads, DEFAULT_EXECUTOR_NAME); 55 | 56 | for (final KafkaStream stream : streams) { 57 | executor.submit((Runnable) ReflectionUtil.newInstance(aClass, writer, stream)); 58 | } 59 | } 60 | 61 | } catch (Exception e) { 62 | log.error("acceptor msg failed.", e); 63 | close(); 64 | 65 | throw e; 66 | } 67 | 68 | } 69 | 70 | 71 | public void close() { 72 | 73 | if (executor != null) { 74 | try { 75 | Thread.sleep(10000); 76 | } catch (InterruptedException e) { 77 | e.printStackTrace(); 78 | } 79 | executor.shutdown(); 80 | } 81 | 82 | if (consumerConnector != null) { 83 | consumerConnector.shutdown(); 84 | } 85 | 86 | try { 87 | if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) { 88 | log.info("Timed out waiting for consumer threads to shut down, exiting uncleanly"); 89 | executor.shutdownNow(); 90 | } 91 | } catch (InterruptedException e) { 92 | log.info("Interrupted during shutdown, exiting uncleanly"); 93 | executor.shutdownNow(); 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/kafka/KafkaAddress.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.kafka; 2 | 3 | import com.google.common.base.Joiner; 4 | 5 | /** 6 | * Created by plin on 9/30/16. 7 | */ 8 | public class KafkaAddress { 9 | private static final Joiner PATH_JOINER = Joiner.on(":").skipNulls(); 10 | 11 | private String kafkaIp; 12 | private String kafkaPort; 13 | 14 | // use spring conf 15 | public KafkaAddress() { 16 | } 17 | 18 | public KafkaAddress(String kafkaIp, String kafkaPort) { 19 | this.kafkaIp = kafkaIp; 20 | this.kafkaPort = kafkaPort; 21 | } 22 | 23 | public void setKafkaPort(String kafkaPort) { 24 | this.kafkaPort = kafkaPort; 25 | } 26 | 27 | public void setKafkaIp(String kafkaIp) { 28 | this.kafkaIp = kafkaIp; 29 | } 30 | 31 | public String getKafkaServers() { 32 | return PATH_JOINER.join(kafkaIp, kafkaPort); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/kafka/KafkaConnectors.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.kafka; 2 | 3 | import com.abc.carrygo.common.zookeeper.ZkAddress; 4 | import kafka.consumer.ConsumerConfig; 5 | import kafka.javaapi.consumer.ConsumerConnector; 6 | import org.apache.kafka.clients.producer.KafkaProducer; 7 | import org.apache.kafka.clients.producer.Producer; 8 | import org.apache.kafka.clients.producer.ProducerConfig; 9 | 10 | import java.util.Properties; 11 | 12 | /** 13 | * Created by plin on 9/26/16. 14 | */ 15 | public class KafkaConnectors { 16 | private static volatile KafkaConnectors instance = new KafkaConnectors(); 17 | 18 | private KafkaConnectors() { 19 | } 20 | 21 | public static KafkaConnectors getInstance() { 22 | return instance; 23 | } 24 | 25 | public static Producer newProducer(KafkaAddress kafkaAddress) { 26 | return initProducer(kafkaAddress); 27 | } 28 | 29 | private static Producer initProducer(KafkaAddress kafkaAddress) { 30 | Properties props = new Properties(); 31 | //分组partition, compression 32 | props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaAddress.getKafkaServers()); 33 | props.setProperty(ProducerConfig.ACKS_CONFIG, "1"); 34 | // props.setProperty(ProducerConfig.RETRIES_CONFIG, "3"); 35 | // props.setProperty(ProducerConfig.BATCH_SIZE_CONFIG, "16384"); 36 | // props.setProperty(ProducerConfig.LINGER_MS_CONFIG, "1"); 37 | // props.setProperty(ProducerConfig.BUFFER_MEMORY_CONFIG, "33554432"); 38 | // props.setProperty(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG, "300"); 39 | // props.setProperty(ProducerConfig.TIMEOUT_CONFIG, "300"); 40 | // props.setProperty(ProducerConfig.RETRY_BACKOFF_MS_CONFIG, "10000"); 41 | // props.setProperty(ProducerConfig.RECONNECT_BACKOFF_MS_CONFIG, "10000"); 42 | props.setProperty(ProducerConfig.COMPRESSION_TYPE_CONFIG, "snappy"); 43 | 44 | props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, 45 | "org.apache.kafka.common.serialization.StringSerializer"); 46 | props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, 47 | "org.apache.kafka.common.serialization.ByteArraySerializer"); 48 | 49 | // props.setProperty("partitioner.class", "com.abc.carrygo.common.kafka.RecordPartitioner"); 50 | 51 | return new KafkaProducer<>(props); 52 | } 53 | 54 | public static ConsumerConnector newConsumer(ZkAddress zkAddress, String groupId) { 55 | return initConsumer(zkAddress, groupId); 56 | } 57 | 58 | private static ConsumerConnector initConsumer(ZkAddress zkAddress, String groupId) { 59 | return kafka.consumer.Consumer.createJavaConsumerConnector( 60 | createConsumerConfig(zkAddress.getZkServers(), groupId, true)); 61 | } 62 | 63 | private static ConsumerConfig createConsumerConfig(String servers, String groupId, boolean isBeginning) { 64 | Properties props = new Properties(); 65 | props.put("zookeeper.connect", servers); 66 | props.put("group.id", groupId); 67 | props.put("zookeeper.session.timeout.ms", "3000"); 68 | props.put("zookeeper.sync.time.ms", "1000"); 69 | props.put("auto.commit.enable", "true"); 70 | props.put("auto.commit.interval.ms", "1000"); 71 | // props.put("queued.max.message.chunks", "50"); 72 | // props.put("rebalance.max.retries", "5"); 73 | if (isBeginning) { 74 | props.put("auto.offset.reset", "smallest"); 75 | } else { 76 | props.put("auto.offset.reset", "latest"); 77 | } 78 | 79 | return new ConsumerConfig(props); 80 | } 81 | 82 | 83 | } 84 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/kafka/KafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.kafka; 2 | 3 | 4 | import kafka.consumer.KafkaStream; 5 | 6 | /** 7 | * Created by plin on 10/17/16. 8 | */ 9 | public class KafkaConsumer implements Runnable { 10 | protected KafkaStream stream; 11 | 12 | public KafkaConsumer(KafkaStream stream) { 13 | this.stream = stream; 14 | } 15 | 16 | @Override 17 | public void run() { 18 | 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/kafka/KafkaMessage.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.kafka; 2 | 3 | import com.abc.carrygo.protocol.SQLEventEntry; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * Created by plin on 9/29/16. 9 | */ 10 | public class KafkaMessage implements Serializable { 11 | private String topic; 12 | private String key; 13 | private SQLEventEntry.SQLEntry sqlEntry; 14 | 15 | public KafkaMessage(String topic, String key, SQLEventEntry.SQLEntry sqlEntry) { 16 | this.topic = topic; 17 | this.key = key; 18 | this.sqlEntry = sqlEntry; 19 | } 20 | 21 | public SQLEventEntry.SQLEntry getSqlEntry() { 22 | return sqlEntry; 23 | } 24 | 25 | 26 | public String getTopic() { 27 | return topic; 28 | } 29 | 30 | public String getKey() { 31 | return key; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/kafka/KafkaPublisher.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.kafka; 2 | 3 | import com.abc.carrygo.common.Constants; 4 | import com.abc.carrygo.common.serde.ProtobufSerde; 5 | import org.apache.commons.lang.StringUtils; 6 | import org.apache.kafka.clients.producer.Producer; 7 | import org.apache.kafka.clients.producer.ProducerRecord; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.List; 12 | 13 | /** 14 | * Created by plin on 9/26/16. 15 | */ 16 | public final class KafkaPublisher { 17 | private static final Logger log = LoggerFactory.getLogger(KafkaPublisher.class); 18 | 19 | private final Producer producer; 20 | 21 | public KafkaPublisher(Producer producer) { 22 | this.producer = producer; 23 | } 24 | 25 | public void send(List kafkaMessages) throws Exception { 26 | // TODO: 9/29/16 压缩数据v0.2 27 | if (kafkaMessages == null || kafkaMessages.isEmpty()) { 28 | log.error("kafkaMessages is empty."); 29 | } 30 | 31 | for (KafkaMessage kafkaMessage : kafkaMessages) { 32 | if (StringUtils.isNotEmpty(kafkaMessage.getTopic()) || 33 | StringUtils.isNotEmpty(kafkaMessage.getKey())) { 34 | ProducerRecord producerRecord = new ProducerRecord<>(kafkaMessage.getTopic(), 35 | kafkaMessage.getKey(), 36 | ProtobufSerde.toBinary(kafkaMessage.getSqlEntry())); 37 | 38 | send(producerRecord); 39 | } 40 | 41 | } 42 | } 43 | 44 | private void send(ProducerRecord record) throws Exception { 45 | // log.info("start send msg, topic:{}, key:{}.", record.topic(), record.key()); 46 | 47 | int times = 0; 48 | 49 | while (true) { 50 | try { 51 | producer.send(record); 52 | return; 53 | } catch (Exception e) { 54 | times += 1; 55 | if (times > Constants.THREAD_RETRY_TIMES) { 56 | throw e; 57 | } else { 58 | try { 59 | Thread.sleep(Constants.THREAD_RETRY_INTERVAL); 60 | } catch (InterruptedException e1) { 61 | throw e1; 62 | } 63 | } 64 | } 65 | } 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/kafka/RecordPartitioner.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.kafka; 2 | 3 | import kafka.producer.Partitioner; 4 | import kafka.utils.VerifiableProperties; 5 | 6 | /** 7 | * Created by plin on 9/29/16. 8 | */ 9 | public class RecordPartitioner implements Partitioner { 10 | 11 | public RecordPartitioner(VerifiableProperties verifiableProperties) { 12 | } 13 | 14 | @Override 15 | public int partition(Object key, int numPartitions) { 16 | try { 17 | int partitionNum = Integer.parseInt((String) key); 18 | return Math.abs(partitionNum % numPartitions); 19 | } catch (Exception e) { 20 | return Math.abs(key.hashCode() % numPartitions); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/serde/ProtobufSerde.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.serde; 2 | 3 | import com.google.protobuf.Message; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | /** 8 | * Created by plin on 10/20/16. 9 | */ 10 | public class ProtobufSerde { 11 | public static final Logger log = LoggerFactory.getLogger(ProtobufSerde.class); 12 | 13 | public static byte[] toBinary(Message msg) { 14 | try { 15 | return msg.toByteArray(); 16 | } catch (Exception e) { 17 | log.warn("Serialize protobuf to binary failed.", e); 18 | return null; 19 | } 20 | } 21 | 22 | public static T fromBinary(byte[] bytes, T obj) { 23 | try { 24 | Message message = obj.newBuilderForType().mergeFrom(bytes).buildPartial(); 25 | return (T) message; 26 | } catch (Exception e) { 27 | log.warn("Deserialize protobuf from binary failed.", e); 28 | return null; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/utils/ConcurrentUtil.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.utils; 2 | 3 | import com.google.common.util.concurrent.ListeningExecutorService; 4 | import com.google.common.util.concurrent.MoreExecutors; 5 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 6 | 7 | import java.util.concurrent.ExecutorService; 8 | import java.util.concurrent.Executors; 9 | import java.util.concurrent.ScheduledExecutorService; 10 | import java.util.concurrent.ThreadFactory; 11 | 12 | public class ConcurrentUtil { 13 | 14 | private static Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { 15 | public void uncaughtException(Thread t, Throwable e) { 16 | System.err.print("thread has an error!!!" + "thread:" + t.getName() + "." + e); 17 | } 18 | }; 19 | 20 | public static ThreadFactory getNamedThreadFactory(String name) { 21 | return getNamedThreadFactory(name, false); 22 | } 23 | 24 | public static ThreadFactory getNamedThreadFactory(String name, boolean isDaemon) { 25 | ThreadFactoryBuilder builder = new ThreadFactoryBuilder(); 26 | builder 27 | .setDaemon(isDaemon) 28 | .setNameFormat(name + "-thread-%d") 29 | .setUncaughtExceptionHandler(handler); 30 | return builder.build(); 31 | } 32 | 33 | public static ListeningExecutorService getListeningExecutorService( 34 | int nThreads, ThreadFactory threadFactory) { 35 | ExecutorService es = Executors.newFixedThreadPool(nThreads, threadFactory); 36 | return getListeningExecutorService(es); 37 | } 38 | 39 | public static ListeningExecutorService getListeningExecutorService( 40 | ExecutorService executorService) { 41 | return MoreExecutors.listeningDecorator(executorService); 42 | } 43 | 44 | public static ScheduledExecutorService getScheduledExecutorService(int corePoolSize) { 45 | ThreadFactory threadFactory = getNamedThreadFactory("SCHEDULED"); 46 | return getScheduledExecutorService(corePoolSize, threadFactory); 47 | } 48 | 49 | public static ScheduledExecutorService getScheduledExecutorService( 50 | int corePoolSize, ThreadFactory threadFactory) { 51 | return Executors.newScheduledThreadPool(corePoolSize, threadFactory); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/utils/ConfigLoader.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.utils; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.FileNotFoundException; 7 | import java.io.IOException; 8 | import java.io.InputStream; 9 | import java.util.Properties; 10 | 11 | /** 12 | * Created by plin on 10/8/16. 13 | */ 14 | public class ConfigLoader { 15 | private static final Logger log = LoggerFactory.getLogger(ConfigLoader.class); 16 | private InputStream inputStream; 17 | private Properties prop; 18 | 19 | public ConfigLoader(String propFileName) { 20 | prop = new Properties(); 21 | try { 22 | inputStream = getClass().getClassLoader().getResourceAsStream(propFileName); 23 | if (inputStream != null) { 24 | prop.load(inputStream); 25 | } else { 26 | throw new FileNotFoundException("property file '" + propFileName + "' not found in the classpath"); 27 | } 28 | } catch (IOException e) { 29 | log.error("Exception: " + e); 30 | } finally { 31 | if (inputStream != null) { 32 | try { 33 | inputStream.close(); 34 | } catch (IOException e) { 35 | e.printStackTrace(); 36 | } 37 | } 38 | } 39 | } 40 | 41 | public String getValue(String key) { 42 | if (prop.containsKey(key)) { 43 | return prop.getProperty(key); 44 | } else { 45 | return ""; 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/utils/ReflectionUtil.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.utils; 2 | 3 | import java.lang.reflect.Constructor; 4 | import java.lang.reflect.Method; 5 | 6 | /** 7 | * Created by plin on 10/18/16. 8 | */ 9 | public class ReflectionUtil { 10 | 11 | public static T newInstance(Class clazz, Object... constructorArgs) { 12 | if (clazz == null) { 13 | return null; 14 | } 15 | 16 | T result; 17 | int argLen = constructorArgs == null ? 0 : constructorArgs.length; 18 | 19 | Class[] parameterTypes = new Class[argLen]; 20 | for (int i = 0; i < argLen; i++) { 21 | parameterTypes[i] = constructorArgs[i].getClass(); 22 | } 23 | 24 | Constructor constructor = null; 25 | 26 | try { 27 | constructor = clazz.getDeclaredConstructor(parameterTypes); 28 | 29 | if (!constructor.isAccessible()) { 30 | constructor.setAccessible(true); 31 | } 32 | 33 | result = constructor.newInstance(constructorArgs); 34 | } catch (Exception e) { 35 | throw new RuntimeException(e); 36 | } 37 | 38 | return result; 39 | } 40 | 41 | public static T invokeMethod(Object object, String methodName, Object... methodArgs) { 42 | if (object == null) { 43 | return null; 44 | } 45 | 46 | T result = null; 47 | 48 | Class clazz = object.getClass(); 49 | 50 | int argLen = methodArgs == null ? 0 : methodArgs.length; 51 | 52 | Class[] methodTypes = new Class[argLen]; 53 | for (int i = 0; i < argLen; i++) { 54 | methodTypes[i] = methodArgs[i].getClass(); 55 | } 56 | 57 | try { 58 | Method method = clazz.getDeclaredMethod(methodName, methodTypes); 59 | 60 | if (method != null) { 61 | if (!method.isAccessible()) { 62 | method.setAccessible(true); 63 | } 64 | 65 | result = (T) method.invoke(object, methodArgs); 66 | } 67 | } catch (Exception e) { 68 | throw new RuntimeException(e); 69 | } 70 | 71 | return result; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/writer/AbstractWriter.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.writer; 2 | 3 | /** 4 | * Created by plin on 10/20/16. 5 | */ 6 | public abstract class AbstractWriter { 7 | public abstract void close() throws Exception; 8 | 9 | protected void process(Writer writer) { 10 | writer.message(); 11 | } 12 | 13 | protected interface Writer { 14 | void message(); 15 | } 16 | 17 | protected interface WriterTransaction { 18 | 19 | void alter(T t); 20 | 21 | void create(T t); 22 | 23 | void put(F t); 24 | 25 | void del(F t); 26 | 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /common/src/main/java/com/abc/carrygo/common/zookeeper/ZkAddress.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.zookeeper; 2 | 3 | import com.google.common.base.Joiner; 4 | 5 | /** 6 | * Created by plin on 9/30/16. 7 | */ 8 | public class ZkAddress { 9 | private static final Joiner PATH_JOINER = Joiner.on(":").skipNulls(); 10 | 11 | private String zkIp; 12 | private String zkPort; 13 | private String zkDestination; 14 | 15 | // use spring conf 16 | public ZkAddress() { 17 | } 18 | 19 | public ZkAddress(String zkDestination, String zkIp, String zkPort) { 20 | this.zkDestination = zkDestination; 21 | this.zkIp = zkIp; 22 | this.zkPort = zkPort; 23 | } 24 | 25 | public String getZkIp() { 26 | return zkIp; 27 | } 28 | 29 | public void setZkIp(String zkIp) { 30 | this.zkIp = zkIp; 31 | } 32 | 33 | public String getZkPort() { 34 | return zkPort; 35 | } 36 | 37 | public void setZkPort(String zkPort) { 38 | this.zkPort = zkPort; 39 | } 40 | 41 | public String getZkServers() { 42 | return PATH_JOINER.join(zkIp, zkPort); 43 | } 44 | 45 | public String getZkDestination() { 46 | return zkDestination; 47 | } 48 | 49 | public void setZkDestination(String zkDestination) { 50 | this.zkDestination = zkDestination; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /common/src/test/java/com/abc/carrgo/common/utils/TestUtils.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrgo.common.utils; 2 | 3 | /** 4 | * Created by plin on 10/20/16. 5 | */ 6 | public class TestUtils { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /install_protobuf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -ex 3 | wget https://github.com/google/protobuf/releases/download/v2.4.1/protobuf-2.4.1.tar.gz 4 | tar -xzvf protobuf-2.4.1.tar.gz 5 | cd protobuf-2.4.1 && ./configure --prefix=/usr && make && sudo make install 6 | -------------------------------------------------------------------------------- /parser/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | carrygo 7 | com.abc.carrygo 8 | 1.0-SNAPSHOT 9 | ../pom.xml 10 | 11 | 4.0.0 12 | 13 | carrygo.parser 14 | jar 15 | 16 | 17 | 18 | 19 | com.abc.carrygo 20 | carrygo.common 21 | 1.0-SNAPSHOT 22 | 23 | 24 | 25 | ch.qos.logback 26 | logback-classic 27 | runtime 28 | 29 | 30 | 31 | com.github.sgroschupf 32 | zkclient 33 | 34 | 35 | 36 | org.slf4j 37 | log4j-over-slf4j 38 | 39 | 40 | 41 | com.google.guava 42 | guava 43 | 44 | 45 | 46 | 47 | com.google.protobuf 48 | protobuf-java 49 | 2.4.1 50 | 51 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/AbstractEventFilter.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Created by plin on 9/20/16. 7 | */ 8 | public abstract class AbstractEventFilter { 9 | 10 | protected List process(Event event) throws Exception { 11 | return event.parser(); 12 | } 13 | 14 | protected interface Event { 15 | List parser() throws Exception; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/EventService.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser; 2 | 3 | import com.abc.carrygo.parser.common.util.EntryParser; 4 | import com.alibaba.otter.canal.client.CanalConnector; 5 | import com.alibaba.otter.canal.protocol.Message; 6 | import org.apache.log4j.MDC; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.util.Assert; 10 | 11 | /** 12 | * Created by plin on 9/19/16. 13 | */ 14 | public class EventService { 15 | 16 | protected static final int BATCH_SIZE = 5 * 1024; 17 | private static final Logger log = LoggerFactory.getLogger(EventService.class); 18 | protected static Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() { 19 | public void uncaughtException(Thread t, Throwable e) { 20 | System.err.print("thread has an error!!!" + "thread:" + t.getName() + "." + e); 21 | } 22 | }; 23 | protected volatile boolean running = false; 24 | protected CanalConnector connector; 25 | protected String destination; 26 | protected Thread thread = null; 27 | 28 | public EventService(String destination) { 29 | this(destination, null); 30 | } 31 | 32 | public EventService(String destination, CanalConnector connector) { 33 | this.destination = destination; 34 | this.connector = connector; 35 | } 36 | 37 | public void setConnector(CanalConnector connector) { 38 | this.connector = connector; 39 | } 40 | 41 | public void startParse() { 42 | Assert.notNull(connector, "connector is null"); 43 | 44 | thread = new Thread(new Runnable() { 45 | @Override 46 | public void run() { 47 | process(); 48 | } 49 | }); 50 | 51 | thread.setUncaughtExceptionHandler(handler); 52 | thread.start(); 53 | 54 | running = true; 55 | } 56 | 57 | public void stopParse() { 58 | if (!running) { 59 | return; 60 | } 61 | running = false; 62 | 63 | if (thread != null) { 64 | try { 65 | thread.join(); 66 | } catch (InterruptedException e) { 67 | // 68 | log.warn("interrupt event client!"); 69 | } 70 | } 71 | 72 | MDC.remove("destination"); 73 | } 74 | 75 | public void process() { 76 | while (running) { 77 | try { 78 | MDC.put("destination", destination); 79 | //protobuf与canal中protobuf版本不一致会导致无法connect问题 80 | //启动服务时, 连接不稳定,待查canal是否存在问题 81 | 82 | log.info("start connect."); 83 | connector.connect(); 84 | 85 | log.info("start subscribe."); 86 | connector.subscribe(); 87 | 88 | log.info("start parse event."); 89 | 90 | while (running) { 91 | Message message = connector.getWithoutAck(BATCH_SIZE); 92 | long batchId = message.getId(); 93 | 94 | parseAndPublish(message, batchId); 95 | } 96 | } catch (Exception e) { 97 | log.error("process error!", e); 98 | } finally { 99 | connector.disconnect(); 100 | MDC.remove("destination"); 101 | } 102 | } 103 | } 104 | 105 | protected void parseAndPublish(Message message, long batchId) { 106 | try { 107 | int size = message.getEntries().size(); 108 | if (batchId == -1 || size == 0) { 109 | // log.debug("message is null"); 110 | } else { 111 | // EntryParser.printSummary(message, batchId, size); 112 | // EntryParser.printTransaction(message.getEntries()); 113 | 114 | EntryParser.parseRowData(message.getEntries(), null); 115 | 116 | // publish 117 | } 118 | connector.ack(batchId); 119 | } catch (Exception e) { 120 | log.error("rollback msg, batchId:{}", batchId, e); 121 | connector.rollback(batchId); 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/common/EventFeedManager.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.common; 2 | 3 | import com.abc.carrygo.common.Constants; 4 | import com.abc.carrygo.parser.common.entity.EventFeed; 5 | 6 | import java.util.concurrent.ArrayBlockingQueue; 7 | import java.util.concurrent.ConcurrentHashMap; 8 | 9 | /** 10 | * Created by plin on 10/19/16. 11 | */ 12 | public class EventFeedManager { 13 | protected static volatile ConcurrentHashMap> blockingQueueMap; 14 | 15 | static { 16 | blockingQueueMap = new ConcurrentHashMap<>(); 17 | } 18 | 19 | protected void add(String type, EventFeed eventFeed) throws InterruptedException { 20 | ArrayBlockingQueue queue = new ArrayBlockingQueue(Constants.CAPACITY); 21 | queue.put(eventFeed); 22 | 23 | blockingQueueMap.putIfAbsent(type, queue); 24 | 25 | ArrayBlockingQueue actualQueue = blockingQueueMap.get(type); 26 | if (actualQueue != queue) { 27 | actualQueue.put(eventFeed); 28 | } 29 | 30 | } 31 | 32 | protected ArrayBlockingQueue get(String type) { 33 | ArrayBlockingQueue blockingQueue = blockingQueueMap.get(type); 34 | 35 | return blockingQueue; 36 | } 37 | 38 | protected boolean isExists(String type) { 39 | return blockingQueueMap.containsKey(type); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/common/entity/Event.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.common.entity; 2 | 3 | import com.abc.carrygo.protocol.SQLEventEntry; 4 | import com.alibaba.otter.canal.protocol.CanalEntry; 5 | 6 | 7 | /** 8 | * Created by plin on 10/11/16. 9 | */ 10 | public class Event { 11 | //ddl 执行的sql 12 | String sql; 13 | 14 | //dml 执行的数据 15 | CanalEntry.RowData rowData; 16 | 17 | String schemaName; 18 | 19 | String tableName; 20 | 21 | //DML/DDL 22 | SQLEventEntry.EntryType entryType; 23 | 24 | //create, delete, alter... 25 | CanalEntry.EventType eventType; 26 | 27 | public String getSchemaName() { 28 | return schemaName; 29 | } 30 | 31 | public void setSchemaName(String schemaName) { 32 | this.schemaName = schemaName; 33 | } 34 | 35 | public SQLEventEntry.EntryType getEntryType() { 36 | return entryType; 37 | } 38 | 39 | public void setEntryType(SQLEventEntry.EntryType entryType) { 40 | this.entryType = entryType; 41 | } 42 | 43 | public CanalEntry.EventType getEventType() { 44 | return eventType; 45 | } 46 | 47 | public void setEventType(CanalEntry.EventType eventType) { 48 | this.eventType = eventType; 49 | } 50 | 51 | public String getTableName() { 52 | return tableName; 53 | } 54 | 55 | public void setTableName(String tableName) { 56 | this.tableName = tableName; 57 | } 58 | 59 | public CanalEntry.RowData getRowData() { 60 | return rowData; 61 | } 62 | 63 | public void setRowData(CanalEntry.RowData rowData) { 64 | this.rowData = rowData; 65 | } 66 | 67 | public String getSql() { 68 | return sql; 69 | } 70 | 71 | public void setSql(String sql) { 72 | this.sql = sql; 73 | } 74 | } 75 | 76 | 77 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/common/entity/EventFeed.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.common.entity; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Created by plin on 10/12/16. 7 | */ 8 | public class EventFeed { 9 | 10 | public final long batchId; 11 | public final List event; 12 | 13 | public EventFeed(long batchId, List event) { 14 | this.batchId = batchId; 15 | this.event = event; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/common/util/DataLock.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.common.util; 2 | 3 | /** 4 | * Created by plin on 10/11/16. 5 | */ 6 | public class DataLock { 7 | 8 | public static final Object writeDataLock = new Object(); 9 | 10 | public static final Object readDataLock = new Object(); 11 | } 12 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/common/util/EntryParser.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.common.util; 2 | 3 | import com.abc.carrygo.parser.common.entity.Event; 4 | import com.abc.carrygo.protocol.SQLEventEntry.EntryType; 5 | import com.alibaba.otter.canal.protocol.CanalEntry; 6 | import com.alibaba.otter.canal.protocol.Message; 7 | import com.google.protobuf.InvalidProtocolBufferException; 8 | import org.apache.commons.lang.SystemUtils; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.springframework.util.CollectionUtils; 12 | 13 | import java.text.SimpleDateFormat; 14 | import java.util.*; 15 | 16 | /** 17 | * Created by plin on 9/20/16. 18 | */ 19 | public class EntryParser { 20 | protected static final String SEP = SystemUtils.LINE_SEPARATOR; 21 | protected static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; 22 | private static final Logger log = LoggerFactory.getLogger(EntryParser.class); 23 | 24 | private static String context_format = null; 25 | private static String row_format = null; 26 | private static String transaction_format = null; 27 | 28 | static { 29 | context_format = SEP + "****************************************************" + SEP; 30 | context_format += "* Batch Id: [{}] ,count : [{}] , memsize : [{}] , Time : {}" + SEP; 31 | context_format += "* Start : [{}] " + SEP; 32 | context_format += "* End : [{}] " + SEP; 33 | context_format += "****************************************************" + SEP; 34 | row_format = SEP 35 | + "----------------> binlog[{}:{}] , name[{},{}] , eventType : {} , executeTime : {} , delay : {}ms" 36 | + SEP; 37 | transaction_format = SEP + "================> binlog[{}:{}] , executeTime : {} , delay : {}ms" + SEP; 38 | } 39 | 40 | public static void printSummary(Message message, long batchId, int size) { 41 | long memorySize = 0; 42 | for (CanalEntry.Entry entry : message.getEntries()) { 43 | memorySize += entry.getHeader().getEventLength(); 44 | } 45 | 46 | String startPosition = null; 47 | String endPosition = null; 48 | if (!CollectionUtils.isEmpty(message.getEntries())) { 49 | startPosition = buildPositionForDump(message.getEntries().get(0)); 50 | endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1)); 51 | } 52 | 53 | SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT); 54 | log.info(context_format, batchId, size, memorySize, format.format(new Date()), startPosition, 55 | endPosition); 56 | } 57 | 58 | private static String buildPositionForDump(CanalEntry.Entry entry) { 59 | long time = entry.getHeader().getExecuteTime(); 60 | Date date = new Date(time); 61 | SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT); 62 | return entry.getHeader().getLogfileName() + ":" + entry.getHeader().getLogfileOffset() + ":" 63 | + entry.getHeader().getExecuteTime() + "(" + format.format(date) + ")"; 64 | } 65 | 66 | public static void printTransaction(List entrys) { 67 | for (CanalEntry.Entry entry : entrys) { 68 | long executeTime = entry.getHeader().getExecuteTime(); 69 | long delayTime = new Date().getTime() - executeTime; 70 | 71 | if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN 72 | || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) { 73 | if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN) { 74 | CanalEntry.TransactionBegin begin = null; 75 | try { 76 | begin = CanalEntry.TransactionBegin.parseFrom(entry.getStoreValue()); 77 | } catch (InvalidProtocolBufferException e) { 78 | throw new RuntimeException("parse event has an error , data:" + entry.toString(), e); 79 | } 80 | // 打印事务头信息，执行的线程id，事务耗时 81 | log.info(transaction_format, 82 | entry.getHeader().getLogfileName(), 83 | String.valueOf(entry.getHeader().getLogfileOffset()), 84 | String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime)); 85 | log.info(" BEGIN ----> Thread id: {}", begin.getThreadId()); 86 | } else if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) { 87 | CanalEntry.TransactionEnd end = null; 88 | try { 89 | end = CanalEntry.TransactionEnd.parseFrom(entry.getStoreValue()); 90 | } catch (InvalidProtocolBufferException e) { 91 | throw new RuntimeException("parse event has an error , data:" + entry.toString(), e); 92 | } 93 | // 打印事务提交信息，事务id 94 | log.info("----------------\n"); 95 | log.info(" END ----> transaction id: {}", end.getTransactionId()); 96 | log.info(transaction_format, 97 | entry.getHeader().getLogfileName(), 98 | String.valueOf(entry.getHeader().getLogfileOffset()), 99 | String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime)); 100 | } 101 | } 102 | } 103 | } 104 | 105 | public static Map> parseRowData(List entries, List schemas) throws InterruptedException { 106 | Map> eventSchemaFeeds = new HashMap<>(); 107 | 108 | for (CanalEntry.Entry entry : entries) { 109 | 110 | if (entry.getEntryType() == CanalEntry.EntryType.ROWDATA) { 111 | CanalEntry.RowChange rowChange = null; 112 | try { 113 | rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); 114 | } catch (Exception e) { 115 | throw new RuntimeException("parse event has an error , data:" + entry.toString(), e); 116 | } 117 | 118 | CanalEntry.EventType eventType = rowChange.getEventType(); 119 | 120 | // log.info(row_format, 121 | // entry.getHeader().getLogfileName(), 122 | // String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(), 123 | // entry.getHeader().getTableName(), eventType, 124 | // String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf("")); 125 | 126 | if (eventType == CanalEntry.EventType.QUERY) { 127 | continue; 128 | } 129 | 130 | String schema = entry.getHeader().getSchemaName(); 131 | if (schemas != null && !schemas.isEmpty() && !schemas.contains(schema)) { 132 | continue; 133 | } 134 | 135 | 136 | String table = entry.getHeader().getTableName(); 137 | 138 | List events; 139 | 140 | 141 | if (eventSchemaFeeds.containsKey(schema)) { 142 | events = eventSchemaFeeds.get(schema); 143 | } else { 144 | events = new ArrayList<>(); 145 | } 146 | 147 | if (rowChange.getIsDdl()) { 148 | Event event = new Event(); 149 | event.setEntryType(EntryType.DDL); 150 | event.setSql(rowChange.getSql()); 151 | event.setEventType(eventType); 152 | event.setTableName(table); 153 | event.setSchemaName(schema); 154 | 155 | events.add(event); 156 | 157 | synchronized (DataLock.writeDataLock) { 158 | eventSchemaFeeds.put(schema, events); 159 | } 160 | } 161 | 162 | for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) { 163 | Event event = new Event(); 164 | event.setEntryType(EntryType.DML); 165 | event.setRowData(rowData); 166 | event.setEventType(eventType); 167 | event.setTableName(table); 168 | event.setSchemaName(schema); 169 | 170 | events.add(event); 171 | 172 | synchronized (DataLock.writeDataLock) { 173 | eventSchemaFeeds.put(schema, events); 174 | } 175 | } 176 | } 177 | } 178 | 179 | return eventSchemaFeeds; 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/PublishCount.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka; 2 | 3 | import java.util.concurrent.atomic.AtomicLong; 4 | 5 | /** 6 | * Created by plin on 10/12/16. 7 | */ 8 | public class PublishCount { 9 | private static volatile AtomicLong id = new AtomicLong(0); 10 | 11 | public synchronized static void put(long batchId) { 12 | if (id.get() == 0) { 13 | id.set(batchId); 14 | } 15 | } 16 | 17 | public static void set(long batchId) { 18 | id.set(batchId); 19 | } 20 | 21 | public static long get() { 22 | return id.get(); 23 | } 24 | 25 | public static boolean check() { 26 | if (PublishCount.get() != 0) { 27 | return Boolean.TRUE; 28 | } 29 | 30 | return Boolean.FALSE; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/feed/KafkaFeed.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka.feed; 2 | 3 | import com.abc.carrygo.parser.common.entity.Event; 4 | import com.abc.carrygo.parser.common.entity.EventFeed; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * Created by plin on 10/19/16. 10 | */ 11 | public class KafkaFeed extends EventFeed { 12 | public final String topic; 13 | 14 | public KafkaFeed(long batchId, 15 | String topic, 16 | List events) { 17 | super(batchId, events); 18 | 19 | this.topic = topic; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/feed/KafkaFeedTask.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka.feed; 2 | 3 | 4 | import com.abc.carrygo.common.Constants; 5 | import com.abc.carrygo.parser.kafka.manager.KafkaFeedManager; 6 | import com.abc.carrygo.parser.kafka.service.KafkaEventHandler; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * Created by plin on 10/19/16. 12 | */ 13 | public class KafkaFeedTask implements Runnable { 14 | private static final Logger log = LoggerFactory.getLogger(KafkaFeedTask.class); 15 | 16 | private KafkaEventHandler handler; 17 | private KafkaFeedManager feedManager; 18 | private String topic; 19 | 20 | public KafkaFeedTask(String topic) { 21 | this.topic = topic; 22 | } 23 | 24 | @Override 25 | public void run() { 26 | while (true) { 27 | try { 28 | 29 | KafkaFeed event = feedManager.getEventFeed(topic); 30 | 31 | if (event != null) { 32 | handler.extractAndPublish(event); 33 | } else { 34 | Thread.sleep(Constants.THREAD_RETRY_INTERVAL); 35 | } 36 | 37 | } catch (Exception e) { 38 | log.error("publish failed.", e); 39 | } 40 | } 41 | } 42 | 43 | public void setFeedManager(KafkaFeedManager feedManager) { 44 | this.feedManager = feedManager; 45 | } 46 | 47 | public void setHandler(KafkaEventHandler handler) { 48 | this.handler = handler; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/manager/KafkaFeedManager.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka.manager; 2 | 3 | import com.abc.carrygo.parser.common.EventFeedManager; 4 | import com.abc.carrygo.parser.kafka.PublishCount; 5 | import com.abc.carrygo.parser.kafka.feed.KafkaFeed; 6 | 7 | import java.util.concurrent.ArrayBlockingQueue; 8 | 9 | /** 10 | * Created by plin on 10/19/16. 11 | */ 12 | public class KafkaFeedManager extends EventFeedManager { 13 | 14 | public void fill(KafkaFeed feed) throws InterruptedException { 15 | try { 16 | 17 | add(feed.topic, feed); 18 | 19 | //Record the batchId 20 | PublishCount.put(feed.batchId); 21 | } catch (Exception e) { 22 | throw e; 23 | } 24 | } 25 | 26 | public KafkaFeed getEventFeed(String schema) { 27 | ArrayBlockingQueue eventMessages = get(schema); 28 | if (eventMessages == null || eventMessages.isEmpty()) { 29 | return null; 30 | } 31 | 32 | KafkaFeed eventFeed = (KafkaFeed) eventMessages.poll(); 33 | if (eventFeed == null) { 34 | return null; 35 | } 36 | 37 | PublishCount.set(eventFeed.batchId); 38 | 39 | return eventFeed; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/mysql/MySQLDDLEvent.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka.mysql; 2 | 3 | import com.abc.carrygo.protocol.SQLEventEntry.DDLEntry; 4 | import com.abc.carrygo.protocol.SQLEventEntry.KeyWord; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.List; 9 | 10 | import static com.abc.carrygo.parser.common.util.ParserUtil.statementParser; 11 | 12 | 13 | /** 14 | * Created by plin on 9/26/16. 15 | */ 16 | public class MySQLDDLEvent { 17 | 18 | private static final Logger log = LoggerFactory.getLogger(MySQLDDLEvent.class); 19 | 20 | 21 | //DDL:创建表结构, 新增add, modify(改变表的类型,暂不考虑), 修改change或者删除drop列. 22 | public static List filter(final String schemaName, 23 | final String tableName, 24 | final String sql, 25 | final KeyWord keyWord) { 26 | List ddlEntries = statementParser(schemaName, tableName, keyWord, sql); 27 | 28 | log.debug("parser ddlOperas:{}", ddlEntries); 29 | 30 | return ddlEntries; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/mysql/MySQLDMLEvent.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka.mysql; 2 | 3 | import com.abc.carrygo.protocol.SQLEventEntry.DMLEntry; 4 | import com.abc.carrygo.protocol.SQLEventEntry.KeyWord; 5 | import com.alibaba.otter.canal.protocol.CanalEntry; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.List; 10 | 11 | import static com.abc.carrygo.parser.common.util.ParserUtil.columnFilter; 12 | 13 | 14 | /** 15 | * Created by plin on 9/26/16. 16 | */ 17 | public class MySQLDMLEvent { 18 | 19 | private static final Logger log = LoggerFactory.getLogger(MySQLDMLEvent.class); 20 | 21 | public static DMLEntry filter(final String schemaName, 22 | final String tableName, 23 | List columns, 24 | final KeyWord keyWord) { 25 | return columnFilter(schemaName, tableName, keyWord, columns); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/parse/KafkaEntryParser.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka.parse; 2 | 3 | import com.abc.carrygo.parser.common.util.EntryParser; 4 | import com.abc.carrygo.protocol.SQLEventEntry.DDLEntry; 5 | import com.abc.carrygo.protocol.SQLEventEntry.DMLEntry; 6 | import com.abc.carrygo.protocol.SQLEventEntry.KeyWord; 7 | import com.alibaba.otter.canal.protocol.CanalEntry; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.List; 12 | 13 | /** 14 | * Created by plin on 10/11/16. 15 | */ 16 | public class KafkaEntryParser extends EntryParser { 17 | private static final Logger log = LoggerFactory.getLogger(KafkaEntryParser.class); 18 | private static final KafkaEventFilter filter = new KafkaEventFilter(); 19 | 20 | public static List parse(String schemaName, 21 | String tableName, 22 | String sql, 23 | CanalEntry.EventType eventType) throws Exception { 24 | List ddlEntries = null; 25 | switch (eventType) { 26 | case CREATE: 27 | ddlEntries = filter.filter(schemaName, tableName, sql, KeyWord.CREATE); 28 | break; 29 | case ALTER: 30 | ddlEntries = filter.filter(schemaName, tableName, sql, KeyWord.ALTER); 31 | break; 32 | default: 33 | //ignore 34 | log.info("parse entry sql, sql:{}", sql); 35 | } 36 | 37 | return ddlEntries; 38 | } 39 | 40 | public static List parse(String schemaName, 41 | String tableName, 42 | CanalEntry.RowData rowData, 43 | CanalEntry.EventType eventType) throws Exception { 44 | List dmlEntries = null; 45 | switch (eventType) { 46 | case DELETE: 47 | dmlEntries = filter.filter(schemaName, 48 | tableName, 49 | rowData.getBeforeColumnsList(), 50 | KeyWord.DELETE); 51 | break; 52 | case INSERT: 53 | dmlEntries = filter.filter(schemaName, 54 | tableName, 55 | rowData.getAfterColumnsList(), 56 | KeyWord.INSERT); 57 | break; 58 | case UPDATE: 59 | dmlEntries = filter.filter(schemaName, 60 | tableName, 61 | rowData.getAfterColumnsList(), 62 | KeyWord.UPDATE); 63 | break; 64 | default: 65 | //ignore 66 | log.info("parse entry row data, rowData:{}", rowData); 67 | } 68 | 69 | return dmlEntries; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/parse/KafkaEventFilter.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka.parse; 2 | 3 | import com.abc.carrygo.parser.AbstractEventFilter; 4 | import com.abc.carrygo.parser.kafka.mysql.MySQLDDLEvent; 5 | import com.abc.carrygo.parser.kafka.mysql.MySQLDMLEvent; 6 | import com.abc.carrygo.protocol.SQLEventEntry.DDLEntry; 7 | import com.abc.carrygo.protocol.SQLEventEntry.DMLEntry; 8 | import com.abc.carrygo.protocol.SQLEventEntry.KeyWord; 9 | import com.alibaba.otter.canal.protocol.CanalEntry; 10 | 11 | import java.util.Arrays; 12 | import java.util.List; 13 | 14 | /** 15 | * Created by plin on 9/26/16. 16 | */ 17 | public class KafkaEventFilter extends AbstractEventFilter { 18 | 19 | public List filter(final String schemaName, 20 | final String tableName, 21 | final String sql, 22 | final KeyWord keyWord) throws Exception { 23 | return process(new Event() { 24 | @Override 25 | public List parser() throws Exception { 26 | return MySQLDDLEvent.filter(schemaName, tableName, sql, keyWord); 27 | } 28 | }); 29 | } 30 | 31 | public List filter(final String schemaName, 32 | final String tableName, 33 | final List columns, 34 | final KeyWord keyWord) throws Exception { 35 | return process(new Event() { 36 | @Override 37 | public List parser() throws Exception { 38 | DMLEntry entry = MySQLDMLEvent.filter(schemaName, tableName, columns, keyWord); 39 | return Arrays.asList(entry); 40 | } 41 | }); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/service/KafkaEventHandler.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka.service; 2 | 3 | import com.abc.carrygo.common.kafka.KafkaMessage; 4 | import com.abc.carrygo.common.kafka.KafkaPublisher; 5 | import com.abc.carrygo.parser.common.entity.Event; 6 | import com.abc.carrygo.parser.kafka.feed.KafkaFeed; 7 | import com.abc.carrygo.parser.kafka.parse.KafkaEntryParser; 8 | import com.abc.carrygo.protocol.SQLEventEntry.DDLEntry; 9 | import com.abc.carrygo.protocol.SQLEventEntry.DMLEntry; 10 | import com.abc.carrygo.protocol.SQLEventEntry.SQLEntry; 11 | import org.apache.kafka.clients.producer.Producer; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | 18 | /** 19 | * Created by plin on 10/11/16. 20 | */ 21 | public class KafkaEventHandler { 22 | private static final Logger log = LoggerFactory.getLogger(KafkaEventHandler.class); 23 | 24 | private final KafkaPublisher kafkaPublisher; 25 | private final Producer producer; 26 | 27 | public KafkaEventHandler(Producer producer) { 28 | this.producer = producer; 29 | this.kafkaPublisher = new KafkaPublisher(producer); 30 | } 31 | 32 | public void release() { 33 | producer.close(); 34 | } 35 | 36 | 37 | public void extractAndPublish(KafkaFeed feed) throws Exception { 38 | log.info("extractAndPublish, topic:{}, batchId:{}", feed.topic, feed.batchId); 39 | 40 | List kafkaMessages = new ArrayList<>(); 41 | 42 | try { 43 | extract(feed.topic, feed.event, kafkaMessages); 44 | 45 | kafkaPublisher.send(kafkaMessages); 46 | } catch (Exception e) { 47 | log.error("extractAndPublish failed.", e); 48 | 49 | throw e; 50 | } 51 | 52 | } 53 | 54 | private void extract(String schema, List events, List kafkaMessages) throws Exception { 55 | 56 | for (Event event : events) { 57 | SQLEntry.Builder sqlEntry = SQLEntry.newBuilder(); 58 | 59 | switch (event.getEntryType()) { 60 | case DDL: 61 | List ddlEntries = KafkaEntryParser.parse(schema, 62 | event.getTableName(), 63 | event.getSql(), 64 | event.getEventType()); 65 | 66 | if (ddlEntries != null) { 67 | sqlEntry.addAllDdlEntries(ddlEntries); 68 | sqlEntry.setEntry(event.getEntryType()); 69 | } 70 | 71 | break; 72 | case DML: 73 | List dmlEntries = KafkaEntryParser.parse(schema, 74 | event.getTableName(), 75 | event.getRowData(), 76 | event.getEventType()); 77 | 78 | if (dmlEntries != null) { 79 | sqlEntry.addAllDmlEntries(dmlEntries); 80 | sqlEntry.setEntry(event.getEntryType()); 81 | } 82 | 83 | break; 84 | default: 85 | //ignore 86 | } 87 | 88 | if (sqlEntry.hasEntry()) { 89 | String key = schema + event.getEntryType(); 90 | kafkaMessages.add(new KafkaMessage(schema, key, sqlEntry.build())); 91 | } 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/service/KafkaEventService.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka.service; 2 | 3 | import com.abc.carrygo.common.factory.ListeningExecutorFactory; 4 | import com.abc.carrygo.common.kafka.KafkaAddress; 5 | import com.abc.carrygo.common.kafka.KafkaConnectors; 6 | import com.abc.carrygo.parser.EventService; 7 | import com.abc.carrygo.parser.common.entity.Event; 8 | import com.abc.carrygo.parser.kafka.PublishCount; 9 | import com.abc.carrygo.parser.kafka.manager.KafkaFeedManager; 10 | import com.abc.carrygo.parser.kafka.parse.KafkaEntryParser; 11 | import com.alibaba.otter.canal.client.CanalConnectors; 12 | import com.alibaba.otter.canal.protocol.Message; 13 | import com.google.common.base.Splitter; 14 | import com.google.common.util.concurrent.FutureCallback; 15 | import com.google.common.util.concurrent.Futures; 16 | import com.google.common.util.concurrent.ListenableFuture; 17 | import org.apache.kafka.clients.producer.Producer; 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | import java.util.List; 22 | import java.util.Map; 23 | import java.util.concurrent.Callable; 24 | import java.util.concurrent.CountDownLatch; 25 | 26 | 27 | public class KafkaEventService extends EventService { 28 | private static final Logger log = LoggerFactory.getLogger(KafkaEventService.class); 29 | private static final Splitter SPLITTER = Splitter.on(",").omitEmptyStrings(); 30 | private static final String PUBLISH_EXECUTOR_WORK_NAME = "PUBLISH-WORK"; 31 | private static final String PUBLISH_EXECUTOR_LISTENER_NAME = "PUBLISH-LISTENER"; 32 | private static final int EXECUTOR_NUM_THREAD = 1; 33 | private static int default_threads; 34 | private static CountDownLatch latch = new CountDownLatch(1); 35 | private volatile boolean publish_running = false; 36 | private KafkaFeedProcessor processor; 37 | private ListeningExecutorFactory service; 38 | private List topics; 39 | 40 | public KafkaEventService(String servers, 41 | String destination, 42 | KafkaAddress kafkaAddress, 43 | String kafkaTopic) { 44 | super(destination, CanalConnectors.newClusterConnector( 45 | servers, 46 | destination, 47 | "", 48 | "")); 49 | 50 | init(kafkaAddress, kafkaTopic); 51 | } 52 | 53 | public void init(KafkaAddress kafkaAddress, String kafkaTopic) { 54 | this.topics = SPLITTER.splitToList(kafkaTopic); 55 | 56 | 57 | default_threads = Math.max(topics.size(), 58 | Runtime.getRuntime().availableProcessors() * 2); 59 | 60 | Producer producer = KafkaConnectors.newProducer(kafkaAddress); 61 | KafkaEventHandler eventHandler = new KafkaEventHandler(producer); 62 | 63 | KafkaFeedProcessorBuilder builder = new KafkaFeedProcessorBuilder(); 64 | builder 65 | .setEventHandler(eventHandler) 66 | .setFeedManager(new KafkaFeedManager()) 67 | .setEventFactory(new KafkaFeedProcessor(topics)) 68 | .setListeningExecutorFactory(new ListeningExecutorFactory(default_threads, PUBLISH_EXECUTOR_WORK_NAME)); 69 | processor = builder.build(topics); 70 | } 71 | 72 | public void startPublish() { 73 | service = new ListeningExecutorFactory(EXECUTOR_NUM_THREAD, PUBLISH_EXECUTOR_LISTENER_NAME); 74 | final ListenableFuture future = service.submit(new Callable() { 75 | @Override 76 | public Boolean call() throws Exception { 77 | boolean flag = false; 78 | 79 | // do { 80 | // flag = PublishCount.check(); 81 | // 82 | // } while (!flag); 83 | 84 | if (!flag) { 85 | latch.await(); 86 | } 87 | 88 | publish_running = true; 89 | 90 | return Boolean.valueOf(publish_running); 91 | } 92 | }); 93 | 94 | Futures.addCallback(future, new FutureCallback() { 95 | @Override 96 | public void onSuccess(Object result) { 97 | Boolean b = (Boolean) result; 98 | if (b.booleanValue()) { 99 | processor.start(); 100 | } else { 101 | throw new RuntimeException(new Throwable("Publish failed.")); 102 | } 103 | 104 | service.shutdown(); 105 | } 106 | 107 | public void onFailure(Throwable thrown) { 108 | throw new RuntimeException(thrown); 109 | } 110 | }); 111 | } 112 | 113 | 114 | public void stopPublish() { 115 | if (!publish_running) { 116 | return; 117 | } 118 | publish_running = false; 119 | 120 | processor.stop(); 121 | } 122 | 123 | 124 | protected void parseAndPublish(Message message, long batchId) { 125 | try { 126 | int size = message.getEntries().size(); 127 | if (batchId != -1 && size != 0) { 128 | log.info("parseAndPublish, batchId:{}", batchId); 129 | 130 | Map> eventFeedMaps = KafkaEntryParser.parseRowData( 131 | message.getEntries(), 132 | topics); 133 | 134 | // parse event && put event to queue 135 | processor.extract(batchId, eventFeedMaps); 136 | 137 | if (PublishCount.check() && !publish_running) { 138 | latch.countDown(); 139 | } 140 | } 141 | 142 | connector.ack(batchId); 143 | } catch (Exception e) { 144 | long publishBatchId = PublishCount.get(); 145 | if (publishBatchId < batchId) { 146 | log.error("origin batchId:{}", batchId); 147 | 148 | batchId = publishBatchId; 149 | } 150 | 151 | log.error("rollback msg, batchId:{}", batchId, e); 152 | connector.rollback(batchId); 153 | } 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/service/KafkaFeedProcessor.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka.service; 2 | 3 | import com.abc.carrygo.common.factory.ListeningExecutorFactory; 4 | import com.abc.carrygo.parser.common.entity.Event; 5 | import com.abc.carrygo.parser.kafka.feed.KafkaFeed; 6 | import com.abc.carrygo.parser.kafka.feed.KafkaFeedTask; 7 | import com.abc.carrygo.parser.kafka.manager.KafkaFeedManager; 8 | import org.apache.commons.lang3.StringUtils; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | /** 16 | * Created by plin on 10/19/16. 17 | */ 18 | public class KafkaFeedProcessor { 19 | private static final Logger log = LoggerFactory.getLogger(KafkaFeedProcessor.class); 20 | 21 | public List topics; 22 | private ListeningExecutorFactory executor; 23 | private KafkaEventHandler handler; 24 | private KafkaFeedManager feedMgr; 25 | 26 | public KafkaFeedProcessor(List topics) { 27 | this.topics = topics; 28 | } 29 | 30 | public void extract(long batchId, Map> eventMaps) throws InterruptedException { 31 | for (String topic : eventMaps.keySet()) { 32 | 33 | if (StringUtils.isNotEmpty(topic) && topics.contains(topic)) { 34 | KafkaFeed eventFeed = new KafkaFeed(batchId, 35 | topic, 36 | eventMaps.get(topic)); 37 | 38 | log.info("extract, batchId:{}", batchId); 39 | 40 | // TODO: 10/24/16 a topic -> a thread 41 | feedMgr.fill(eventFeed); 42 | 43 | } else { 44 | return; 45 | } 46 | 47 | } 48 | } 49 | 50 | 51 | public void start() { 52 | 53 | for (Object topic : topics) { 54 | log.info("start process task, topic:{}", topic); 55 | 56 | KafkaFeedTask task = new KafkaFeedTask((String) topic); 57 | task.setFeedManager(feedMgr); 58 | task.setHandler(handler); 59 | 60 | executor.execute(task); 61 | } 62 | } 63 | 64 | 65 | public void stop() { 66 | 67 | 68 | try { 69 | executor.shutdown(); 70 | } catch (Exception e) { 71 | log.error("stop error.", e); 72 | 73 | executor.shutdownNow(); 74 | Thread.currentThread().interrupt(); 75 | } finally { 76 | handler.release(); 77 | } 78 | } 79 | 80 | 81 | public void setFeedManager(KafkaFeedManager feedMgr) { 82 | this.feedMgr = feedMgr; 83 | } 84 | 85 | public void setHandler(KafkaEventHandler handler) { 86 | this.handler = handler; 87 | } 88 | 89 | public void setExecutorFactory(ListeningExecutorFactory executor) { 90 | this.executor = executor; 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/kafka/service/KafkaFeedProcessorBuilder.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.kafka.service; 2 | 3 | import com.abc.carrygo.common.factory.ListeningExecutorFactory; 4 | import com.abc.carrygo.parser.kafka.manager.KafkaFeedManager; 5 | import org.springframework.util.Assert; 6 | 7 | import java.util.List; 8 | 9 | /** 10 | * Created by plin on 10/19/16. 11 | */ 12 | public final class KafkaFeedProcessorBuilder { 13 | 14 | private KafkaFeedManager feedManager = null; 15 | private ListeningExecutorFactory listeningExecutorFactory = null; 16 | private KafkaEventHandler eventHandler = null; 17 | private KafkaFeedProcessor processor = null; 18 | 19 | public KafkaFeedProcessorBuilder() { 20 | } 21 | 22 | private static KafkaFeedProcessor build(KafkaFeedProcessorBuilder builder, List topics) { 23 | final KafkaEventHandler eventHandler = builder.eventHandler; 24 | final ListeningExecutorFactory listeningExecutorFactory = builder.listeningExecutorFactory; 25 | final KafkaFeedManager feedManager = builder.feedManager; 26 | 27 | final KafkaFeedProcessor processor = 28 | (builder.processor != null) 29 | ? builder.processor 30 | : new KafkaFeedProcessor(topics); 31 | 32 | processor.setHandler(eventHandler); 33 | processor.setFeedManager(feedManager); 34 | processor.setExecutorFactory(listeningExecutorFactory); 35 | 36 | return processor; 37 | } 38 | 39 | public KafkaFeedProcessorBuilder setFeedManager(KafkaFeedManager feedManager) { 40 | Assert.notNull(feedManager); 41 | 42 | this.feedManager = feedManager; 43 | 44 | return this; 45 | } 46 | 47 | public KafkaFeedProcessorBuilder setListeningExecutorFactory(ListeningExecutorFactory listeningExecutorFactory) { 48 | Assert.notNull(listeningExecutorFactory); 49 | 50 | this.listeningExecutorFactory = listeningExecutorFactory; 51 | 52 | return this; 53 | } 54 | 55 | public KafkaFeedProcessorBuilder setEventFactory(KafkaFeedProcessor processor) { 56 | Assert.notNull(processor); 57 | 58 | this.processor = processor; 59 | 60 | return this; 61 | } 62 | 63 | public KafkaFeedProcessorBuilder setEventHandler(KafkaEventHandler eventHandler) { 64 | Assert.notNull(eventHandler); 65 | 66 | this.eventHandler = eventHandler; 67 | 68 | return this; 69 | } 70 | 71 | public KafkaFeedProcessor build(List topics) { 72 | return build(this, topics); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /parser/src/main/java/com/abc/carrygo/parser/server/CarryServer.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.parser.server; 2 | 3 | import com.abc.carrygo.common.conf.Configured; 4 | import com.abc.carrygo.common.kafka.KafkaAddress; 5 | import com.abc.carrygo.common.zookeeper.ZkAddress; 6 | import com.abc.carrygo.parser.kafka.service.KafkaEventService; 7 | import org.apache.commons.lang.exception.ExceptionUtils; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | /** 12 | * Created by plin on 9/30/16. 13 | */ 14 | public class CarryServer { 15 | private static final Logger log = LoggerFactory.getLogger(CarryServer.class); 16 | private static ZkAddress zkAddress; 17 | private static KafkaAddress kafkaAddress; 18 | private KafkaEventService service; 19 | 20 | @SuppressWarnings("resource") 21 | public static void main(String[] args) { 22 | // ZkAddress zkAddress = SpringConf.zkAddress; 23 | // KafkaAddress kafkaAddress = SpringConf.kafkaAddress; 24 | 25 | zkAddress = Configured.zkAddress; 26 | kafkaAddress = Configured.kafkaAddress; 27 | 28 | final CarryServer server = new CarryServer(); 29 | server.start(); 30 | 31 | Runtime.getRuntime().addShutdownHook(new Thread() { 32 | 33 | public void run() { 34 | try { 35 | 36 | server.stop(); 37 | 38 | } catch (Throwable e) { 39 | log.warn("something goes wrong when stopping carry server:\n{}", ExceptionUtils.getFullStackTrace(e)); 40 | } finally { 41 | log.info("carry server is down."); 42 | } 43 | } 44 | 45 | }); 46 | } 47 | 48 | public void start() { 49 | log.info("start the carry server"); 50 | 51 | service = new KafkaEventService( 52 | zkAddress.getZkServers(), 53 | zkAddress.getZkDestination(), 54 | kafkaAddress, 55 | Configured.kafkaTopic); 56 | 57 | service.startParse(); 58 | service.startPublish(); 59 | } 60 | 61 | public void stop() { 62 | log.info("stop the carry server"); 63 | 64 | service.stopParse(); 65 | service.stopPublish(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /parser/src/main/resources/carrygo-conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 10 | 11 | 12 | 13 | 14 | 15 | classpath:carrygo.properties 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /parser/src/main/resources/carrygo.properties: -------------------------------------------------------------------------------- 1 | [kafka] 2 | kafka.ip = 127.0.0.1 3 | kafka.port = 9092 4 | kafka.topic = test 5 | 6 | [zookeeper] 7 | zookeeper.destination = example 8 | zookeeper.ip = 127.0.0.1 9 | zookeeper.port = 2181 10 | 11 | -------------------------------------------------------------------------------- /parser/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | log 6 | 7 | 9 | log.%d{yyyy-MM-dd}.%i 10 | 11 | 12 | 500MB 13 | 14 | 15 | 3 16 | 17 | 18 | 19 | UTF-8 20 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /protocol/carrgo.protocol.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /protocol/carrygo.protocol.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /protocol/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | carrygo 7 | com.abc.carrygo 8 | 1.0-SNAPSHOT 9 | ../pom.xml 10 | 11 | 4.0.0 12 | 13 | carrygo.protocol 14 | jar 15 | 16 | 17 | 18 | com.google.protobuf 19 | protobuf-java 20 | 21 | 22 | 23 | 24 | 25 | 26 | org.xolstice.maven.plugins 27 | protobuf-maven-plugin 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /protocol/src/main/proto/SQLProtocol.proto: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.common.protocol; 2 | 3 | option java_package = "com.abc.carrygo.protocol"; 4 | 5 | option java_outer_classname = "SQLEventEntry"; 6 | 7 | option optimize_for = SPEED; 8 | 9 | message SQLEntry { 10 | required EntryType entry = 1; 11 | repeated DMLEntry dmlEntries = 2; 12 | repeated DDLEntry ddlEntries = 3; 13 | } 14 | 15 | message DDLEntry { 16 | required string db = 1; 17 | required string table = 2; 18 | required KeyWord keyword = 3; 19 | repeated DDLCol ddlCols = 4; 20 | } 21 | 22 | message DDLCol { 23 | required KeyWord keyword = 1; 24 | optional bool isKey = 2 [default = false]; 25 | optional string col = 3; 26 | optional string changeCol = 4; 27 | } 28 | 29 | message DMLEntry { 30 | required string db = 1; 31 | required string table = 2; 32 | required KeyWord keyword = 3; 33 | repeated DMLCol dmlCols = 4; 34 | } 35 | 36 | message DMLCol { 37 | required string col = 1; 38 | required bool isKey = 2; 39 | required bool isUpdate = 3; 40 | required string value = 4; 41 | } 42 | 43 | enum EntryType { 44 | DML = 1; 45 | DDL = 2; 46 | } 47 | 48 | enum KeyWord { 49 | CREATE = 1; 50 | ALTER = 2; 51 | ADD = 3; 52 | DROP = 4; 53 | CHANGE = 5; 54 | UPDATE = 6; 55 | INSERT = 7; 56 | DELETE = 8; 57 | } -------------------------------------------------------------------------------- /store/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | carrygo 7 | com.abc.carrygo 8 | 1.0-SNAPSHOT 9 | ../pom.xml 10 | 11 | 4.0.0 12 | 13 | carrygo.store 14 | jar 15 | 16 | 17 | 18 | 19 | com.abc.carrygo 20 | carrygo.common 21 | 1.0-SNAPSHOT 22 | 23 | 24 | 25 | ch.qos.logback 26 | logback-classic 27 | runtime 28 | 29 | 30 | 31 | org.apache.hbase 32 | hbase-client 33 | 34 | 35 | 36 | 37 | com.google.guava 38 | guava 39 | 12.0.1 40 | 41 | 42 | 43 | 44 | com.google.protobuf 45 | protobuf-java 46 | 2.5.0 47 | 48 | 49 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /store/src/main/java/com/abc/carrygo/store/AbstractWriterProcessor.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.store; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | /** 7 | * Created by plin on 10/17/16. 8 | */ 9 | public abstract class AbstractWriterProcessor { 10 | private static final Logger log = LoggerFactory.getLogger(AbstractWriterProcessor.class); 11 | 12 | 13 | public abstract void process(); 14 | 15 | } 16 | -------------------------------------------------------------------------------- /store/src/main/java/com/abc/carrygo/store/hbase/HBaseWriterProcessor.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.store.hbase; 2 | 3 | import com.abc.carrygo.common.kafka.KafkaAcceptor; 4 | import com.abc.carrygo.store.AbstractWriterProcessor; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.List; 9 | 10 | /** 11 | * Created by plin on 10/17/16. 12 | */ 13 | public class HBaseWriterProcessor extends AbstractWriterProcessor { 14 | private static final Logger log = LoggerFactory.getLogger(HBaseWriterProcessor.class); 15 | private static final int N_THREADS = 1; 16 | private HBaseWriter writer; 17 | private List acceptors; 18 | 19 | public void process() { 20 | 21 | try { 22 | for (KafkaAcceptor acceptor : acceptors) { 23 | acceptor.run(N_THREADS, writer, MessageReceived.class.getName()); 24 | } 25 | } catch (Exception e) { 26 | log.error("hbase event writer failed.", e); 27 | } 28 | } 29 | 30 | public void close() { 31 | 32 | for (KafkaAcceptor acceptor : acceptors) { 33 | acceptor.close(); 34 | } 35 | 36 | if (writer != null) { 37 | try { 38 | writer.close(); 39 | } catch (Exception e1) { 40 | log.error("hbase event writer close failed.", e1); 41 | } 42 | } 43 | } 44 | 45 | public void setAcceptors(List acceptors) { 46 | this.acceptors = acceptors; 47 | } 48 | 49 | public void setWriter(HBaseWriter writer) { 50 | this.writer = writer; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /store/src/main/java/com/abc/carrygo/store/hbase/HBaseWriterProcessorBuilder.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.store.hbase; 2 | 3 | import com.abc.carrygo.common.kafka.KafkaAcceptor; 4 | import org.springframework.util.Assert; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * Created by plin on 10/19/16. 10 | */ 11 | public final class HBaseWriterProcessorBuilder { 12 | 13 | private HBaseWriterProcessor processor = null; 14 | private List acceptors = null; 15 | private HBaseWriter writer = null; 16 | 17 | public HBaseWriterProcessorBuilder() { 18 | } 19 | 20 | private static HBaseWriterProcessor build(HBaseWriterProcessorBuilder builder) { 21 | final List acceptors = builder.acceptors; 22 | final HBaseWriter writer = builder.writer; 23 | 24 | final HBaseWriterProcessor processor = 25 | (builder.processor != null) 26 | ? builder.processor 27 | : new HBaseWriterProcessor(); 28 | 29 | processor.setAcceptors(acceptors); 30 | processor.setWriter(writer); 31 | 32 | return processor; 33 | } 34 | 35 | public HBaseWriterProcessorBuilder setAcceptors(List acceptors) { 36 | Assert.notNull(acceptors); 37 | this.acceptors = acceptors; 38 | return this; 39 | } 40 | 41 | public HBaseWriterProcessorBuilder setWriter(HBaseWriter writer) { 42 | Assert.notNull(writer); 43 | this.writer = writer; 44 | return this; 45 | } 46 | 47 | public HBaseWriterProcessor build() { 48 | return build(this); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /store/src/main/java/com/abc/carrygo/store/hbase/MessageReceived.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.store.hbase; 2 | 3 | import com.abc.carrygo.common.kafka.KafkaConsumer; 4 | import com.abc.carrygo.common.serde.ProtobufSerde; 5 | import com.abc.carrygo.protocol.SQLEventEntry.SQLEntry; 6 | import kafka.consumer.ConsumerIterator; 7 | import kafka.consumer.KafkaStream; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | /** 12 | * Created by plin on 10/18/16. 13 | */ 14 | public class MessageReceived extends KafkaConsumer { 15 | private static final Logger log = LoggerFactory.getLogger(MessageReceived.class); 16 | 17 | private HBaseWriter writer; 18 | 19 | public MessageReceived(HBaseWriter writer, KafkaStream stream) { 20 | super(stream); 21 | this.writer = writer; 22 | } 23 | 24 | @Override 25 | public void run() { 26 | 27 | ConsumerIterator it = stream.iterator(); 28 | while (it.hasNext()) { 29 | // log.info("msg:{}", new String(it.next().message())); 30 | 31 | byte[] bytes = it.next().message(); 32 | SQLEntry sqlEntry = ProtobufSerde.fromBinary(bytes, SQLEntry.getDefaultInstance()); 33 | writer.write(sqlEntry); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /store/src/main/java/com/abc/carrygo/store/hbase/service/HBaseWriterService.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.store.hbase.service; 2 | 3 | import com.abc.carrygo.common.hbase.HBaseConnector; 4 | import com.abc.carrygo.common.kafka.KafkaAcceptor; 5 | import com.abc.carrygo.common.zookeeper.ZkAddress; 6 | import com.abc.carrygo.store.hbase.HBaseWriter; 7 | import com.abc.carrygo.store.hbase.HBaseWriterProcessor; 8 | import com.abc.carrygo.store.hbase.HBaseWriterProcessorBuilder; 9 | import org.apache.commons.lang3.StringUtils; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.io.IOException; 14 | import java.util.ArrayList; 15 | import java.util.Arrays; 16 | import java.util.List; 17 | 18 | /** 19 | * Created by plin on 10/20/16. 20 | */ 21 | public class HBaseWriterService { 22 | private static final Logger log = LoggerFactory.getLogger(HBaseWriterService.class); 23 | 24 | private static final String SEPARATOR_CHAR = ","; 25 | 26 | 27 | private HBaseWriterProcessor processor; 28 | 29 | public void init(ZkAddress zkAddress, String specifiedTopics) { 30 | try { 31 | HBaseConnector.init(zkAddress); 32 | } catch (IOException e) { 33 | log.error("hbase connector init failed.", e); 34 | } 35 | 36 | String[] strings = StringUtils.split(specifiedTopics, SEPARATOR_CHAR); 37 | List topics = Arrays.asList(strings); 38 | 39 | List acceptors = new ArrayList<>(topics.size()); 40 | for (String topic : topics) { 41 | KafkaAcceptor acceptor = new KafkaAcceptor(zkAddress, topic); 42 | acceptors.add(acceptor); 43 | } 44 | 45 | 46 | HBaseWriter writer = new HBaseWriter(); 47 | HBaseWriterProcessorBuilder builder = new HBaseWriterProcessorBuilder(); 48 | builder 49 | .setAcceptors(acceptors) 50 | .setWriter(writer); 51 | this.processor = builder.build(); 52 | } 53 | 54 | 55 | public void write() { 56 | processor.process(); 57 | } 58 | 59 | public void close() { 60 | processor.close(); 61 | } 62 | 63 | 64 | } 65 | -------------------------------------------------------------------------------- /store/src/main/java/com/abc/carrygo/store/server/WriterServer.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.store.server; 2 | 3 | import com.abc.carrygo.common.conf.Configured; 4 | import com.abc.carrygo.common.zookeeper.ZkAddress; 5 | import com.abc.carrygo.store.hbase.service.HBaseWriterService; 6 | import org.apache.commons.lang.exception.ExceptionUtils; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * Created by plin on 10/20/16. 12 | */ 13 | public class WriterServer { 14 | private static final Logger log = LoggerFactory.getLogger(WriterServer.class); 15 | 16 | private static ZkAddress zkAddress; 17 | private static String specifiedTopics; 18 | 19 | private HBaseWriterService service; 20 | 21 | public static void main(String[] args) { 22 | zkAddress = Configured.zkAddress; 23 | specifiedTopics = Configured.kafkaTopic; 24 | 25 | final WriterServer server = new WriterServer(); 26 | server.start(); 27 | 28 | Runtime.getRuntime().addShutdownHook(new Thread() { 29 | 30 | public void run() { 31 | try { 32 | log.info("stop the writer server"); 33 | server.stop(); 34 | 35 | } catch (Throwable e) { 36 | log.warn("something goes wrong when stopping writer server:\n{}", ExceptionUtils.getFullStackTrace(e)); 37 | } finally { 38 | log.info("writer server is down."); 39 | } 40 | } 41 | }); 42 | } 43 | 44 | public void start() { 45 | log.info("start the writer server"); 46 | startWriter(); 47 | } 48 | 49 | public void startWriter() { 50 | service = new HBaseWriterService(); 51 | service.init(zkAddress, specifiedTopics); 52 | 53 | service.write(); 54 | } 55 | 56 | public void stop() { 57 | service.close(); 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /store/src/main/resources/carrygo-conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 10 | 11 | 12 | 13 | 14 | 15 | classpath:carrygo.properties 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /store/src/main/resources/carrygo.properties: -------------------------------------------------------------------------------- 1 | [kafka] 2 | kafka.ip = 127.0.0.1 3 | kafka.port = 9092 4 | kafka.topic = test 5 | 6 | [zookeeper] 7 | zookeeper.destination = example 8 | zookeeper.ip = 127.0.0.1 9 | zookeeper.port = 2181 10 | 11 | -------------------------------------------------------------------------------- /store/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | log 6 | 7 | 9 | log.%d{yyyy-MM-dd}.%i 10 | 11 | 12 | 500MB 13 | 14 | 15 | 3 16 | 17 | 18 | 19 | UTF-8 20 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /store/src/test/java/com/abc/carrygo/store/hbase/HBaseWriterBase.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.store.hbase; 2 | 3 | import com.abc.carrygo.protocol.SQLEventEntry.*; 4 | 5 | /** 6 | * Created by plin on 10/25/16. 7 | */ 8 | public class HBaseWriterBase { 9 | protected static DDLEntry.Builder ddlBuilder; 10 | protected static DMLEntry.Builder dmlBuilder; 11 | protected static SQLEntry.Builder builder; 12 | 13 | public void init() { 14 | ddlBuilder = DDLEntry.newBuilder(); 15 | dmlBuilder = DMLEntry.newBuilder(); 16 | builder = SQLEntry.newBuilder(); 17 | } 18 | 19 | 20 | public SQLEntry buildSQLEntry(DDLEntry ddlEntry) { 21 | SQLEntry sqlEntry = builder 22 | .setEntry(EntryType.DDL) 23 | .addDdlEntries(ddlEntry) 24 | .build(); 25 | return sqlEntry; 26 | } 27 | 28 | public SQLEntry buildSQLEntry(DMLEntry dmlEntry) { 29 | SQLEntry sqlEntry = builder 30 | .setEntry(EntryType.DML) 31 | .addDmlEntries(dmlEntry) 32 | .build(); 33 | return sqlEntry; 34 | } 35 | 36 | public DDLEntry buildDDLEntry(KeyWord keyWord, DDLCol col) { 37 | DDLEntry ddlEntry = ddlBuilder 38 | .setKeyword(keyWord) 39 | .addDdlCols(col) 40 | .build(); 41 | 42 | return ddlEntry; 43 | } 44 | 45 | public DMLEntry buildDMLEntry(KeyWord keyWord, DMLCol col) { 46 | DMLEntry dmlEntry = dmlBuilder 47 | .setKeyword(keyWord) 48 | .addDmlCols(col) 49 | .build(); 50 | 51 | return dmlEntry; 52 | } 53 | 54 | public DDLCol buildDDLCol(String colName, KeyWord keyWord, boolean isKey) { 55 | DDLCol ddlCol = DDLCol.newBuilder() 56 | .setCol(colName) 57 | .setIsKey(isKey) 58 | .setKeyword(keyWord) 59 | .build(); 60 | 61 | return ddlCol; 62 | } 63 | 64 | public DMLCol buildDMLCol(String colName, 65 | String value, 66 | boolean isUpdate, 67 | boolean isKey) { 68 | DMLCol dmlCol = DMLCol.newBuilder() 69 | .setCol(colName) 70 | .setIsKey(isKey) 71 | .setIsUpdate(isUpdate) 72 | .setValue(value) 73 | .build(); 74 | 75 | return dmlCol; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /store/src/test/java/com/abc/carrygo/store/hbase/HBaseWriterMultiThreadTest.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.store.hbase; 2 | 3 | public class HBaseWriterMultiThreadTest extends HBaseWriterBase { 4 | 5 | 6 | } 7 | -------------------------------------------------------------------------------- /store/src/test/java/com/abc/carrygo/store/hbase/HBaseWriterTest.java: -------------------------------------------------------------------------------- 1 | package com.abc.carrygo.store.hbase; 2 | 3 | import com.abc.carrygo.common.hbase.HBaseConnector; 4 | import com.abc.carrygo.common.zookeeper.ZkAddress; 5 | import com.abc.carrygo.protocol.SQLEventEntry.*; 6 | import org.junit.After; 7 | import org.junit.Assert; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | 11 | import java.io.IOException; 12 | import java.util.List; 13 | import java.util.concurrent.atomic.AtomicLong; 14 | 15 | /** 16 | * Created by plin on 10/21/16. 17 | */ 18 | public class HBaseWriterTest extends HBaseWriterBase { 19 | private static final String NS = "example"; 20 | private static final String DB = "test"; 21 | private static final String TABLE = "user"; 22 | 23 | private static final String ZK_IP = "127.0.0.1"; 24 | private static final String ZK_PORT = "2181"; 25 | private static final String ZK_DESTINATION = "example"; 26 | private static final String PRIMARY_KEY = "id"; 27 | private static HBaseWriter hbaseWriter; 28 | private AtomicLong atomicId = new AtomicLong(1); 29 | 30 | @Before 31 | public void setUp() { 32 | init(); 33 | 34 | ddlBuilder = ddlBuilder.setDb(DB).setTable(TABLE); 35 | dmlBuilder.setDb(DB).setTable(TABLE); 36 | 37 | ZkAddress zkAddress = new ZkAddress(ZK_DESTINATION, ZK_IP, ZK_PORT); 38 | try { 39 | HBaseConnector.init(zkAddress); 40 | } catch (IOException e) { 41 | e.printStackTrace(); 42 | } 43 | hbaseWriter = new HBaseWriter(); 44 | } 45 | 46 | // @Test 47 | public void testCreate() { 48 | 49 | DDLCol col = buildDDLCol(PRIMARY_KEY, KeyWord.CREATE, true); 50 | DDLEntry ddlEntry = buildDDLEntry(KeyWord.CREATE, col); 51 | SQLEntry sqlEntry = buildSQLEntry(ddlEntry); 52 | 53 | test(sqlEntry); 54 | } 55 | 56 | // @Test 57 | public void testAlter() { 58 | String colName = "name"; 59 | 60 | DDLCol col = buildDDLCol(colName, KeyWord.ADD, false); 61 | DDLEntry ddlEntry = buildDDLEntry(KeyWord.ALTER, col); 62 | SQLEntry sqlEntry = buildSQLEntry(ddlEntry); 63 | 64 | test(sqlEntry); 65 | } 66 | 67 | // @Test 68 | public void testInsert() { 69 | long value = atomicId.addAndGet(System.nanoTime()); 70 | 71 | DMLCol col = buildDMLCol(PRIMARY_KEY, String.valueOf(value), true, true); 72 | DMLEntry dmlEntry = buildDMLEntry(KeyWord.INSERT, col); 73 | SQLEntry sqlEntry = buildSQLEntry(dmlEntry); 74 | 75 | test(sqlEntry); 76 | } 77 | 78 | // @Test 79 | public void testUpdate() { 80 | long value = atomicId.get(); 81 | 82 | DMLCol col = buildDMLCol(PRIMARY_KEY, String.valueOf(value), true, true); 83 | DMLEntry dmlEntry = buildDMLEntry(KeyWord.UPDATE, col); 84 | SQLEntry sqlEntry = buildSQLEntry(dmlEntry); 85 | 86 | test(sqlEntry); 87 | } 88 | 89 | // @Test 90 | public void testDel() { 91 | long value = atomicId.get(); 92 | 93 | DMLCol col = buildDMLCol(PRIMARY_KEY, String.valueOf(value), true, true); 94 | DMLEntry dmlEntry = buildDMLEntry(KeyWord.DELETE, col); 95 | SQLEntry sqlEntry = buildSQLEntry(dmlEntry); 96 | 97 | test(sqlEntry); 98 | } 99 | 100 | // @Test 101 | public void testDelTable() { 102 | HBaseWriter.EventWriter eventWriter = hbaseWriter.getWriter(); 103 | 104 | try { 105 | eventWriter.del(TABLE); 106 | } catch (Exception e) { 107 | e.printStackTrace(); 108 | 109 | Assert.fail(); 110 | } 111 | 112 | } 113 | 114 | @Test 115 | public void testDelTables() { 116 | HBaseWriter.EventWriter eventWriter = hbaseWriter.getWriter(); 117 | List tables = eventWriter.getTables(NS); 118 | 119 | try { 120 | for (String table : tables) { 121 | eventWriter.del(table); 122 | } 123 | } catch (Exception e) { 124 | e.printStackTrace(); 125 | 126 | Assert.fail(); 127 | } 128 | 129 | } 130 | 131 | private void test(SQLEntry entry) { 132 | try { 133 | hbaseWriter.write(entry); 134 | } catch (Exception e) { 135 | e.printStackTrace(); 136 | 137 | Assert.fail(); 138 | } 139 | 140 | Assert.assertTrue(true); 141 | } 142 | 143 | @After 144 | public void tearDown() { 145 | try { 146 | hbaseWriter.close(); 147 | } catch (Exception e) { 148 | e.printStackTrace(); 149 | 150 | Assert.fail(); 151 | } 152 | } 153 | } 154 | --------------------------------------------------------------------------------