├── .gitignore ├── LICENSE ├── README.md ├── pom.xml └── src └── main ├── java └── cn │ └── thinkingdata │ └── kafka │ ├── cache │ └── KafkaCache.java │ ├── close │ ├── DaemonCloseThread.java │ ├── ScanTermMethod.java │ ├── SignalTermMethod.java │ └── TermMethod.java │ ├── constant │ └── KafkaMysqlOffsetParameter.java │ ├── consumer │ ├── KafkaConsumerRebalancerListener.java │ ├── KafkaSubscribeConsumeThread.java │ ├── KafkaSubscribeConsumer.java │ ├── KafkaSubscribeConsumerManager.java │ ├── NewIDataLineProcessor.java │ ├── dao │ │ └── KafkaConsumerOffset.java │ ├── exception │ │ ├── ExceptionHandler.java │ │ └── TaKafkaCommonException.java │ ├── offset │ │ ├── MysqlOffsetManager.java │ │ └── OffsetManager.java │ └── persist │ │ ├── DBPoolConnection.java │ │ ├── DefaultStorePersist.java │ │ ├── MysqlOffsetPersist.java │ │ ├── OffsetPersist.java │ │ ├── RedisStorePersistService.java │ │ └── StorePersist.java │ ├── test │ ├── ProcessDataThread.java │ └── TestMain.java │ └── util │ ├── CommonUtils.java │ └── RetryerUtil.java └── resources └── log4j.properties /.gitignore: -------------------------------------------------------------------------------- 1 | ################### 2 | ## Folders ## 3 | ############################## 4 | .settings 5 | target 6 | .idea/ 7 | 8 | ############################## 9 | ## Eclipse conf ## 10 | ############################## 11 | .classpath 12 | .project -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-common 2 | 3 | This is the code for the exactly-once consumer persists the offsets in mysql for kafka 0.10.0. 4 | 5 | You can create a table in mysql to store the offsets in mysql. 6 | 7 | Example for the offset table: 8 | 9 | CREATE TABLE `kafka_consumer_offset` ( 10 | `oid` int(11) NOT NULL AUTO_INCREMENT COMMENT 'id', 11 | `topic` varchar(100) NOT NULL COMMENT 'kafka主题', 12 | `kafka_partition` int(11) NOT NULL COMMENT 'kafka主题下的leader分区号', 13 | `consumer_group` varchar(100) NOT NULL COMMENT '消费组', 14 | `offset` bigint(100) NOT NULL DEFAULT '0' COMMENT '偏移量', 15 | `last_flush_offset` bigint(100) NOT NULL DEFAULT '0' COMMENT '上一次的偏移量', 16 | `kafka_cluster_name` varchar(100) NOT NULL DEFAULT '' COMMENT 'kafka集群唯一标识键', 17 | `owner` varchar(255) NOT NULL DEFAULT '' COMMENT 'kafka消费者名称', 18 | `update_time` timestamp NOT NULL DEFAULT '1971-01-01 00:00:00' COMMENT '更新时间', 19 | `create_time` timestamp NOT NULL DEFAULT '1971-01-01 00:00:00' COMMENT '入库时间', 20 | PRIMARY KEY (`oid`), 21 | UNIQUE KEY `topic_partition_consumer` (`kafka_cluster_name`,`topic`,`kafka_partition`,`consumer_group`) USING BTREE 22 | ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4; 23 | 24 | 25 | If you have any problems or you find any bugs, please do not hesitate to contact me(yangruochen@thinkingdata.cn). 26 | 27 | ## maven central repo: 28 | ```xml 29 | 30 | cn.thinkingdata 31 | kafka-common 32 | 0.3.1 33 | 34 | ``` 35 | ## 2021/03/22 36 | version 0.3.1 fix bug 37 | 38 | ## 2021/03/19 39 | version 0.3.0 reform code and public to central repo 40 | 41 | ## 2020/11/06 42 | version 0.2.1 fix bug, replace stop(timeout) to stopWithException, optimize the time complexity 43 | 44 | ## 2020/03/19 45 | version 0.2.0 fix bug, change KafkaSubscribeConsumer constructor, kafka-client change to 2.3.0 46 | 47 | ## 2019/08/21 48 | version 0.1.3 change name to kafka-common 49 | 50 | ## 2019/08/21 51 | version 0.1.2 change new interface 52 | 53 | ## 2019/07/24 54 | version 0.1.1 fix bug 55 | 56 | ## 2019/07/19 57 | version 0.1.0 this version is used in our product (thinking analytics, see: https://www.thinkingdata.cn/manual.html), and it is served for more than 50 companies. 58 | 59 | ## 2018/09/05 60 | version 0.0.2 fix the problems when the kafka coordinate dead 61 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | cn.thinkingdata 5 | kafka-common 6 | 0.4.0 7 | thinkingdata-kafka-common 8 | kafka-common 9 | http://www.thinkinggame.cn 10 | 11 | org.sonatype.oss 12 | oss-parent 13 | 7 14 | 15 | 16 | 17 | The Apache Software License, Version 2.0 18 | http://www.apache.org/licenses/LICENSE-2.0.txt 19 | 20 | 21 | 22 | 23 | https://github.com/ThinkingDataAnalytics/kafka-common 24 | https://github.com/ThinkingDataAnalytics/kafka-common 25 | https://github.com/ThinkingDataAnalytics/kafka-common 26 | 27 | 28 | 29 | thinkingdata 30 | yangruochen@thinkingdata.cn 31 | http://www.thinkinggame.cn 32 | 33 | 34 | 35 | 36 | UTF-8 37 | 38 | 39 | 40 | 41 | commons-codec 42 | commons-codec 43 | 1.10 44 | 45 | 46 | commons-logging 47 | commons-logging 48 | 1.2 49 | 50 | 51 | org.slf4j 52 | slf4j-api 53 | 1.6.1 54 | jar 55 | compile 56 | 57 | 58 | org.slf4j 59 | slf4j-log4j12 60 | 1.6.1 61 | jar 62 | compile 63 | 64 | 65 | org.apache.commons 66 | commons-lang3 67 | 3.4 68 | 69 | 70 | com.google.guava 71 | guava 72 | 18.0 73 | 74 | 75 | org.apache.kafka 76 | kafka-clients 77 | 2.3.0 78 | 79 | 80 | org.apache.kafka 81 | kafka_2.12 82 | 2.3.0 83 | 84 | 85 | com.alibaba 86 | fastjson 87 | 1.2.71 88 | 89 | 90 | mysql 91 | mysql-connector-java 92 | 5.1.46 93 | 94 | 95 | com.github.rholder 96 | guava-retrying 97 | 2.0.0 98 | 99 | 100 | com.google.guava 101 | guava 102 | 103 | 104 | 105 | 106 | joda-time 107 | joda-time 108 | 2.9.9 109 | 110 | 111 | 112 | com.alibaba 113 | druid 114 | 1.1.14 115 | 116 | 117 | commons-collections 118 | commons-collections 119 | 3.2.2 120 | 121 | 122 | 123 | 124 | 125 | kafka-mysql-offset 126 | 127 | 128 | org.apache.maven.plugins 129 | maven-compiler-plugin 130 | 3.5.1 131 | 132 | 1.8 133 | 1.8 134 | UTF-8 135 | 136 | 137 | 138 | org.apache.maven.plugins 139 | maven-javadoc-plugin 140 | 2.10.4 141 | 142 | true 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | ossrh 173 | 174 | 175 | releases 176 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 177 | 178 | 179 | snapshots 180 | https://oss.sonatype.org/content/repositories/snapshots/ 181 | 182 | 183 | 184 | 185 | compile 186 | 187 | 188 | 189 | org.apache.maven.plugins 190 | maven-source-plugin 191 | 3.0.1 192 | 193 | 194 | package 195 | 196 | jar-no-fork 197 | 198 | 199 | 200 | 201 | 202 | 203 | org.apache.maven.plugins 204 | maven-javadoc-plugin 205 | 2.10.4 206 | 207 | 208 | package 209 | 210 | jar 211 | 212 | 213 | 214 | 215 | 216 | 217 | org.apache.maven.plugins 218 | maven-gpg-plugin 219 | 1.6 220 | 221 | 222 | sign-artifacts 223 | verify 224 | 225 | sign 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/cache/KafkaCache.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.cache; 2 | 3 | import cn.thinkingdata.kafka.consumer.KafkaConsumerRebalancerListener; 4 | import cn.thinkingdata.kafka.consumer.KafkaSubscribeConsumeThread; 5 | import cn.thinkingdata.kafka.consumer.dao.KafkaConsumerOffset; 6 | import org.apache.kafka.common.TopicPartition; 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.concurrent.ConcurrentHashMap; 11 | import java.util.concurrent.CopyOnWriteArrayList; 12 | 13 | public class KafkaCache { 14 | 15 | public static Map kafkaConsumerOffsetMaps = new ConcurrentHashMap(); 16 | public static List consumeThreadList = new CopyOnWriteArrayList(); 17 | public static List rebalancerListenerList = new CopyOnWriteArrayList(); 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/close/DaemonCloseThread.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.close; 2 | 3 | import cn.thinkingdata.kafka.consumer.KafkaSubscribeConsumer; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | public class DaemonCloseThread extends Thread { 8 | 9 | private static final Logger logger = LoggerFactory 10 | .getLogger(DaemonCloseThread.class); 11 | 12 | KafkaSubscribeConsumer consumers; 13 | 14 | TermMethod closeMethod; 15 | 16 | Boolean flag = true; 17 | 18 | public DaemonCloseThread(KafkaSubscribeConsumer consumers, TermMethod closeMethod) { 19 | this.consumers = consumers; 20 | this.closeMethod = closeMethod; 21 | } 22 | 23 | public static void destroy(KafkaSubscribeConsumer consumers) { 24 | consumers.destroy(); 25 | } 26 | 27 | @Override 28 | public void run() { 29 | close(); 30 | } 31 | 32 | public void shutdown() { 33 | flag = false; 34 | } 35 | 36 | public void afterDestroyConsumer() { 37 | closeMethod.afterDestroyConsumer(); 38 | } 39 | 40 | 41 | private void close() { 42 | logger.info("start DaemonCloseThread!"); 43 | while (flag) { 44 | Boolean receiveTermSignal = closeMethod.receiveTermSignal(); 45 | if (receiveTermSignal) { 46 | logger.info("start to destroy consumers"); 47 | destroy(consumers); 48 | break; 49 | } else { 50 | try { 51 | Thread.sleep(50); 52 | } catch (InterruptedException e) { 53 | logger.error("------- thread can not sleep ---------------------" + e.toString()); 54 | } 55 | } 56 | } 57 | logger.info("DaemonCloseThread stop!"); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/close/ScanTermMethod.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.close; 2 | 3 | import java.util.Scanner; 4 | 5 | public class ScanTermMethod implements TermMethod { 6 | 7 | Scanner scan = new Scanner(System.in); 8 | 9 | @Override 10 | public Boolean receiveTermSignal() { 11 | System.out.println("Stop it now?(yes/no):"); 12 | String result = scan.nextLine(); 13 | return result.equals("yes"); 14 | 15 | } 16 | 17 | @Override 18 | public void afterDestroyConsumer() { 19 | scan.close(); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/close/SignalTermMethod.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.close; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import sun.misc.Signal; 6 | import sun.misc.SignalHandler; 7 | 8 | public class SignalTermMethod implements TermMethod, SignalHandler { 9 | private static final Logger logger = LoggerFactory.getLogger(SignalTermMethod.class); 10 | 11 | Signal termSignal = new Signal("TERM"); 12 | 13 | public Boolean termSignalFlag = false; 14 | 15 | public SignalTermMethod() { 16 | init(); 17 | } 18 | 19 | public void init() { 20 | Signal.handle(termSignal, this); 21 | } 22 | 23 | @Override 24 | public Boolean receiveTermSignal() { 25 | return termSignalFlag; 26 | } 27 | 28 | @Override 29 | public void afterDestroyConsumer() { 30 | } 31 | 32 | @Override 33 | public void handle(Signal signal) { 34 | if (signal.getName().equals("TERM")) { 35 | logger.info("signal term received"); 36 | termSignalFlag = true; 37 | } 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/close/TermMethod.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.close; 2 | 3 | public interface TermMethod { 4 | 5 | Boolean receiveTermSignal(); 6 | 7 | void afterDestroyConsumer(); 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/constant/KafkaMysqlOffsetParameter.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.constant; 2 | 3 | import cn.thinkingdata.kafka.util.CommonUtils; 4 | import org.apache.commons.lang3.StringUtils; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.Map; 9 | import java.util.Properties; 10 | import java.util.concurrent.atomic.AtomicBoolean; 11 | 12 | public class KafkaMysqlOffsetParameter { 13 | 14 | private static final Logger logger = LoggerFactory 15 | .getLogger(KafkaMysqlOffsetParameter.class); 16 | 17 | public static final AtomicBoolean kafkaSubscribeConsumerClosed = new AtomicBoolean(true); 18 | public static final AtomicBoolean mysqlAndBackupStoreConnState = new AtomicBoolean(true); 19 | public static final String hostname = CommonUtils.getHostName(); 20 | 21 | public static String jdbcUrl; 22 | public static String username; 23 | public static String password; 24 | public static String tableName; 25 | public static String brokerList; 26 | public static String kafkaClusterName; 27 | public static String topic; 28 | public static String consumerGroup; 29 | public static Integer processThreadNum; 30 | public static Integer flushOffsetSize; 31 | public static Integer flushInterval; 32 | private static String maxPartitionFetchBytes = "524288"; 33 | private static String heartbeatInterval = "10000"; 34 | public static String sessionTimeout = "30000"; 35 | private static String requestTimeout = "40000"; 36 | public static String autoOffsetReset = "latest"; 37 | public static Integer pollInterval = 50; 38 | public static Integer maxPollRecords = 1000; 39 | public static String partitionAssignmentStrategy; 40 | public static Properties kafkaConf; 41 | public static Properties kafkaConfExtraProperties = new Properties(); 42 | 43 | // public static void setMaxPartitionFetchBytes(Long maxPartitionFetchBytes) 44 | // { 45 | // KafkaMysqlOffsetParameter.maxPartitionFetchBytes = 46 | // maxPartitionFetchBytes.toString(); 47 | // } 48 | 49 | public static void createKafkaConfProp() { 50 | kafkaConf = new Properties(); 51 | kafkaConf.put("bootstrap.servers", KafkaMysqlOffsetParameter.brokerList); 52 | kafkaConf.put("group.id", KafkaMysqlOffsetParameter.consumerGroup); 53 | // Below is a key setting to turn off the auto commit. 54 | kafkaConf.put("enable.auto.commit", "false"); 55 | kafkaConf.put("heartbeat.interval.ms", KafkaMysqlOffsetParameter.heartbeatInterval); 56 | kafkaConf.put("session.timeout.ms", KafkaMysqlOffsetParameter.sessionTimeout); 57 | kafkaConf.put("max.poll.records", maxPollRecords); 58 | kafkaConf.put("request.timeout.ms", KafkaMysqlOffsetParameter.requestTimeout); 59 | // Control maximum data on each poll, make sure this value is bigger 60 | // than the maximum single record size 61 | kafkaConf.put("max.partition.fetch.bytes", KafkaMysqlOffsetParameter.maxPartitionFetchBytes); 62 | kafkaConf.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 63 | kafkaConf.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 64 | kafkaConf.put("auto.offset.reset", KafkaMysqlOffsetParameter.autoOffsetReset); 65 | if (StringUtils.isNotBlank(partitionAssignmentStrategy)) { 66 | kafkaConf.put("partition.assignment.strategy", KafkaMysqlOffsetParameter.partitionAssignmentStrategy); 67 | } 68 | kafkaConf.putAll(kafkaConfExtraProperties); 69 | } 70 | 71 | public static void createKafkaConfProp(Map prop) { 72 | kafkaConfExtraProperties.putAll(prop); 73 | jdbcUrl = prop.get("jdbc.url"); 74 | kafkaConfExtraProperties.remove("jdbc.url"); 75 | username = prop.get("username"); 76 | kafkaConfExtraProperties.remove("username"); 77 | password = prop.get("password"); 78 | kafkaConfExtraProperties.remove("password"); 79 | tableName = prop.get("table.name"); 80 | kafkaConfExtraProperties.remove("table.name"); 81 | brokerList = prop.get("broker.list"); 82 | kafkaConfExtraProperties.remove("broker.list"); 83 | kafkaClusterName = prop.get("kafka.cluster.name"); 84 | kafkaConfExtraProperties.remove("kafka.cluster.name"); 85 | topic = prop.get("topic"); 86 | kafkaConfExtraProperties.remove("topic"); 87 | consumerGroup = prop.get("consumer.group"); 88 | kafkaConfExtraProperties.remove("consumer.group"); 89 | processThreadNum = Integer.parseInt(prop.get("process.thread.num")); 90 | kafkaConfExtraProperties.remove("process.thread.num"); 91 | flushOffsetSize = Integer.parseInt(prop.get("flush.offset.size")); 92 | kafkaConfExtraProperties.remove("flush.offset.size"); 93 | flushInterval = Integer.parseInt(prop.get("flush.interval")); 94 | kafkaConfExtraProperties.remove("flush.interval"); 95 | 96 | assert null != jdbcUrl; 97 | assert null != username; 98 | assert null != password; 99 | assert null != tableName; 100 | assert null != brokerList; 101 | assert null != kafkaClusterName; 102 | assert null != topic; 103 | assert null != consumerGroup; 104 | assert null != processThreadNum; 105 | assert null != flushOffsetSize; 106 | assert null != flushInterval; 107 | 108 | 109 | if (prop.get("heartbeat.interval") != null) { 110 | heartbeatInterval = String.valueOf(Integer.parseInt(prop.get("heartbeat.interval")) * 1000); 111 | kafkaConfExtraProperties.remove("heartbeat.interval"); 112 | } 113 | if (prop.get("session.timeout") != null) { 114 | sessionTimeout = String.valueOf(Integer.parseInt(prop.get("session.timeout")) * 1000); 115 | kafkaConfExtraProperties.remove("session.timeout"); 116 | } 117 | if (prop.get("request.timeout") != null) { 118 | requestTimeout = String.valueOf(Integer.parseInt(prop.get("request.timeout")) * 1000); 119 | kafkaConfExtraProperties.remove("request.timeout"); 120 | } 121 | if (prop.get("max.partition.fetch.bytes") != null) { 122 | maxPartitionFetchBytes = prop.get("max.partition.fetch.bytes"); 123 | kafkaConfExtraProperties.remove("max.partition.fetch.bytes"); 124 | } 125 | if (prop.get("auto.offset.reset") != null) { 126 | autoOffsetReset = prop.get("auto.offset.reset"); 127 | assert (autoOffsetReset.equals("latest") || autoOffsetReset.equals("earliest") || autoOffsetReset.equals("none")); 128 | kafkaConfExtraProperties.remove("auto.offset.reset"); 129 | } 130 | if (prop.get("poll.interval") != null) { 131 | pollInterval = Integer.parseInt(prop.get("poll.interval")); 132 | kafkaConfExtraProperties.remove("poll.interval"); 133 | } 134 | if (prop.get("max.poll.records") != null) { 135 | maxPollRecords = Integer.parseInt(prop.get("max.poll.records")); 136 | kafkaConfExtraProperties.remove("max.poll.records"); 137 | } 138 | if (prop.get("partition.assignment.strategy") != null) { 139 | partitionAssignmentStrategy = prop.get("partition.assignment.strategy"); 140 | kafkaConfExtraProperties.remove("partition.assignment.strategy"); 141 | } 142 | createKafkaConfProp(); 143 | } 144 | 145 | } 146 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/KafkaConsumerRebalancerListener.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer; 2 | 3 | import cn.thinkingdata.kafka.cache.KafkaCache; 4 | import cn.thinkingdata.kafka.constant.KafkaMysqlOffsetParameter; 5 | import cn.thinkingdata.kafka.consumer.dao.KafkaConsumerOffset; 6 | import cn.thinkingdata.kafka.consumer.offset.MysqlOffsetManager; 7 | import cn.thinkingdata.kafka.consumer.offset.OffsetManager; 8 | import cn.thinkingdata.kafka.consumer.persist.MysqlOffsetPersist; 9 | import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; 10 | import org.apache.kafka.clients.consumer.KafkaConsumer; 11 | import org.apache.kafka.common.TopicPartition; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.Collection; 16 | import java.util.Date; 17 | 18 | /** 19 | * Re-balancer for any subscription changes. 20 | */ 21 | public class KafkaConsumerRebalancerListener implements ConsumerRebalanceListener { 22 | 23 | private static final Logger logger = LoggerFactory.getLogger(KafkaConsumerRebalancerListener.class); 24 | 25 | private final OffsetManager offsetManager = MysqlOffsetManager.getInstance(); 26 | 27 | private final KafkaConsumer consumer; 28 | 29 | public KafkaConsumerRebalancerListener(KafkaConsumer consumer) { 30 | this.consumer = consumer; 31 | } 32 | 33 | 34 | // onPartitionsRevoked 这个方法会在 Consumer 停止拉取数据之后、group 进行 rebalance 35 | // 操作之前调用，作用是对已经 ack 的 msg 进行 commit； 36 | @Override 37 | public void onPartitionsRevoked(Collection partitions) { 38 | logger.info("start onPartitionsRevoked!"); 39 | for (TopicPartition partition : partitions) { 40 | KafkaConsumerOffset kafkaConsumerOffset = KafkaCache.kafkaConsumerOffsetMaps.get(partition); 41 | if (kafkaConsumerOffset != null) { 42 | kafkaConsumerOffset.setOffset(consumer.position(partition)); 43 | KafkaCache.kafkaConsumerOffsetMaps.put(partition, kafkaConsumerOffset); 44 | MysqlOffsetPersist.getInstance().flush(kafkaConsumerOffset); 45 | //删除kafkaConsumerOffsetSet里的kafkaConsumerOffset 46 | for (KafkaSubscribeConsumeThread consumeThread : KafkaCache.consumeThreadList) { 47 | if (consumeThread.consumer.equals(consumer)) { 48 | logger.debug("consumeThread.kafkaConsumerOffsetSet remove kafkaConsumerOffset, the kafkaConsumerOffset is " + kafkaConsumerOffset); 49 | consumeThread.kafkaConsumerOffsetSet.remove(kafkaConsumerOffset); 50 | } 51 | } 52 | } 53 | } 54 | //有特殊的情况，就是两个线程同时拥有一个partition，这时，需要手动清空kafkaConsumerOffsetSet 55 | for (KafkaSubscribeConsumeThread consumeThread : KafkaCache.consumeThreadList) { 56 | if (consumeThread.consumer.equals(consumer)) { 57 | consumeThread.kafkaConsumerOffsetSet.clear(); 58 | if (consumeThread.assignedPartitions != null) { 59 | consumeThread.assignedPartitions = null; 60 | } 61 | } 62 | } 63 | logger.info("finish onPartitionsRevoked!"); 64 | } 65 | 66 | // onPartitionsAssigned 这个方法 group 已经进行 reassignment 67 | // 之后，开始拉取数据之前调用，作用是清理内存中不属于这个线程的 msg、获取 partition 的 last committed offset。 68 | @Override 69 | public void onPartitionsAssigned(Collection partitions) { 70 | logger.info("start onPartitionsAssigned!"); 71 | Date now = new Date(); 72 | for (TopicPartition partition : partitions) { 73 | consumer.seek(partition, offsetManager.readOffsetFromCache(partition.topic(), partition.partition()).getOffset()); 74 | TopicPartition topicPartition = new TopicPartition(partition.topic(), partition.partition()); 75 | KafkaConsumerOffset kafkaConsumerOffset = KafkaCache.kafkaConsumerOffsetMaps.get(topicPartition); 76 | // 设定owner 77 | kafkaConsumerOffset.setOwner(KafkaMysqlOffsetParameter.kafkaClusterName 78 | + "-" 79 | + partition.topic() 80 | + "-" 81 | + partition.partition() 82 | + "-" 83 | + KafkaMysqlOffsetParameter.consumerGroup 84 | + "-" 85 | + now.getTime() 86 | + "-" 87 | + KafkaMysqlOffsetParameter.hostname 88 | + "-" 89 | + consumer.toString().substring( 90 | consumer.toString().lastIndexOf("@") + 1)); 91 | MysqlOffsetPersist.getInstance().updateOwner(kafkaConsumerOffset); 92 | for (KafkaSubscribeConsumeThread consumeThread : KafkaCache.consumeThreadList) { 93 | if (consumeThread.consumer.equals(consumer)) { 94 | consumeThread.assignedPartitions = partitions; 95 | } 96 | } 97 | } 98 | logger.info("finish onPartitionsAssigned!"); 99 | } 100 | } -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/KafkaSubscribeConsumeThread.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer; 2 | 3 | import cn.thinkingdata.kafka.cache.KafkaCache; 4 | import cn.thinkingdata.kafka.constant.KafkaMysqlOffsetParameter; 5 | import cn.thinkingdata.kafka.consumer.dao.KafkaConsumerOffset; 6 | import cn.thinkingdata.kafka.consumer.offset.MysqlOffsetManager; 7 | import cn.thinkingdata.kafka.consumer.offset.OffsetManager; 8 | import cn.thinkingdata.kafka.consumer.persist.MysqlOffsetPersist; 9 | import cn.thinkingdata.kafka.util.CommonUtils; 10 | import org.apache.commons.collections.CollectionUtils; 11 | import org.apache.commons.lang3.StringUtils; 12 | import org.apache.kafka.clients.consumer.ConsumerRecord; 13 | import org.apache.kafka.clients.consumer.ConsumerRecords; 14 | import org.apache.kafka.clients.consumer.KafkaConsumer; 15 | import org.apache.kafka.clients.consumer.OffsetOutOfRangeException; 16 | import org.apache.kafka.common.TopicPartition; 17 | import org.apache.kafka.common.errors.WakeupException; 18 | import org.joda.time.DateTime; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.util.Arrays; 23 | import java.util.Collection; 24 | import java.util.Date; 25 | import java.util.HashSet; 26 | import java.util.List; 27 | import java.util.Set; 28 | import java.util.concurrent.BlockingQueue; 29 | import java.util.concurrent.CountDownLatch; 30 | import java.util.concurrent.CyclicBarrier; 31 | import java.util.concurrent.LinkedBlockingQueue; 32 | import java.util.concurrent.TimeUnit; 33 | import java.util.stream.Collectors; 34 | 35 | public class KafkaSubscribeConsumeThread implements Runnable { 36 | 37 | private static final Logger logger = LoggerFactory.getLogger(KafkaSubscribeConsumeThread.class); 38 | private final NewIDataLineProcessor dataProcessor; 39 | public KafkaConsumer consumer; 40 | private final OffsetManager offsetManager = MysqlOffsetManager.getInstance(); 41 | public volatile Boolean kafkaPollFlag = false; 42 | public volatile Boolean kafkaConsumerFlag = false; 43 | // public volatile Boolean offsetFlushFlag = false; 44 | private final CyclicBarrier offsetFlushBarrier; 45 | public volatile Collection assignedPartitions = null; 46 | private final BlockingQueue> unsent = new LinkedBlockingQueue<>(); 47 | public Set kafkaConsumerOffsetSet = new HashSet(); 48 | // records的初始值是1000，所以这边capacity的值设为3000 49 | private final BlockingQueue> processDataQueue = 50 | new LinkedBlockingQueue<>(3000); 51 | private volatile Thread consumerThread; 52 | public ProcessDataWorker processDataWorker = new ProcessDataWorker(); 53 | 54 | 55 | /** 56 | * The consumer is currently paused due to a slow execution data. The consumer will be 57 | * resumed when the current batch of records has been processed but will continue 58 | * to be polled. 59 | */ 60 | private volatile Boolean paused = false; 61 | 62 | public KafkaSubscribeConsumeThread(KafkaConsumer consumer, NewIDataLineProcessor dataProcessor, CyclicBarrier offsetFlushBarrier) { 63 | this.consumer = consumer; 64 | this.dataProcessor = dataProcessor; 65 | this.offsetFlushBarrier = offsetFlushBarrier; 66 | } 67 | 68 | @Override 69 | public void run() { 70 | consumerThread = Thread.currentThread(); 71 | kafkaConsumerFlag = true; 72 | //启动processDataWorker 73 | new Thread(processDataWorker, consumerThread.getName() + "-" + "working thread").start(); 74 | Set> lastConsumerRecordSet = new HashSet>(); 75 | Long count = 0L; 76 | DateTime sessionTimeoutDataTime = new DateTime().plusSeconds(Integer.parseInt(KafkaMysqlOffsetParameter.sessionTimeout)); 77 | try { 78 | while (!KafkaMysqlOffsetParameter.kafkaSubscribeConsumerClosed.get()) { 79 | if (KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.get()) { 80 | kafkaPollFlag = true; 81 | count = 0L; 82 | // 如果执行dataExecute的时间超过了sessionTimeout 83 | if (sessionTimeoutDataTime.isAfterNow()) { 84 | // 如果有新的consumer，则调用rebalance，并阻塞线程 85 | ConsumerRecords records = null; 86 | try { 87 | records = consumer.poll(KafkaMysqlOffsetParameter.pollInterval); 88 | } catch (OffsetOutOfRangeException e) { 89 | logger.error("consumer poll out of range, the error is " + CommonUtils.getStackTraceAsString(e)); 90 | for (TopicPartition topicPartition : consumer.assignment()) { 91 | logger.error("the topicPartition is " + topicPartition.toString() + ",the offset is " + consumer.position(topicPartition)); 92 | } 93 | synchronized (OffsetManager.class) { 94 | MysqlOffsetManager.getInstance().getExternalStorePersist().executeWhenOffsetReset(this); 95 | } 96 | } 97 | // 计算开始时间 98 | sessionTimeoutDataTime = new DateTime().plusSeconds(Integer.parseInt(KafkaMysqlOffsetParameter.sessionTimeout) / 1000); 99 | logger.debug("sessionTimeoutDataTime is " + sessionTimeoutDataTime.toString()); 100 | if (records != null) { 101 | if (records.count() > 0) { 102 | logger.debug("poll records size: " + records.count() 103 | + ", partition is " + records.partitions() 104 | + ", thread is " 105 | + Thread.currentThread().getName()); 106 | } 107 | //先暂停 108 | pause(); 109 | //放到队列 110 | sendToQueue(records); 111 | //恢复 112 | if (isResume()) { 113 | resume(); 114 | } else { 115 | // 休息30豪秒 116 | Thread.sleep(30); 117 | } 118 | if (CollectionUtils.isNotEmpty(assignedPartitions)) { 119 | for (TopicPartition assignedPartition : assignedPartitions) { 120 | List> recordsListPerPartition = records.records(assignedPartition); 121 | if (CollectionUtils.isNotEmpty(recordsListPerPartition)) { 122 | ConsumerRecord lastRecord = recordsListPerPartition.get(recordsListPerPartition.size() - 1); 123 | lastConsumerRecordSet.add(lastRecord); 124 | } 125 | } 126 | } 127 | // 更新offset 128 | saveLastConsumerRecordSet(this, lastConsumerRecordSet, count, false); 129 | lastConsumerRecordSet.clear(); 130 | } 131 | } else { 132 | // sessionTimeOut了，进行异常处理 133 | logger.info("kafka session time out, the consumer is " + consumer.toString()); 134 | MysqlOffsetManager.getInstance() 135 | .getExternalStorePersist() 136 | .executeWhenExecuteDataSessionTimeout(this); 137 | break; 138 | } 139 | } else { 140 | logger.info("mysql and backup store connect error, the mysqlAndBackupStoreConnState is " + KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.get()); 141 | kafkaPollFlag = false; 142 | try { 143 | Thread.sleep(50); 144 | } catch (InterruptedException e) { 145 | logger.error("------- thread can not sleep ---------------------" + e.toString()); 146 | } 147 | } 148 | } 149 | kafkaPollFlag = false; 150 | logger.info("kafka consumer close, the kafkaSubscribeConsumerClosed is " 151 | + KafkaMysqlOffsetParameter.kafkaSubscribeConsumerClosed.get()); 152 | } catch (WakeupException | InterruptedException e) { 153 | // 外部thread中断kafka的poll操作 154 | logger.info("stop consumer with wakeup or interupted, the kafkaSubscribeConsumerClosed is " 155 | + KafkaMysqlOffsetParameter.kafkaSubscribeConsumerClosed.get() 156 | + ", the thread is " 157 | + Thread.currentThread().getName() 158 | + ", the consumeThreadList is " 159 | + StringUtils.join(KafkaCache.consumeThreadList.stream().map(o -> o.consumerThread.getName()).collect(Collectors.toList()), ",") 160 | + " the kafka cluster name is " 161 | + KafkaMysqlOffsetParameter.kafkaClusterName 162 | + ", the Exception is " + CommonUtils.getStackTraceAsString(e)); 163 | // 更新offset 164 | saveLastConsumerRecordSet(this, lastConsumerRecordSet, count, true); 165 | kafkaPollFlag = false; 166 | logger.info("stop consumer with wakeup finished"); 167 | } catch (Exception e) { 168 | // 更新offset 169 | saveLastConsumerRecordSet(this, lastConsumerRecordSet, count, false); 170 | logger.error("stop consumer with exception, the kafkaSubscribeConsumerClosed is " 171 | + KafkaMysqlOffsetParameter.kafkaSubscribeConsumerClosed.get() 172 | + ", the thread is " 173 | + Thread.currentThread().getName() 174 | + ", the consumeThreadList is " 175 | + StringUtils.join(KafkaCache.consumeThreadList.stream().map(o -> o.consumerThread.getName()).collect(Collectors.toList()), ",") 176 | + " the kafka cluster name is " 177 | + KafkaMysqlOffsetParameter.kafkaClusterName 178 | + ", the kafkaConsumerOffsetMaps In cache is" + Arrays.toString(KafkaCache.kafkaConsumerOffsetMaps.entrySet().toArray()) + ", the Exception is " + CommonUtils.getStackTraceAsString(e)); 179 | kafkaPollFlag = false; 180 | logger.info("stop consumer finished"); 181 | synchronized (OffsetManager.class) { 182 | MysqlOffsetManager.getInstance().getExternalStorePersist().executeWhenException(); 183 | } 184 | } finally { 185 | try { 186 | closeKafkaSubscribeConsumeThread(); 187 | } catch (Exception e) { 188 | logger.error("closeKafkaSubscribeConsumeThread error, the thread is " + Thread.currentThread().getName() + ", the Exception is " + CommonUtils.getStackTraceAsString(e)); 189 | } 190 | } 191 | } 192 | 193 | private void saveLastConsumerRecordSet(KafkaSubscribeConsumeThread consumeThread, Set> lastConsumerRecordSet, Long count, Boolean cleanOwner) { 194 | for (ConsumerRecord lastConsumerRecord : lastConsumerRecordSet) { 195 | Date now = new Date(); 196 | TopicPartition topicPartition = new TopicPartition(lastConsumerRecord.topic(), lastConsumerRecord.partition()); 197 | KafkaConsumerOffset kafkaConsumerOffset = KafkaCache.kafkaConsumerOffsetMaps.get(topicPartition); 198 | if (kafkaConsumerOffset == null) { 199 | logger.error("kafkaConsumerOffset is null in cache, the lastConsumerRecord is " 200 | + lastConsumerRecord 201 | + ", the kafkaConsumerOffsetMaps is " 202 | + Arrays.toString(KafkaCache.kafkaConsumerOffsetMaps.entrySet().toArray())); 203 | kafkaConsumerOffset = offsetManager.readOffsetFromCache( 204 | lastConsumerRecord.topic(), 205 | lastConsumerRecord.partition()); 206 | // 设定owner 207 | kafkaConsumerOffset 208 | .setOwner(KafkaMysqlOffsetParameter.kafkaClusterName 209 | + "-" 210 | + lastConsumerRecord.topic() 211 | + "-" 212 | + lastConsumerRecord.partition() 213 | + "-" 214 | + KafkaMysqlOffsetParameter.consumerGroup 215 | + "-" 216 | + now.getTime() 217 | + "-" 218 | + KafkaMysqlOffsetParameter.hostname 219 | + "-" 220 | + consumer.toString().substring( 221 | consumer.toString().lastIndexOf("@") + 1)); 222 | } 223 | kafkaConsumerOffset.setTopic(lastConsumerRecord.topic()); 224 | kafkaConsumerOffset.setPartition(lastConsumerRecord.partition()); 225 | kafkaConsumerOffset.setConsumer_group(KafkaMysqlOffsetParameter.consumerGroup); 226 | kafkaConsumerOffset.setOffset(lastConsumerRecord.offset() + 1L); 227 | kafkaConsumerOffset.setKafka_cluster_name(KafkaMysqlOffsetParameter.kafkaClusterName); 228 | kafkaConsumerOffset.setCount(count); 229 | if (cleanOwner) { 230 | //退出的时候清除Owner 231 | kafkaConsumerOffset.setOwner(""); 232 | logger.info("clean owner, the thread is " + Thread.currentThread().getName() + ", the kafkaConsumerOffset is " + kafkaConsumerOffset.toString()); 233 | } 234 | kafkaConsumerOffsetSet.add(kafkaConsumerOffset); 235 | offsetManager.saveOffsetInCache(consumeThread, kafkaConsumerOffset); 236 | } 237 | } 238 | 239 | public void closeKafkaSubscribeConsumeThread() throws InterruptedException { 240 | logger.info("start to stop processDataWorker " + processDataWorker.executingThread.getName()); 241 | processDataWorker.stop(); 242 | logger.info("wait for the mysql persist finish"); 243 | // 等待MysqlOffsetPersist的persist动作完成 244 | for (; ; ) { 245 | if (!MysqlOffsetPersist.runFlag && KafkaMysqlOffsetParameter.kafkaSubscribeConsumerClosed.get()) { 246 | break; 247 | } else { 248 | try { 249 | Thread.sleep(10); 250 | } catch (InterruptedException e) { 251 | logger.error("------- thread can not sleep ---------------------" + e.toString()); 252 | } 253 | } 254 | } 255 | logger.info("flush before kafka consumer close"); 256 | logger.debug("kafkaConsumerOffsetMaps is " 257 | + Arrays.toString(KafkaCache.kafkaConsumerOffsetMaps.entrySet().toArray())); 258 | logger.debug("kafkaConsumerOffsetSet is " + kafkaConsumerOffsetSet + " ,the thread is " + Thread.currentThread().getName()); 259 | try { 260 | for (KafkaConsumerOffset kafkaConsumerOffset : kafkaConsumerOffsetSet) { 261 | TopicPartition topicPartition = new TopicPartition(kafkaConsumerOffset.getTopic(), kafkaConsumerOffset.getPartition()); 262 | KafkaConsumerOffset kafkaConsumerOffsetInCache = KafkaCache.kafkaConsumerOffsetMaps.get(topicPartition); 263 | // 因为有Marking the coordinator 264 | // dead的情况，所以可能kafkaConsumerOffsetSet里有该partition， 265 | // 而另一个线程的kafkaConsumerOffsetSet也有该partition，前一个已经在KafkaCache.kafkaConsumerOffsets中remove了， 266 | // 所以有可能查出来是null 267 | if (kafkaConsumerOffsetInCache != null) { 268 | // 因为有可能mysql里的kafka_consumer_offset为空，consumer拿lastest，这时候的offset不是0，是lastest，是需要保存的 269 | Long consumerPosition = null; 270 | try { 271 | consumerPosition = consumer.position(topicPartition); 272 | } catch (Exception e) { 273 | logger.info("the consumer get position error, the error is " 274 | + e.toString() 275 | + ", the topicPartition is " 276 | + topicPartition); 277 | } 278 | logger.debug("consumer position is " + consumerPosition); 279 | if (consumerPosition != null 280 | && consumerPosition != 0L 281 | && kafkaConsumerOffsetInCache != null 282 | && consumerPosition > kafkaConsumerOffsetInCache.getOffset()) { 283 | logger.info("consumer position " 284 | + consumerPosition 285 | + "is bigger than the offset in kafkaConsumerOffsetInCache " 286 | + kafkaConsumerOffsetInCache); 287 | kafkaConsumerOffsetInCache.setOffset(consumerPosition); 288 | } 289 | MysqlOffsetPersist.getInstance().flush( 290 | kafkaConsumerOffsetInCache); 291 | } else { 292 | logger.error("kafkaConsumerOffsetInCache is null, kafkaConsumerOffset is " 293 | + kafkaConsumerOffset 294 | + ", kafkaConsumerOffsetSet is " 295 | + kafkaConsumerOffsetSet 296 | + ", kafkaConsumerOffsetMaps is " 297 | + Arrays.toString(KafkaCache.kafkaConsumerOffsetMaps.entrySet().toArray())); 298 | } 299 | } 300 | offsetFlushBarrier.await(); 301 | logger.info("start to flush the rest KafkaCache.kafkaConsumerOffsetMaps " 302 | + Arrays.toString(KafkaCache.kafkaConsumerOffsetMaps.entrySet().toArray()) 303 | + ", the thread is " 304 | + Thread.currentThread().getName()); 305 | flushKafkaConsumerOffsetsInKafkaCache(); 306 | consumer.close(); 307 | } catch (Exception e) { 308 | logger.error("close consumer error, the exception is " + CommonUtils.getStackTraceAsString(e)); 309 | consumer.close(); 310 | } 311 | kafkaConsumerFlag = false; 312 | sendUnsentToProcessDataQueue(true); 313 | logger.info("kafka consumer finally close"); 314 | } 315 | 316 | private synchronized void flushKafkaConsumerOffsetsInKafkaCache() { 317 | for (KafkaConsumerOffset kafkaConsumerOffset : KafkaCache.kafkaConsumerOffsetMaps.values()) { 318 | logger.info("kafkaConsumerOffset in cache is not be consumed, kafkaConsumerOffset is " 319 | + kafkaConsumerOffset); 320 | // 因为有可能mysql里的kafka_consumer_offset为空，consumer拿lastest，这时候的offset不是0，是lastest，是需要保存的 321 | TopicPartition topicPartition = new TopicPartition( 322 | kafkaConsumerOffset.getTopic(), 323 | kafkaConsumerOffset.getPartition()); 324 | Long consumerPosition = null; 325 | try { 326 | consumerPosition = consumer.position(topicPartition); 327 | } catch (IllegalArgumentException | IllegalStateException e) { 328 | logger.info("flushKafkaConsumerOffsetsInKafkaCache, the consumer get position error, the error is " 329 | + e.toString() 330 | + ", the topicPartition is " 331 | + topicPartition); 332 | } 333 | if (kafkaConsumerOffset != null) { 334 | if (consumerPosition != null && consumerPosition != 0L 335 | && consumerPosition > kafkaConsumerOffset.getOffset()) { 336 | logger.debug("consumer position is " + consumerPosition); 337 | logger.info("consumer position " + consumerPosition 338 | + " is bigger than the offset in kafkaConsumerOffset " 339 | + kafkaConsumerOffset); 340 | kafkaConsumerOffset.setOffset(consumerPosition); 341 | } 342 | MysqlOffsetPersist.getInstance().flush(kafkaConsumerOffset); 343 | } 344 | } 345 | } 346 | 347 | // Shutdown hook which can be called from a separate thread 348 | public void shutdown() { 349 | KafkaMysqlOffsetParameter.kafkaSubscribeConsumerClosed.set(true); 350 | if (consumer != null) { 351 | consumer.wakeup(); 352 | } 353 | } 354 | 355 | public void pause() { 356 | if (this.assignedPartitions != null) { 357 | // avoid group management rebalance due to a slow 358 | // consumer 359 | this.consumer.pause(this.assignedPartitions); 360 | this.paused = true; 361 | } 362 | } 363 | 364 | public Boolean isResume() { 365 | // 如果unsent为空，则恢复消费 366 | return CollectionUtils.isEmpty(unsent); 367 | } 368 | 369 | public void resume() { 370 | if (this.assignedPartitions != null) { 371 | // avoid group management rebalance due to a slow 372 | // consumer 373 | this.consumer.resume(this.assignedPartitions); 374 | this.paused = false; 375 | } 376 | } 377 | 378 | private void sendToQueue(ConsumerRecords records) throws InterruptedException { 379 | Boolean flag = true; 380 | if (CollectionUtils.isEmpty(unsent)) { 381 | for (ConsumerRecord record : records) { 382 | if (flag) { 383 | flag = this.processDataQueue.offer(record, 200, TimeUnit.MILLISECONDS); 384 | //如果没有放入成功说明队列已满 385 | if (!flag) { 386 | logger.info("the processDataQueue is full..."); 387 | unsent.put(record); 388 | } 389 | } else { 390 | unsent.put(record); 391 | } 392 | } 393 | } else { 394 | if (records.count() > 0) { 395 | logger.info("the unsent is not empty but the consummer still polling records, it can be only happed after rebalanced"); 396 | } 397 | for (ConsumerRecord record : records) { 398 | unsent.put(record); 399 | } 400 | try { 401 | Thread.sleep(100); 402 | } catch (InterruptedException e) { 403 | logger.error("------- thread can not sleep ---------------------" 404 | + e.toString()); 405 | } 406 | // 试着将unsent里的records放入processDataQueue 407 | sendUnsentToProcessDataQueue(false); 408 | } 409 | } 410 | 411 | private void sendUnsentToProcessDataQueue(Boolean shutdown) throws InterruptedException { 412 | while (CollectionUtils.isNotEmpty(unsent)) { 413 | //拿出队首元素但不出栈 414 | ConsumerRecord recordInUnsent = unsent.peek(); 415 | if (recordInUnsent != null) { 416 | Boolean flag = this.processDataQueue.offer(recordInUnsent, 200, TimeUnit.MILLISECONDS); 417 | if (!flag) { 418 | //如果没有放入processDataQueue成功说明队列已满 419 | logger.info("the processDataQueue is full... and the unsent is not empty"); 420 | //如果没有停止，则跳出，否则需要将unsent清空才能退出 421 | if (!shutdown) { 422 | break; 423 | } else { 424 | try { 425 | Thread.sleep(10); 426 | } catch (InterruptedException e) { 427 | logger.error("------- thread can not sleep ---------------------" + e.toString()); 428 | } 429 | } 430 | } else { 431 | //如果放入processDataQueue成功则出栈 432 | unsent.poll(); 433 | } 434 | } else { 435 | logger.error("the unsent is not empty, but the recordInUnsent is null!!"); 436 | } 437 | } 438 | } 439 | 440 | //processData线程消费queue 441 | public final class ProcessDataWorker implements Runnable { 442 | 443 | private final CountDownLatch exitLatch = new CountDownLatch(1); 444 | private volatile Thread executingThread; 445 | private volatile Boolean workerStopFlag = false; 446 | public volatile Boolean workingFlag = false; 447 | private volatile ConsumerRecord consumerRecord; 448 | private static final long MAX_WAIT_MS = 1000; 449 | 450 | @Override 451 | public void run() { 452 | workingFlag = true; 453 | try { 454 | this.executingThread = Thread.currentThread(); 455 | while (true) { 456 | processOperationData(); 457 | // 如果queue是空，并且stop为true则退出 458 | if (processDataQueue.size() == 0 && workerStopFlag && unsent.size() == 0) { 459 | break; 460 | } 461 | } 462 | } catch (Exception e) { 463 | logger.error("processDataWorker thread is failed, the error is " + e.toString()); 464 | } finally { 465 | logger.info("processDataWorker " + Thread.currentThread().getName() + " is safely closed..."); 466 | exitLatch.countDown(); 467 | workingFlag = false; 468 | } 469 | } 470 | 471 | private void processOperationData() throws InterruptedException { 472 | // 如果出现除InterruptedException的错误，则必须catch住，要不然，线程会中断！ 473 | try { 474 | consumerRecord = processDataQueue.poll(MAX_WAIT_MS, TimeUnit.MILLISECONDS); 475 | if (consumerRecord != null) { 476 | dataProcessor.processData(consumerRecord); 477 | } 478 | } catch (InterruptedException e) { 479 | throw e; 480 | } catch (Exception e) { 481 | logger.error("processOperationData error, the error is " + CommonUtils.getStackTraceAsString(e)); 482 | } 483 | } 484 | 485 | public void stopWithException() { 486 | try { 487 | Thread.sleep(1000); 488 | } catch (InterruptedException e) { 489 | logger.error("------- thread can not sleep ---------------------" + e.toString()); 490 | } 491 | workerStopFlag = true; 492 | } 493 | 494 | 495 | private void stop() { 496 | // 等待拉取动作结束 497 | for (; ; ) { 498 | if (kafkaPollFlag == false) { 499 | break; 500 | } else { 501 | try { 502 | Thread.sleep(10); 503 | } catch (InterruptedException e) { 504 | logger.error("------- thread can not sleep ---------------------" + e.toString()); 505 | } 506 | } 507 | } 508 | workerStopFlag = true; 509 | } 510 | } 511 | } 512 | 513 | 514 | 515 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/KafkaSubscribeConsumer.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer; 2 | 3 | import cn.thinkingdata.kafka.cache.KafkaCache; 4 | import cn.thinkingdata.kafka.close.DaemonCloseThread; 5 | import cn.thinkingdata.kafka.close.TermMethod; 6 | import cn.thinkingdata.kafka.constant.KafkaMysqlOffsetParameter; 7 | import cn.thinkingdata.kafka.consumer.offset.MysqlOffsetManager; 8 | import cn.thinkingdata.kafka.consumer.persist.MysqlOffsetPersist; 9 | import cn.thinkingdata.kafka.consumer.persist.StorePersist; 10 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 11 | import org.apache.kafka.clients.consumer.KafkaConsumer; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.ArrayList; 16 | import java.util.Arrays; 17 | import java.util.List; 18 | import java.util.Map; 19 | import java.util.concurrent.CyclicBarrier; 20 | import java.util.concurrent.ExecutorService; 21 | import java.util.concurrent.Executors; 22 | import java.util.concurrent.ThreadFactory; 23 | import java.util.concurrent.TimeUnit; 24 | 25 | public class KafkaSubscribeConsumer { 26 | 27 | private static final Logger logger = LoggerFactory.getLogger(KafkaSubscribeConsumer.class); 28 | 29 | protected NewIDataLineProcessor dataProcessor; 30 | protected volatile ExecutorService executorService; 31 | private final TermMethod closeMethod; 32 | private volatile DaemonCloseThread closeSignal; 33 | private static volatile Integer startCount = 0; 34 | 35 | public KafkaSubscribeConsumer(Map map, NewIDataLineProcessor dataProcessor, TermMethod closeMethod) { 36 | KafkaMysqlOffsetParameter.createKafkaConfProp(map); 37 | this.dataProcessor = dataProcessor; 38 | this.closeMethod = closeMethod; 39 | } 40 | 41 | public KafkaSubscribeConsumer(Map map, NewIDataLineProcessor dataProcessor, TermMethod closeMethod, StorePersist externalStorePersist){ 42 | this(map, dataProcessor, closeMethod); 43 | MysqlOffsetManager.getInstance().setExternalStorePersist(externalStorePersist); 44 | } 45 | 46 | public void run() { 47 | //判断mysql和redis是否通 48 | Boolean mysqlStateCheck = MysqlOffsetPersist.getInstance().mysqlStateCheckWithRetry(); 49 | if(!mysqlStateCheck){ 50 | logger.info("mysql is not connected!"); 51 | System.exit(-1); 52 | } 53 | Boolean backupStoreStateCheck = MysqlOffsetPersist.getInstance().backupStoreStateCheckWithRetry(); 54 | if(!backupStoreStateCheck){ 55 | logger.info("backup store is not connected!"); 56 | System.exit(-1); 57 | } 58 | KafkaMysqlOffsetParameter.kafkaSubscribeConsumerClosed.set(false); 59 | List topicList = new ArrayList(); 60 | topicList.addAll(Arrays.asList(KafkaMysqlOffsetParameter.topic.split(","))); 61 | ThreadFactory kafkaConsumeThreadNamedThreadFactory = new ThreadFactoryBuilder().setNameFormat("kafka-consume-thread-%d").build(); 62 | executorService = Executors.newFixedThreadPool(KafkaMysqlOffsetParameter.processThreadNum,kafkaConsumeThreadNamedThreadFactory); 63 | CyclicBarrier offsetFlushBarrier = new CyclicBarrier(KafkaMysqlOffsetParameter.processThreadNum); 64 | for (int i = 0; i < KafkaMysqlOffsetParameter.processThreadNum; i++) { 65 | KafkaSubscribeConsumerManager kafkaSubscribeConsumer = KafkaSubscribeConsumerManager.getInstance(); 66 | KafkaConsumer consumer = kafkaSubscribeConsumer.createKafkaConsumer(topicList, KafkaMysqlOffsetParameter.kafkaConf); 67 | KafkaSubscribeConsumeThread consumeThread = new KafkaSubscribeConsumeThread(consumer, dataProcessor, offsetFlushBarrier); 68 | KafkaCache.consumeThreadList.add(consumeThread); 69 | executorService.submit(consumeThread); 70 | } 71 | // 启动定时刷数据入mysql 72 | if(startCount.equals(0)){ 73 | MysqlOffsetPersist.getInstance().start(); 74 | startCount = startCount + 1; 75 | } 76 | closeSignal = new DaemonCloseThread(this, closeMethod); 77 | closeSignal.setDaemon(true); 78 | closeSignal.start(); 79 | } 80 | 81 | public void stop() { 82 | stop(120000); 83 | } 84 | 85 | public void stop(long stopTimeOut) { 86 | long startTime = System.currentTimeMillis(); 87 | logger.info("consumers start shutdown"); 88 | for (KafkaSubscribeConsumeThread consumeThread : KafkaCache.consumeThreadList) { 89 | if(consumeThread != null){ 90 | consumeThread.shutdown(); 91 | } 92 | } 93 | Boolean stopExceptionFlag = false; 94 | // 等待所有拉取线程自动停止 95 | for (;;) { 96 | if(stopExceptionFlag){ 97 | break; 98 | } 99 | Boolean kafkaPollFlag = false; 100 | for (KafkaSubscribeConsumeThread consumeThread : KafkaCache.consumeThreadList) { 101 | if (consumeThread != null && consumeThread.kafkaPollFlag) { 102 | kafkaPollFlag = true; 103 | } 104 | } 105 | if (!kafkaPollFlag) { 106 | break; 107 | } 108 | if(System.currentTimeMillis()-startTime > stopTimeOut){ 109 | stopWithTimeOUt(); 110 | stopExceptionFlag = true; 111 | } 112 | } 113 | logger.info("kafka polling closed"); 114 | // 等待所有consumer关闭 115 | for (;;) { 116 | if(stopExceptionFlag){ 117 | break; 118 | } 119 | Boolean kafkaConsumerFlag = false; 120 | for (KafkaSubscribeConsumeThread consumeThread : KafkaCache.consumeThreadList) { 121 | if (consumeThread != null && consumeThread.kafkaConsumerFlag) { 122 | kafkaConsumerFlag = true; 123 | } 124 | } 125 | if (!kafkaConsumerFlag) { 126 | break; 127 | } 128 | if(System.currentTimeMillis()-startTime > stopTimeOut){ 129 | stopWithTimeOUt(); 130 | stopExceptionFlag = true; 131 | } 132 | } 133 | logger.info("kafka consumer closed"); 134 | // 等待所有consumer的working线程关闭 135 | for (;;) { 136 | if(stopExceptionFlag){ 137 | break; 138 | } 139 | Boolean processDataWorkingFlag = false; 140 | for (KafkaSubscribeConsumeThread consumeThread : KafkaCache.consumeThreadList) { 141 | if (consumeThread != null && consumeThread.processDataWorker.workingFlag) { 142 | processDataWorkingFlag = true; 143 | } 144 | } 145 | if (!processDataWorkingFlag) { 146 | break; 147 | } 148 | if(System.currentTimeMillis()-startTime > stopTimeOut){ 149 | stopWithTimeOUt(); 150 | stopExceptionFlag = true; 151 | } 152 | } 153 | logger.info("process data worker closed"); 154 | // 关闭线程池 155 | if (executorService != null) 156 | executorService.shutdown(); 157 | try { 158 | if (!executorService.awaitTermination(120000, TimeUnit.MILLISECONDS)) { 159 | logger.error("Timed out waiting for consumer threads to shut down, exiting uncleanly"); 160 | } 161 | } catch (InterruptedException e) { 162 | logger.error("Interrupted during shutdown, exiting uncleanly"); 163 | } 164 | logger.info("dataProcessor start to shutdown"); 165 | dataProcessor.finishProcess(); 166 | KafkaCache.kafkaConsumerOffsetMaps.clear(); 167 | KafkaCache.consumeThreadList.clear(); 168 | KafkaCache.rebalancerListenerList.clear(); 169 | closeSignal.shutdown(); 170 | } 171 | 172 | private void stopWithTimeOUt() { 173 | logger.info("kafka polling/kafka consumer/process data worker closed with timeout"); 174 | for (KafkaSubscribeConsumeThread consumeThread : KafkaCache.consumeThreadList) { 175 | if(consumeThread != null && consumeThread.processDataWorker != null){ 176 | consumeThread.processDataWorker.stopWithException(); 177 | } 178 | } 179 | try { 180 | // 等待所有拉取线程自动停止 181 | Thread.sleep(5000); 182 | // 等待所有consumer关闭 183 | Thread.sleep(5000); 184 | // 等待所有consumer的working线程关闭 185 | Thread.sleep(5000); 186 | } catch (InterruptedException e2) { 187 | logger.error("------- thread can not sleep ---------------------" 188 | + e2.toString()); 189 | } 190 | } 191 | 192 | public void destroy() { 193 | MysqlOffsetPersist.destoryFlag = true; 194 | stop(); 195 | closeSignal.afterDestroyConsumer(); 196 | logger.info("mysql start to shutdown"); 197 | // 关闭mysql连接 198 | MysqlOffsetPersist.getInstance().shutdown(); 199 | } 200 | 201 | public void destroy(long stopTimeOut) { 202 | MysqlOffsetPersist.destoryFlag = true; 203 | stop(stopTimeOut); 204 | closeSignal.afterDestroyConsumer(); 205 | logger.info("mysql start to shutdown"); 206 | // 关闭mysql连接 207 | MysqlOffsetPersist.getInstance().shutdown(); 208 | } 209 | 210 | } 211 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/KafkaSubscribeConsumerManager.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer; 2 | 3 | import cn.thinkingdata.kafka.cache.KafkaCache; 4 | import org.apache.kafka.clients.consumer.KafkaConsumer; 5 | 6 | import java.util.List; 7 | import java.util.Properties; 8 | 9 | public class KafkaSubscribeConsumerManager { 10 | 11 | private static KafkaSubscribeConsumerManager instance; 12 | 13 | private KafkaSubscribeConsumerManager() { 14 | } 15 | 16 | public static synchronized KafkaSubscribeConsumerManager getInstance() { 17 | if (instance == null) { 18 | instance = new KafkaSubscribeConsumerManager(); 19 | } 20 | return instance; 21 | } 22 | 23 | public KafkaConsumer createKafkaConsumer(List topicList, Properties props) { 24 | KafkaConsumer consumer = new KafkaConsumer(props); 25 | KafkaConsumerRebalancerListener rebalancerListener = new KafkaConsumerRebalancerListener(consumer); 26 | KafkaCache.rebalancerListenerList.add(rebalancerListener); 27 | consumer.subscribe(topicList, rebalancerListener); 28 | return consumer; 29 | } 30 | } -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/NewIDataLineProcessor.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer; 2 | 3 | import org.apache.kafka.clients.consumer.ConsumerRecord; 4 | 5 | public interface NewIDataLineProcessor { 6 | 7 | void processData(ConsumerRecord consumerRecord); 8 | 9 | void finishProcess(); 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/dao/KafkaConsumerOffset.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer.dao; 2 | 3 | import java.util.Date; 4 | 5 | public class KafkaConsumerOffset { 6 | private Integer oid; 7 | private String topic; 8 | private Integer partition; 9 | private String consumer_group; 10 | private Long offset; 11 | // 上一次flush记录下的offset值 12 | private Long last_flush_offset; 13 | // 计数器 每消费一个记录，计数器加一，flush完，计数器清零 14 | private Long count; 15 | private String kafka_cluster_name; 16 | private String owner; 17 | private Date update_time; 18 | private Date create_time; 19 | 20 | public Integer getOid() { 21 | return oid; 22 | } 23 | 24 | public void setOid(Integer oid) { 25 | this.oid = oid; 26 | } 27 | 28 | public String getTopic() { 29 | return topic; 30 | } 31 | 32 | public void setTopic(String topic) { 33 | this.topic = topic; 34 | } 35 | 36 | public Integer getPartition() { 37 | return partition; 38 | } 39 | 40 | public void setPartition(Integer partition) { 41 | this.partition = partition; 42 | } 43 | 44 | public String getConsumer_group() { 45 | return consumer_group; 46 | } 47 | 48 | public void setConsumer_group(String consumer_group) { 49 | this.consumer_group = consumer_group; 50 | } 51 | 52 | public Long getOffset() { 53 | return offset; 54 | } 55 | 56 | public void setOffset(Long offset) { 57 | this.offset = offset; 58 | } 59 | 60 | public Long getLast_flush_offset() { 61 | return last_flush_offset; 62 | } 63 | 64 | public void setLast_flush_offset(Long last_flush_offset) { 65 | this.last_flush_offset = last_flush_offset; 66 | } 67 | 68 | public Long getCount() { 69 | return count; 70 | } 71 | 72 | public void setCount(Long count) { 73 | this.count = count; 74 | } 75 | 76 | public String getKafka_cluster_name() { 77 | return kafka_cluster_name; 78 | } 79 | 80 | public void setKafka_cluster_name(String kafka_cluster_name) { 81 | this.kafka_cluster_name = kafka_cluster_name; 82 | } 83 | 84 | public String getOwner() { 85 | return owner; 86 | } 87 | 88 | public void setOwner(String owner) { 89 | this.owner = owner; 90 | } 91 | 92 | public Date getUpdate_time() { 93 | return update_time; 94 | } 95 | 96 | public void setUpdate_time(Date update_time) { 97 | this.update_time = update_time; 98 | } 99 | 100 | public Date getCreate_time() { 101 | return create_time; 102 | } 103 | 104 | public void setCreate_time(Date create_time) { 105 | this.create_time = create_time; 106 | } 107 | 108 | @Override 109 | public String toString() { 110 | return "KafkaConsumerOffset [oid=" + oid + ", topic=" + topic 111 | + ", partition=" + partition + ", consumer_group=" 112 | + consumer_group + ", offset=" + offset 113 | + ", last_flush_offset=" + last_flush_offset + ", count=" 114 | + count + ", kafka_cluster_name=" + kafka_cluster_name 115 | + ", owner=" + owner + ", update_time=" + update_time 116 | + ", create_time=" + create_time + "]"; 117 | } 118 | 119 | @Override 120 | public int hashCode() { 121 | final int prime = 31; 122 | int result = 1; 123 | result = prime * result + ((consumer_group == null) ? 0 : consumer_group.hashCode()); 124 | result = prime * result + ((kafka_cluster_name == null) ? 0 : kafka_cluster_name.hashCode()); 125 | result = prime * result + ((partition == null) ? 0 : partition.hashCode()); 126 | result = prime * result + ((topic == null) ? 0 : topic.hashCode()); 127 | return result; 128 | } 129 | 130 | @Override 131 | public boolean equals(Object obj) { 132 | if (this == obj) 133 | return true; 134 | if (obj == null) 135 | return false; 136 | if (getClass() != obj.getClass()) 137 | return false; 138 | KafkaConsumerOffset other = (KafkaConsumerOffset) obj; 139 | if (consumer_group == null) { 140 | if (other.consumer_group != null) 141 | return false; 142 | } else if (!consumer_group.equals(other.consumer_group)) 143 | return false; 144 | if (kafka_cluster_name == null) { 145 | if (other.kafka_cluster_name != null) 146 | return false; 147 | } else if (!kafka_cluster_name.equals(other.kafka_cluster_name)) 148 | return false; 149 | if (partition == null) { 150 | if (other.partition != null) 151 | return false; 152 | } else if (!partition.equals(other.partition)) 153 | return false; 154 | if (topic == null) { 155 | return other.topic == null; 156 | } else return topic.equals(other.topic); 157 | } 158 | 159 | public boolean isNull() { 160 | if (this == null) 161 | return true; 162 | return this.topic == null || this.partition == null || this.kafka_cluster_name == null || this.consumer_group == null; 163 | 164 | } 165 | 166 | 167 | } 168 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/exception/ExceptionHandler.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer.exception; 2 | 3 | import cn.thinkingdata.kafka.consumer.KafkaSubscribeConsumeThread; 4 | import cn.thinkingdata.kafka.consumer.dao.KafkaConsumerOffset; 5 | 6 | public interface ExceptionHandler { 7 | 8 | KafkaConsumerOffset executeWhenReadNullFromBackupExternalStore(String topic, Integer partition); 9 | 10 | KafkaConsumerOffset executeWhenReadNullFromMysql(String topic, Integer partition); 11 | 12 | Boolean executeWhenSaveOffsetFailInMysqlAndExternalStore(KafkaConsumerOffset kafkaConsumerOffset); 13 | 14 | void executeWhenSessionTimeout(Integer count); 15 | 16 | void executeWhenExecuteDataSessionTimeout(KafkaSubscribeConsumeThread kafkaSubscribeConsumeThread); 17 | 18 | void executeWhenOffsetReset(KafkaSubscribeConsumeThread consumeThread); 19 | 20 | void executeWhenException(); 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/exception/TaKafkaCommonException.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer.exception; 2 | 3 | public class TaKafkaCommonException extends RuntimeException { 4 | 5 | public TaKafkaCommonException(String msg) { 6 | super(msg); 7 | } 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/offset/MysqlOffsetManager.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer.offset; 2 | 3 | import cn.thinkingdata.kafka.constant.KafkaMysqlOffsetParameter; 4 | import cn.thinkingdata.kafka.consumer.dao.KafkaConsumerOffset; 5 | import cn.thinkingdata.kafka.consumer.persist.DBPoolConnection; 6 | import cn.thinkingdata.kafka.util.CommonUtils; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.sql.Connection; 11 | import java.sql.PreparedStatement; 12 | import java.sql.ResultSet; 13 | import java.sql.SQLException; 14 | import java.sql.Statement; 15 | import java.sql.Timestamp; 16 | import java.util.Date; 17 | 18 | public class MysqlOffsetManager extends OffsetManager { 19 | 20 | private static MysqlOffsetManager instance; 21 | 22 | private static final Logger logger = LoggerFactory 23 | .getLogger(MysqlOffsetManager.class); 24 | 25 | public static synchronized MysqlOffsetManager getInstance() { 26 | if (instance == null) { 27 | instance = new MysqlOffsetManager(); 28 | } 29 | return instance; 30 | } 31 | 32 | DBPoolConnection dbp = DBPoolConnection.getInstance(); 33 | 34 | private MysqlOffsetManager() { 35 | } 36 | 37 | //去掉synchronized，因为MysqlOffsetPersist的flush和persist里有synchronized方法 38 | @Override 39 | protected Boolean saveOffsetInExternalStore(KafkaConsumerOffset kafkaConsumerOffset) { 40 | logger.debug("because of the muti-thread, the value is not exactly right, kafkaConsumerOffset is " + kafkaConsumerOffset.toString()); 41 | try (Connection conn = dbp.getConnection()) { 42 | String sql = "INSERT INTO " 43 | + KafkaMysqlOffsetParameter.tableName 44 | + " VALUES" 45 | + " (null,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY" 46 | + " UPDATE offset=?, last_flush_offset=?, kafka_cluster_name=?," 47 | + " owner=?, update_time=?;"; 48 | PreparedStatement ps = conn.prepareStatement(sql); 49 | ps.setString(1, kafkaConsumerOffset.getTopic()); 50 | ps.setInt(2, kafkaConsumerOffset.getPartition()); 51 | ps.setString(3, kafkaConsumerOffset.getConsumer_group()); 52 | ps.setLong(5, kafkaConsumerOffset.getLast_flush_offset()); 53 | ps.setLong(4, kafkaConsumerOffset.getOffset()); 54 | ps.setString(6, kafkaConsumerOffset.getKafka_cluster_name()); 55 | ps.setString(7, kafkaConsumerOffset.getOwner()); 56 | ps.setTimestamp(8, new Timestamp(kafkaConsumerOffset.getUpdate_time().getTime())); 57 | ps.setTimestamp(9, new Timestamp(kafkaConsumerOffset.getCreate_time().getTime())); 58 | ps.setLong(11, kafkaConsumerOffset.getLast_flush_offset()); 59 | ps.setLong(10, kafkaConsumerOffset.getOffset()); 60 | ps.setString(12, kafkaConsumerOffset.getKafka_cluster_name()); 61 | ps.setString(13, kafkaConsumerOffset.getOwner()); 62 | ps.setTimestamp(14, new Timestamp(kafkaConsumerOffset.getUpdate_time().getTime())); 63 | ps.execute(); 64 | return true; 65 | } catch (SQLException e) { 66 | logger.error("mysql save offset error, the error is " + CommonUtils.getStackTraceAsString(e)); 67 | return false; 68 | } 69 | } 70 | 71 | @Override 72 | protected KafkaConsumerOffset readOffsetFromExternalStore(String topic, 73 | int partition) { 74 | KafkaConsumerOffset kafkaConsumerOffset = new KafkaConsumerOffset(); 75 | Date now = new Date(); 76 | String sql = "select * from " + KafkaMysqlOffsetParameter.tableName 77 | + " where kafka_cluster_name = '" 78 | + KafkaMysqlOffsetParameter.kafkaClusterName 79 | + "' and topic = '" + topic + "' and kafka_partition = " 80 | + partition + " and consumer_group = '" 81 | + KafkaMysqlOffsetParameter.consumerGroup + "';"; 82 | try (Connection conn = dbp.getConnection(); Statement statement = conn.createStatement(); ResultSet rs = statement.executeQuery(sql)){ 83 | int count = 0; 84 | while (rs.next()) { 85 | count++; 86 | if (count > 1) { 87 | logger.error("DUPLICATE KEY in " 88 | + KafkaMysqlOffsetParameter.tableName 89 | + " , the kafka cluster name is " 90 | + KafkaMysqlOffsetParameter.kafkaClusterName 91 | + " , the topic is " + topic 92 | + ", the partition is " + partition 93 | + ", the consumerGroup is " 94 | + KafkaMysqlOffsetParameter.consumerGroup); 95 | return kafkaConsumerOffset; 96 | } 97 | kafkaConsumerOffset.setOid(rs.getInt("oid")); 98 | kafkaConsumerOffset.setTopic(topic); 99 | kafkaConsumerOffset.setPartition(partition); 100 | kafkaConsumerOffset.setConsumer_group(KafkaMysqlOffsetParameter.consumerGroup); 101 | kafkaConsumerOffset.setOffset(rs.getLong("offset")); 102 | kafkaConsumerOffset.setLast_flush_offset(rs.getLong("offset")); 103 | kafkaConsumerOffset.setKafka_cluster_name(KafkaMysqlOffsetParameter.kafkaClusterName); 104 | kafkaConsumerOffset.setOwner(rs.getString("owner")); 105 | kafkaConsumerOffset.setCount(0L); 106 | kafkaConsumerOffset.setUpdate_time(rs.getDate("update_time")); 107 | kafkaConsumerOffset.setCreate_time(rs.getDate("create_time")); 108 | } 109 | if (count == 0) { 110 | logger.info("offset is not in " 111 | + KafkaMysqlOffsetParameter.tableName 112 | + " , the kafka cluster name is " 113 | + KafkaMysqlOffsetParameter.kafkaClusterName 114 | + " , the topic is " + topic + ", the partition is " 115 | + partition + ", the consumerGroup is " 116 | + KafkaMysqlOffsetParameter.consumerGroup); 117 | kafkaConsumerOffset.setTopic(topic); 118 | kafkaConsumerOffset.setPartition(partition); 119 | kafkaConsumerOffset.setConsumer_group(KafkaMysqlOffsetParameter.consumerGroup); 120 | kafkaConsumerOffset.setOffset(0L); 121 | kafkaConsumerOffset.setLast_flush_offset(0L); 122 | kafkaConsumerOffset.setKafka_cluster_name(KafkaMysqlOffsetParameter.kafkaClusterName); 123 | kafkaConsumerOffset.setCount(0L); 124 | kafkaConsumerOffset.setUpdate_time(now); 125 | return kafkaConsumerOffset; 126 | } 127 | } catch (Exception e) { 128 | logger.error("mysql read offset error, the error is " + CommonUtils.getStackTraceAsString(e)); 129 | return null; 130 | } 131 | return kafkaConsumerOffset; 132 | } 133 | 134 | public void shutdown() { 135 | logger.info("mysql shutdown!"); 136 | try { 137 | dbp.close(); 138 | } catch (Exception e) { 139 | logger.error("can not close mysql connection pool, the error is " + CommonUtils.getStackTraceAsString(e)); 140 | } 141 | } 142 | 143 | //去掉synchronized，因为MysqlOffsetPersist的flush和persist里有synchronized方法 144 | public Boolean saveOffsetInCacheToMysql(KafkaConsumerOffset kafkaConsumerOffset) { 145 | Long lag = kafkaConsumerOffset.getOffset() - kafkaConsumerOffset.getLast_flush_offset(); 146 | if (!lag.equals(0L)) { 147 | logger.debug("because of the muti-thread, the value is not exactly right, the lag is " + lag); 148 | return saveOffsetInExternalStore(kafkaConsumerOffset); 149 | } 150 | return true; 151 | } 152 | 153 | public Boolean mysqlStateCheck() { 154 | String sql = "select * from " + KafkaMysqlOffsetParameter.tableName + " limit 10;"; 155 | try (Connection conn = dbp.getConnection(); Statement statement = conn.createStatement()) { 156 | logger.info("mysql reconnected!"); 157 | statement.execute(sql); 158 | return true; 159 | } catch (SQLException e) { 160 | return false; 161 | } 162 | } 163 | 164 | public Boolean updateOwner(KafkaConsumerOffset kafkaConsumerOffset) { 165 | logger.debug("update the owner for kafkaConsumerOffset, kafkaConsumerOffset is " 166 | + kafkaConsumerOffset.toString()); 167 | Date now = new Date(); 168 | Boolean flag = true; 169 | try (Connection conn = dbp.getConnection(); Statement statement = conn.createStatement()){ 170 | if (kafkaConsumerOffset.getOffset() == 0L) { 171 | kafkaConsumerOffset.setUpdate_time(now); 172 | if (kafkaConsumerOffset.getCreate_time() == null) 173 | kafkaConsumerOffset.setCreate_time(now); 174 | flag = saveOffsetInExternalStore(kafkaConsumerOffset); 175 | } else { 176 | String sql = "UPDATE " + KafkaMysqlOffsetParameter.tableName 177 | + " set owner='" + kafkaConsumerOffset.getOwner() + "', update_time = NOW()" 178 | + " where kafka_cluster_name = '" 179 | + KafkaMysqlOffsetParameter.kafkaClusterName 180 | + "' and topic = '" + kafkaConsumerOffset.getTopic() 181 | + "' and kafka_partition = " 182 | + kafkaConsumerOffset.getPartition() 183 | + " and consumer_group = '" 184 | + KafkaMysqlOffsetParameter.consumerGroup + "';"; 185 | statement.execute(sql); 186 | } 187 | if (flag) { 188 | KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.set(true); 189 | return true; 190 | } else { 191 | KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.set(false); 192 | logger.error("mysql update the owner error"); 193 | return false; 194 | } 195 | 196 | } catch (SQLException e) { 197 | KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.set(false); 198 | logger.error("mysql update the owner error, the error is " + CommonUtils.getStackTraceAsString(e)); 199 | return false; 200 | } 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/offset/OffsetManager.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer.offset; 2 | 3 | import cn.thinkingdata.kafka.cache.KafkaCache; 4 | import cn.thinkingdata.kafka.constant.KafkaMysqlOffsetParameter; 5 | import cn.thinkingdata.kafka.consumer.KafkaSubscribeConsumeThread; 6 | import cn.thinkingdata.kafka.consumer.dao.KafkaConsumerOffset; 7 | import cn.thinkingdata.kafka.consumer.persist.DefaultStorePersist; 8 | import cn.thinkingdata.kafka.consumer.persist.StorePersist; 9 | import cn.thinkingdata.kafka.util.CommonUtils; 10 | import cn.thinkingdata.kafka.util.RetryerUtil; 11 | import com.github.rholder.retry.RetryException; 12 | import com.github.rholder.retry.Retryer; 13 | import com.google.common.base.Predicates; 14 | import org.apache.kafka.common.TopicPartition; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | import java.util.concurrent.ExecutionException; 19 | 20 | public abstract class OffsetManager { 21 | 22 | private static final Logger logger = LoggerFactory.getLogger(OffsetManager.class); 23 | 24 | private final Retryer retryerWithResultNull = RetryerUtil.initRetryerByTimesWithIfResult(3, 300, Predicates.isNull()); 25 | 26 | private StorePersist externalStorePersist = new DefaultStorePersist(); 27 | 28 | public StorePersist getExternalStorePersist() { 29 | return externalStorePersist; 30 | } 31 | 32 | public void setExternalStorePersist(StorePersist externalStorePersist) { 33 | this.externalStorePersist = externalStorePersist; 34 | } 35 | 36 | public void saveOffsetInCache(KafkaSubscribeConsumeThread consumeThread, KafkaConsumerOffset kafkaConsumerOffset) { 37 | TopicPartition topicPartition = new TopicPartition(kafkaConsumerOffset.getTopic(), kafkaConsumerOffset.getPartition()); 38 | KafkaConsumerOffset kafkaConsumerOffsetOld = KafkaCache.kafkaConsumerOffsetMaps.get(topicPartition); 39 | // compare kafkaConsumerOffsetOld and kafkaConsumerOffset, avoid reset 40 | if (kafkaConsumerOffsetOld != null && kafkaConsumerOffsetOld.getOffset() > kafkaConsumerOffset.getOffset()) { 41 | logger.info("kafka consumer offset reset, the old kafkaConsumerOffset is " + kafkaConsumerOffsetOld + ", the kafkaConsumerOffset is " + kafkaConsumerOffset); 42 | synchronized (OffsetManager.class) { 43 | externalStorePersist.executeWhenOffsetReset(consumeThread); 44 | } 45 | } else if (kafkaConsumerOffsetOld == null 46 | || !kafkaConsumerOffset.getCount().equals(0L)) { 47 | KafkaCache.kafkaConsumerOffsetMaps.put(topicPartition, kafkaConsumerOffset); 48 | kafkaConsumerOffset.setCount(0L); 49 | } 50 | } 51 | 52 | public KafkaConsumerOffset readOffsetFromMysql(final String topic, final Integer partition) { 53 | KafkaConsumerOffset kafkaConsumerOffset = null; 54 | try { 55 | kafkaConsumerOffset = retryerWithResultNull 56 | .call(() -> readOffsetFromExternalStore(topic, partition)); 57 | if (kafkaConsumerOffset == null) { 58 | logger.error("the kafkaConsumerOffset read from mysql is null , the topic is " + topic + "the partition is " + partition); 59 | } 60 | } catch (ExecutionException | RetryException e) { 61 | logger.error("retry to read kafkaConsumerOffset from mysql error, the error is " + CommonUtils.getStackTraceAsString(e)); 62 | return null; 63 | } 64 | return kafkaConsumerOffset; 65 | } 66 | 67 | public KafkaConsumerOffset readOffsetFromBackupExternalStore( 68 | final String topic, final Integer partition) { 69 | KafkaConsumerOffset kafkaConsumerOffset = null; 70 | try { 71 | kafkaConsumerOffset = retryerWithResultNull 72 | .call(() -> externalStorePersist.readOffsetFromBackupExternalStore(topic, partition)); 73 | if (kafkaConsumerOffset == null) { 74 | logger.error("the kafkaConsumerOffset read from backup external store is null , the topic is " + topic + "the partition is " + partition); 75 | } 76 | } catch (ExecutionException | RetryException e) { 77 | logger.error("retry to read kafkaConsumerOffset from backup external store error, the error is " + CommonUtils.getStackTraceAsString(e)); 78 | return null; 79 | } 80 | return kafkaConsumerOffset; 81 | } 82 | 83 | public synchronized KafkaConsumerOffset readOffsetFromCache(String topic, Integer partition) { 84 | TopicPartition topicPartition = new TopicPartition(topic, partition); 85 | KafkaConsumerOffset kafkaConsumerOffset = KafkaCache.kafkaConsumerOffsetMaps.get(topicPartition); 86 | if (kafkaConsumerOffset == null) { 87 | kafkaConsumerOffset = readOffsetFromMysql(topic, partition); 88 | if (kafkaConsumerOffset == null) { 89 | logger.error("can not read offset from mysql! the topic is " + topic + ",the partition is " + partition); 90 | kafkaConsumerOffset = externalStorePersist.executeWhenReadNullFromMysql(topic, partition); 91 | } 92 | // 从另一个备用存储读取的接口如果读取成功，默认是空 93 | KafkaConsumerOffset kafkaConsumerOffsetFromBackupExternalStore = readOffsetFromBackupExternalStore(topic, partition); 94 | if (kafkaConsumerOffsetFromBackupExternalStore == null) { 95 | logger.error("can not read offset from backup external store! the topic is " + topic + ",the partition is " + partition); 96 | kafkaConsumerOffsetFromBackupExternalStore = externalStorePersist.executeWhenReadNullFromBackupExternalStore(topic, partition); 97 | } 98 | // 判断两个存储中的数值，然后确定用offset更大的那个 99 | kafkaConsumerOffset = getKafkaConsumerOffsetFromMysqlAndBackupExternalStore(kafkaConsumerOffset, kafkaConsumerOffsetFromBackupExternalStore); 100 | if (kafkaConsumerOffset != null) { 101 | KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.set(true); 102 | KafkaCache.kafkaConsumerOffsetMaps.put(topicPartition, kafkaConsumerOffset); 103 | } else { 104 | KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.set(false); 105 | logger.error("the kafkaConsumerOffset read from external store is null , the topic is " + topic + ",the partition is " + partition); 106 | } 107 | } 108 | return kafkaConsumerOffset; 109 | } 110 | 111 | public KafkaConsumerOffset getKafkaConsumerOffsetFromMysqlAndBackupExternalStore( 112 | KafkaConsumerOffset kafkaConsumerOffset, 113 | KafkaConsumerOffset kafkaConsumerOffsetFromBackupExternalStore) { 114 | if (kafkaConsumerOffsetFromBackupExternalStore == null) { 115 | logger.error("getKafkaConsumerOffsetFromMysqlAndBackupExternalStore, the kafka consumer offset from backup external store is null!"); 116 | System.exit(-1); 117 | return kafkaConsumerOffset; 118 | } 119 | if (kafkaConsumerOffsetFromBackupExternalStore.isNull()) { 120 | logger.info("getKafkaConsumerOffsetFromMysqlAndBackupExternalStore, the kafka consumer offset from backup external store is null, the offset is " 121 | + kafkaConsumerOffsetFromBackupExternalStore); 122 | return kafkaConsumerOffset; 123 | } 124 | if (!kafkaConsumerOffsetFromBackupExternalStore 125 | .equals(kafkaConsumerOffset)) { 126 | logger.error("getKafkaConsumerOffsetFromMysqlAndBackupExternalStore error, the kafkaConsumerOffsetFromBackupExternalStore is " 127 | + kafkaConsumerOffsetFromBackupExternalStore 128 | + ", the kafkaConsumerOffset is " 129 | + kafkaConsumerOffset 130 | + ", they should be equal!"); 131 | System.exit(-1); 132 | return kafkaConsumerOffset; 133 | } 134 | if (kafkaConsumerOffsetFromBackupExternalStore.getOffset() > kafkaConsumerOffset.getOffset()) { 135 | kafkaConsumerOffset = kafkaConsumerOffsetFromBackupExternalStore; 136 | } 137 | return kafkaConsumerOffset; 138 | } 139 | 140 | abstract Boolean saveOffsetInExternalStore(KafkaConsumerOffset kafkaConsumerOffset); 141 | 142 | abstract KafkaConsumerOffset readOffsetFromExternalStore(String topic, int partition); 143 | 144 | } 145 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/persist/DBPoolConnection.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer.persist; 2 | 3 | import cn.thinkingdata.kafka.constant.KafkaMysqlOffsetParameter; 4 | import cn.thinkingdata.kafka.util.CommonUtils; 5 | import com.alibaba.druid.pool.DruidDataSource; 6 | import com.alibaba.druid.pool.DruidDataSourceFactory; 7 | import com.alibaba.druid.pool.DruidPooledConnection; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.sql.SQLException; 12 | import java.util.Properties; 13 | 14 | 15 | public class DBPoolConnection { 16 | 17 | private static final Logger logger = LoggerFactory.getLogger(MysqlOffsetPersist.class); 18 | private static DBPoolConnection dbPoolConnection = null; 19 | private static DruidDataSource druidDataSource = null; 20 | 21 | static { 22 | try { 23 | Properties properties = new Properties(); 24 | // properties.load(DBPoolConnection.class.getResourceAsStream("/db_server.properties")); 25 | properties.put("driverClassName", "com.mysql.jdbc.Driver"); 26 | properties.put("url", KafkaMysqlOffsetParameter.jdbcUrl); 27 | properties.put("username", KafkaMysqlOffsetParameter.username); 28 | properties.put("password", KafkaMysqlOffsetParameter.password); 29 | properties.put("filters", "stat"); 30 | properties.put("initialSize", "1"); 31 | properties.put("minIdle", "1"); 32 | properties.put("maxActive", "30"); 33 | properties.put("maxWait", "60000"); 34 | properties.put("timeBetweenEvictionRunsMillis", "60000"); 35 | properties.put("minEvictableIdleTimeMillis", "300000"); 36 | properties.put("validationQuery", "SELECT 1"); 37 | properties.put("testWhileIdle", "true"); 38 | properties.put("testOnBorrow", "false"); 39 | properties.put("testOnReturn", "false"); 40 | properties.put("poolPreparedStatements", "true"); 41 | properties.put("maxPoolPreparedStatementPerConnectionSize", "20"); 42 | druidDataSource = (DruidDataSource) DruidDataSourceFactory.createDataSource(properties); 43 | } catch (Exception e) { 44 | logger.error("get druidDataSource error, the error is " + CommonUtils.getStackTraceAsString(e)); 45 | System.exit(-1); 46 | } 47 | } 48 | 49 | /** 50 | * 数据库连接池单例 51 | * 52 | * @return 53 | */ 54 | public static synchronized DBPoolConnection getInstance() { 55 | if (null == dbPoolConnection) { 56 | dbPoolConnection = new DBPoolConnection(); 57 | } 58 | return dbPoolConnection; 59 | } 60 | 61 | private DBPoolConnection() { 62 | } 63 | 64 | /** 65 | * 返回druid数据库连接 66 | * 67 | * @return 68 | * @throws SQLException 69 | */ 70 | public DruidPooledConnection getConnection() throws SQLException { 71 | return druidDataSource.getConnection(); 72 | } 73 | 74 | public void close() { 75 | druidDataSource.close(); 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/persist/DefaultStorePersist.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer.persist; 2 | 3 | import cn.thinkingdata.kafka.constant.KafkaMysqlOffsetParameter; 4 | import cn.thinkingdata.kafka.consumer.KafkaSubscribeConsumeThread; 5 | import cn.thinkingdata.kafka.consumer.dao.KafkaConsumerOffset; 6 | import cn.thinkingdata.kafka.consumer.exception.TaKafkaCommonException; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | public class DefaultStorePersist implements StorePersist { 11 | 12 | private static final Logger logger = LoggerFactory.getLogger(DefaultStorePersist.class); 13 | 14 | @Override 15 | public KafkaConsumerOffset readOffsetFromBackupExternalStore(String topic, int partition) { 16 | return new KafkaConsumerOffset(); 17 | } 18 | 19 | @Override 20 | public Boolean saveOffsetInBackupExternalStore(KafkaConsumerOffset kafkaConsumerOffset) { 21 | return true; 22 | } 23 | 24 | @Override 25 | public KafkaConsumerOffset executeWhenReadNullFromBackupExternalStore(String topic, Integer partition) { 26 | logger.error("can not read offset from backup external store!"); 27 | throw new TaKafkaCommonException("executeWhenReadNullFromBackupExternalStore, can not read offset from backup external store!"); 28 | } 29 | 30 | @Override 31 | public KafkaConsumerOffset executeWhenReadNullFromMysql(String topic, Integer partition) { 32 | logger.error("can not read offset from mysql!"); 33 | throw new TaKafkaCommonException("executeWhenReadNullFromMysql, can not read offset from mysql!"); 34 | } 35 | 36 | @Override 37 | public Boolean executeWhenSaveOffsetFailInMysqlAndExternalStore( 38 | KafkaConsumerOffset kafkaConsumerOffset) { 39 | logger.error("save offset fail in mysql or external store!"); 40 | throw new TaKafkaCommonException("executeWhenSaveOffsetFailInMysqlAndExternalStore, save offset fail in mysql or external store!"); 41 | } 42 | 43 | @Override 44 | public Boolean backupStoreStateCheck() { 45 | return true; 46 | } 47 | 48 | @Override 49 | public Boolean updateOwner(KafkaConsumerOffset kafkaConsumerOffset) { 50 | return true; 51 | } 52 | 53 | @Override 54 | public void executeWhenSessionTimeout(Integer count) { 55 | logger.info("session will time out! the count is " + count 56 | + ", the session time out is " 57 | + KafkaMysqlOffsetParameter.sessionTimeout); 58 | throw new TaKafkaCommonException("executeWhenSessionTimeout, session will time out! the count is " + count 59 | + ", the session time out is " 60 | + KafkaMysqlOffsetParameter.sessionTimeout); 61 | } 62 | 63 | @Override 64 | public void executeWhenExecuteDataSessionTimeout(KafkaSubscribeConsumeThread kafkaSubscribeConsumeThread) { 65 | logger.info("session time out! the count is, the session time out is " 66 | + KafkaMysqlOffsetParameter.sessionTimeout); 67 | throw new TaKafkaCommonException("executeWhenExecuteDataSessionTimeout, session time out! the count is, the session time out is " 68 | + KafkaMysqlOffsetParameter.sessionTimeout); 69 | } 70 | 71 | @Override 72 | public void executeWhenOffsetReset(KafkaSubscribeConsumeThread consumeThread) { 73 | logger.info("offset reset!"); 74 | throw new TaKafkaCommonException("executeWhenOffsetReset, kafka offset reset!"); 75 | } 76 | 77 | @Override 78 | public void executeWhenException() { 79 | logger.info("exception!"); 80 | throw new TaKafkaCommonException("executeWhenException!"); 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/persist/MysqlOffsetPersist.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer.persist; 2 | 3 | import cn.thinkingdata.kafka.cache.KafkaCache; 4 | import cn.thinkingdata.kafka.constant.KafkaMysqlOffsetParameter; 5 | import cn.thinkingdata.kafka.consumer.dao.KafkaConsumerOffset; 6 | import cn.thinkingdata.kafka.consumer.offset.MysqlOffsetManager; 7 | import cn.thinkingdata.kafka.util.CommonUtils; 8 | import cn.thinkingdata.kafka.util.RetryerUtil; 9 | import com.github.rholder.retry.RetryException; 10 | import com.github.rholder.retry.Retryer; 11 | import com.google.common.base.Predicates; 12 | import org.apache.kafka.common.TopicPartition; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.util.Date; 17 | import java.util.concurrent.ExecutionException; 18 | 19 | public class MysqlOffsetPersist extends Thread implements OffsetPersist { 20 | 21 | private static final Logger logger = LoggerFactory.getLogger(MysqlOffsetPersist.class); 22 | 23 | private static MysqlOffsetPersist instance; 24 | public static volatile Boolean destoryFlag = false; 25 | public static volatile Boolean runFlag = false; 26 | 27 | private final StorePersist externalStorePersist = MysqlOffsetManager.getInstance().getExternalStorePersist(); 28 | 29 | // public static Boolean mysqlOffsetPersistFlag = false; 30 | 31 | public static synchronized MysqlOffsetPersist getInstance() { 32 | if (instance == null) { 33 | instance = new MysqlOffsetPersist(); 34 | } 35 | return instance; 36 | } 37 | 38 | private final Retryer retryerWithResultFails = RetryerUtil.initRetryerByTimesWithIfResult(3, 300, Predicates.equalTo(false)); 39 | 40 | private MysqlOffsetPersist() { 41 | } 42 | 43 | // persist和flush互斥 44 | @Override 45 | public void persist(KafkaConsumerOffset kafkaConsumerOffset) { 46 | Boolean saveOffsetFlag = saveOffset(kafkaConsumerOffset); 47 | if (!saveOffsetFlag) { 48 | logger.error("can not persist in both mysql or backup store"); 49 | externalStorePersist.executeWhenSaveOffsetFailInMysqlAndExternalStore(kafkaConsumerOffset); 50 | } 51 | } 52 | 53 | @Override 54 | public void shutdown() { 55 | // 关闭定时线程 56 | logger.info("------- Shutting mysql offset thread Down ---------------------"); 57 | MysqlOffsetManager.getInstance().shutdown(); 58 | } 59 | 60 | public void mysqlAndBackupStoreStateCheckJob() { 61 | // 如果mysql和BackupStoreState连接不通，看看有没有恢复 62 | int count = -1; 63 | while (!KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.get()) { 64 | if (count == -1) { 65 | logger.info("------- mysql or backup store down, check mysql and backup store status ---------------------"); 66 | count = 0; 67 | } 68 | Boolean mysqlStateCheck = mysqlStateCheckWithRetry(); 69 | Boolean backupStoreStateCheck = backupStoreStateCheckWithRetry(); 70 | if (!mysqlStateCheck || !backupStoreStateCheck) { 71 | count++; 72 | // 如果mysql或者back up store超过sessionTimeout，就要停止 73 | if (count > Integer.parseInt(KafkaMysqlOffsetParameter.sessionTimeout) - 10) { 74 | externalStorePersist.executeWhenSessionTimeout(count); 75 | } 76 | } else { 77 | count = 0; 78 | } 79 | KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.set(mysqlStateCheck || backupStoreStateCheck); 80 | try { 81 | Thread.sleep(1000); 82 | } catch (InterruptedException e) { 83 | logger.error("------- thread can not sleep ---------------------" + e.toString()); 84 | } 85 | } 86 | } 87 | 88 | public Boolean mysqlStateCheckWithRetry() { 89 | Boolean flag = false; 90 | try { 91 | flag = retryerWithResultFails.call(() -> MysqlOffsetManager.getInstance().mysqlStateCheck()); 92 | } catch (ExecutionException | RetryException e) { 93 | logger.error("retry mysqlStateCheck error, the error is " + CommonUtils.getStackTraceAsString(e)); 94 | flag = false; 95 | } 96 | return flag; 97 | } 98 | 99 | public Boolean backupStoreStateCheckWithRetry() { 100 | Boolean flag = false; 101 | try { 102 | flag = retryerWithResultFails.call(() -> externalStorePersist.backupStoreStateCheck()); 103 | } catch (ExecutionException | RetryException e) { 104 | logger.error("retry backupStoreStateCheck error, the error is " + CommonUtils.getStackTraceAsString(e)); 105 | flag = false; 106 | } 107 | return flag; 108 | } 109 | 110 | @Override 111 | public void run() { 112 | while (!destoryFlag) { 113 | if (!KafkaMysqlOffsetParameter.kafkaSubscribeConsumerClosed.get()) { 114 | runFlag = true; 115 | mysqlAndBackupStoreStateCheckJob(); 116 | // 如果通，并且没有进行rebalance，则定时刷数据进mysql 117 | if (KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.get()) { 118 | persisit(); 119 | } 120 | } 121 | runFlag = false; 122 | try { 123 | Thread.sleep(new Long(KafkaMysqlOffsetParameter.flushInterval) * 100); 124 | } catch (InterruptedException e) { 125 | logger.error("------- thread can not sleep ---------------------" + e.toString()); 126 | } 127 | } 128 | runFlag = false; 129 | logger.info("mysql persist stop, runFlag is " + runFlag); 130 | } 131 | 132 | private synchronized void persisit() { 133 | Date now = new Date(); 134 | for (KafkaConsumerOffset kafkaConsumerOffsetInCache : KafkaCache.kafkaConsumerOffsetMaps.values()) { 135 | // 根据同步offset的size，同步offset的时间 136 | Long lag = kafkaConsumerOffsetInCache.getOffset() - kafkaConsumerOffsetInCache.getLast_flush_offset(); 137 | Long updateInterval = now.getTime() - kafkaConsumerOffsetInCache.getUpdate_time().getTime(); 138 | if (lag >= KafkaMysqlOffsetParameter.flushOffsetSize 139 | || updateInterval >= new Long(KafkaMysqlOffsetParameter.flushInterval) * 1000) { 140 | persist(kafkaConsumerOffsetInCache); 141 | } 142 | } 143 | } 144 | 145 | public Boolean saveOffset(final KafkaConsumerOffset kafkaConsumerOffset) { 146 | Boolean flagMysqlStore = false; 147 | Date now = new Date(); 148 | // 更新的Update_time 149 | kafkaConsumerOffset.setUpdate_time(now); 150 | if (kafkaConsumerOffset.getCreate_time() == null) 151 | kafkaConsumerOffset.setCreate_time(now); 152 | // 得到Last_flush_offset防止consumeThread线程修改数据 153 | Long last_flush_offset = kafkaConsumerOffset.getOffset(); 154 | try { 155 | flagMysqlStore = retryerWithResultFails.call(() -> MysqlOffsetManager.getInstance().saveOffsetInCacheToMysql(kafkaConsumerOffset)); 156 | } catch (ExecutionException | RetryException e) { 157 | logger.error("retry to save kafkaConsumerOffset to mysql and backup external store error, the error is " + CommonUtils.getStackTraceAsString(e)); 158 | flagMysqlStore = false; 159 | } 160 | 161 | Boolean flagBackupStore = false; 162 | try { 163 | // 写一个存到备用存储的接口，默认是空 164 | flagBackupStore = retryerWithResultFails.call(() -> MysqlOffsetManager.getInstance().getExternalStorePersist().saveOffsetInBackupExternalStore(kafkaConsumerOffset)); 165 | } catch (ExecutionException | RetryException e) { 166 | logger.error("retry to save kafkaConsumerOffset to mysql and backup external store error, the error is " + CommonUtils.getStackTraceAsString(e)); 167 | flagBackupStore = false; 168 | } 169 | Boolean saveOffsetFlag = flagMysqlStore || flagBackupStore; 170 | if (saveOffsetFlag) { 171 | kafkaConsumerOffset.setLast_flush_offset(last_flush_offset); 172 | kafkaConsumerOffset.setCount(0L); 173 | KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.set(true); 174 | } else { 175 | KafkaMysqlOffsetParameter.mysqlAndBackupStoreConnState.set(false); 176 | } 177 | return saveOffsetFlag; 178 | } 179 | 180 | @Override 181 | public synchronized Boolean flush(KafkaConsumerOffset kafkaConsumerOffset) { 182 | logger.info("------- flush offset in cache to mysql ---------------------"); 183 | Boolean saveOffsetFlag = saveOffset(kafkaConsumerOffset); 184 | if (!saveOffsetFlag) { 185 | logger.error("can not flush in mysql or backup store"); 186 | externalStorePersist.executeWhenSaveOffsetFailInMysqlAndExternalStore(kafkaConsumerOffset); 187 | } 188 | TopicPartition topicPartition = new TopicPartition(kafkaConsumerOffset.getTopic(), kafkaConsumerOffset.getPartition()); 189 | KafkaConsumerOffset kafkaConsumerOffsetInMap = KafkaCache.kafkaConsumerOffsetMaps.get(topicPartition); 190 | if (kafkaConsumerOffsetInMap != null && kafkaConsumerOffsetInMap.equals(kafkaConsumerOffset)) { 191 | KafkaCache.kafkaConsumerOffsetMaps.remove(topicPartition); 192 | } 193 | kafkaConsumerOffset.setOwner(""); 194 | updateOwner(kafkaConsumerOffset); 195 | return saveOffsetFlag; 196 | 197 | } 198 | 199 | public synchronized Boolean updateOwner(final KafkaConsumerOffset kafkaConsumerOffset) { 200 | try { 201 | Boolean flagMysqlStore = false; 202 | flagMysqlStore = retryerWithResultFails.call(() -> MysqlOffsetManager.getInstance().updateOwner(kafkaConsumerOffset)); 203 | Boolean flagBackupStore = false; 204 | flagBackupStore = retryerWithResultFails.call(() -> MysqlOffsetManager.getInstance().getExternalStorePersist().updateOwner(kafkaConsumerOffset)); 205 | Boolean flag = flagMysqlStore || flagBackupStore; 206 | return flag; 207 | } catch (ExecutionException | RetryException e) { 208 | logger.error("retry to updateOwner from mysql error, the error is " + CommonUtils.getStackTraceAsString(e)); 209 | return false; 210 | } 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/persist/OffsetPersist.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer.persist; 2 | 3 | import cn.thinkingdata.kafka.consumer.dao.KafkaConsumerOffset; 4 | 5 | public interface OffsetPersist { 6 | 7 | void persist(KafkaConsumerOffset kafkaConsumerOffsetInCache); 8 | 9 | void shutdown(); 10 | 11 | Boolean flush(KafkaConsumerOffset kafkaConsumerOffsetInCache); 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/persist/RedisStorePersistService.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer.persist; 2 | 3 | public class RedisStorePersistService{ 4 | 5 | } 6 | //import cn.thinkingdata.kafka.constant.KafkaMysqlOffsetParameter; 7 | //import cn.thinkingdata.kafka.consumer.KafkaSubscribeConsumeThread; 8 | //import cn.thinkingdata.kafka.consumer.dao.KafkaConsumerOffset; 9 | //import cn.thinkingdata.kafka.consumer.persist.MysqlOffsetPersist; 10 | //import cn.thinkingdata.kafka.consumer.persist.StorePersist; 11 | //import cn.thinkingdata.kafka.util.CommonUtils; 12 | //import cn.thinkingdata.kafka.util.RetryerUtil; 13 | //import cn.thinkingdata.ta.etl.domain.Do_ta_company_info; 14 | //import cn.thinkingdata.ta.etl.mapper.MapperTaMysql; 15 | //import cn.thinkingdata.ta.etl.service.TaCenteredAlertSendService; 16 | //import com.github.rholder.retry.RetryException; 17 | //import com.github.rholder.retry.Retryer; 18 | //import com.google.common.base.Predicates; 19 | //import com.google.common.base.Throwables; 20 | //import org.apache.commons.collections.MapUtils; 21 | //import org.apache.commons.lang.StringUtils; 22 | //import org.joda.time.DateTime; 23 | //import org.slf4j.Logger; 24 | //import org.slf4j.LoggerFactory; 25 | //import org.springframework.beans.factory.annotation.Autowired; 26 | //import org.springframework.beans.factory.annotation.Qualifier; 27 | //import org.springframework.data.redis.connection.RedisConnection; 28 | //import org.springframework.data.redis.core.RedisCallback; 29 | //import org.springframework.data.redis.core.RedisTemplate; 30 | //import org.springframework.data.redis.serializer.RedisSerializer; 31 | //import org.springframework.stereotype.Service; 32 | // 33 | //import java.util.Date; 34 | //import java.util.HashMap; 35 | //import java.util.Map; 36 | //import java.util.concurrent.Callable; 37 | //import java.util.concurrent.ExecutionException; 38 | // 39 | //@Service 40 | //public class RedisStorePersistService implements StorePersist { 41 | // 42 | // private static final Logger logger = LoggerFactory 43 | // .getLogger(RedisStorePersistService.class); 44 | // 45 | // public static volatile Boolean restartKafkaConsumerFlag = false; 46 | // public static volatile Boolean stopEtlWithExceptionFlag = false; 47 | // 48 | // @Autowired 49 | // @Qualifier("redisKafkaTemplate") 50 | // private RedisTemplate redisKafkaTemplate; 51 | // 52 | // @Autowired 53 | // private TaCenteredAlertSendService taCenteredAlertSendService; 54 | // 55 | // @Autowired 56 | // private MapperTaMysql mapperTaMysql; 57 | // 58 | // private DateTime lastMailForOffsetReset = null; 59 | // 60 | // 61 | // // private Retryer retryerWithResultNull = RetryerUtil 62 | // // .initRetryerWithIfResult(Predicates.isNull()); 63 | // 64 | // // 4分钟重试，不行就退出 65 | // private Retryer retryerWithResultFails = RetryerUtil 66 | // .initRetryerWithStopTimeIfResult(240L, Predicates.equalTo(false)); 67 | // 68 | // // 无限重试 69 | // private Retryer retryer = cn.thinkingdata.ta.etl.util.RetryerUtil.initRetryer(); 70 | // 71 | // 72 | // public KafkaConsumerOffset readOffset(String topic, Integer partition) { 73 | // return redisKafkaTemplate.execute(new RedisCallback() { 74 | // @Override 75 | // public KafkaConsumerOffset doInRedis(RedisConnection redisConnection) { 76 | // try { 77 | // if (partition == null || StringUtils.isBlank(topic)) { 78 | // logger.error("read kafkaConsumerOffset from redis error, the topic is " 79 | // + topic + ", the partition is " + partition); 80 | // return null; 81 | // } 82 | // KafkaConsumerOffset kafkaConsumerOffset = new KafkaConsumerOffset(); 83 | // RedisSerializer serializer = redisKafkaTemplate 84 | // .getStringSerializer(); 85 | //// KafkaMysqlOffsetParameter.kafkaClusterName = "tga"; 86 | //// KafkaMysqlOffsetParameter.consumerGroup = "taRealtimeEtlGroup"; 87 | // String redisKeystr = KafkaMysqlOffsetParameter.kafkaClusterName 88 | // + "-" 89 | // + topic 90 | // + "-" 91 | // + partition 92 | // + "-" 93 | // + KafkaMysqlOffsetParameter.consumerGroup; 94 | // byte[] redisKey = serializer.serialize(redisKeystr); 95 | // Map kafkaConsumerOffsetMapByte = redisConnection.hGetAll(redisKey); 96 | // if (MapUtils.isNotEmpty(kafkaConsumerOffsetMapByte)) { 97 | // for (byte[] key : kafkaConsumerOffsetMapByte.keySet()) { 98 | // if ("oid".equals(serializer.deserialize(key))) { 99 | // byte[] oidByte = kafkaConsumerOffsetMapByte.get(key); 100 | // if (oidByte != null) { 101 | //// Integer oid = Integer.parseInt(serializer 102 | //// .deserialize(oidByte)); 103 | // kafkaConsumerOffset.setOid(Integer.parseInt(serializer.deserialize(oidByte))); 104 | // } 105 | // } 106 | // if ("partition".equals(serializer.deserialize(key))) { 107 | // byte[] partitionByte = kafkaConsumerOffsetMapByte 108 | // .get(key); 109 | // if (partitionByte != null) { 110 | // kafkaConsumerOffset.setPartition(Integer.parseInt(serializer.deserialize(partitionByte))); 111 | // } 112 | // } 113 | // if ("topic".equals(serializer.deserialize(key))) { 114 | // byte[] topicByte = kafkaConsumerOffsetMapByte 115 | // .get(key); 116 | // if (topicByte != null) { 117 | // kafkaConsumerOffset.setTopic(serializer.deserialize(topicByte)); 118 | // } 119 | // } 120 | // if ("consumer_group".equals(serializer.deserialize(key))) { 121 | // byte[] consumer_groupByte = kafkaConsumerOffsetMapByte 122 | // .get(key); 123 | // if (consumer_groupByte != null) { 124 | // kafkaConsumerOffset.setConsumer_group(serializer.deserialize(consumer_groupByte)); 125 | // } 126 | // } 127 | // if ("offset".equals(serializer.deserialize(key))) { 128 | // byte[] offsetByte = kafkaConsumerOffsetMapByte 129 | // .get(key); 130 | // if (offsetByte != null) { 131 | // kafkaConsumerOffset.setOffset(Long.parseLong(serializer.deserialize(offsetByte))); 132 | // } 133 | // } 134 | // if ("last_flush_offset".equals(serializer.deserialize(key))) { 135 | // byte[] last_flush_offsetByte = kafkaConsumerOffsetMapByte 136 | // .get(key); 137 | // if (last_flush_offsetByte != null) { 138 | // kafkaConsumerOffset.setLast_flush_offset(Long.parseLong(serializer.deserialize(last_flush_offsetByte))); 139 | // } 140 | // } 141 | // if ("kafka_cluster_name".equals(serializer.deserialize(key))) { 142 | // byte[] kafka_cluster_nameByte = kafkaConsumerOffsetMapByte 143 | // .get(key); 144 | // if (kafka_cluster_nameByte != null) { 145 | // kafkaConsumerOffset.setKafka_cluster_name(serializer.deserialize(kafka_cluster_nameByte)); 146 | // } 147 | // } 148 | // if ("owner".equals(serializer.deserialize(key))) { 149 | // byte[] ownerByte = kafkaConsumerOffsetMapByte 150 | // .get(key); 151 | // if (ownerByte != null) { 152 | // kafkaConsumerOffset.setOwner(serializer.deserialize(ownerByte)); 153 | // } 154 | // } 155 | // if ("update_time".equals(serializer.deserialize(key))) { 156 | // byte[] update_timeByte = kafkaConsumerOffsetMapByte 157 | // .get(key); 158 | // if (update_timeByte != null) { 159 | // kafkaConsumerOffset.setUpdate_time(new Date(Long.parseLong(serializer.deserialize(update_timeByte)))); 160 | // } 161 | // } 162 | // if ("create_time".equals(serializer.deserialize(key))) { 163 | // byte[] create_timeByte = kafkaConsumerOffsetMapByte 164 | // .get(key); 165 | // if (create_timeByte != null) { 166 | // kafkaConsumerOffset.setCreate_time(new Date(Long.parseLong(serializer.deserialize(create_timeByte)))); 167 | // } 168 | // } 169 | // } 170 | // } 171 | // if (!kafkaConsumerOffset.isNull()) { 172 | // if (kafkaConsumerOffset.getCount() == null) { 173 | // kafkaConsumerOffset.setCount(0L); 174 | // } 175 | // } 176 | // logger.info("kafkaConsumerOffset in redis is " + kafkaConsumerOffset); 177 | // return kafkaConsumerOffset; 178 | // } catch (Exception e) { 179 | // logger.error("read kafkaConsumerOffset from redis error, the topic is " 180 | // + topic 181 | // + ", the partition is " 182 | // + partition 183 | // + ", the error is " 184 | // + CommonUtils.getStackTraceAsString(e)); 185 | // return null; 186 | // } 187 | // } 188 | // }); 189 | // } 190 | // 191 | // // private Boolean clearOffset(String topic, Integer partition) { 192 | // // return redisTemplate.execute(new RedisCallback() { 193 | // // @Override 194 | // // public Boolean doInRedis(RedisConnection redisConnection) { 195 | // // try { 196 | // // String redisKeystr = KafkaMysqlOffsetParameter.kafkaClusterName 197 | // // + "-" 198 | // // + topic 199 | // // + "-" 200 | // // + partition 201 | // // + "-" 202 | // // + KafkaMysqlOffsetParameter.consumerGroup; 203 | // // RedisSerializer serializer = redisTemplate 204 | // // .getStringSerializer(); 205 | // // byte[] redisKey = serializer.serialize(redisKeystr); 206 | // // Long del = redisConnection.del(redisKey); 207 | // // if (del != 1L) { 208 | // // logger.error("clear kafkaConsumerOffset from redis error, the topic is " 209 | // // + topic 210 | // // + ", the partition is " 211 | // // + partition 212 | // // + ", the del number is " + del.toString()); 213 | // // } 214 | // // return true; 215 | // // } catch (Exception e) { 216 | // // logger.error("clear kafkaConsumerOffset from redis error, the topic is " 217 | // // + topic 218 | // // + ", the partition is " 219 | // // + partition 220 | // // + ", the error is " 221 | // // + CommonUtils.getStackTraceAsString(e)); 222 | // // return false; 223 | // // } 224 | // // } 225 | // // }); 226 | // // } 227 | // 228 | // private Boolean saveOffset(KafkaConsumerOffset kafkaConsumerOffset) { 229 | // return redisKafkaTemplate.execute(new RedisCallback() { 230 | // @Override 231 | // public Boolean doInRedis(RedisConnection redisConnection) { 232 | // try { 233 | // if (kafkaConsumerOffset == null 234 | // || kafkaConsumerOffset.isNull()) { 235 | // logger.error("save offset fail in redis! kafkaConsumerOffset is " 236 | // + kafkaConsumerOffset); 237 | // return true; 238 | // } 239 | // RedisSerializer serializer = redisKafkaTemplate 240 | // .getStringSerializer(); 241 | // String redisKeystr = kafkaConsumerOffset 242 | // .getKafka_cluster_name() 243 | // + "-" 244 | // + kafkaConsumerOffset.getTopic() 245 | // + "-" 246 | // + kafkaConsumerOffset.getPartition() 247 | // + "-" 248 | // + kafkaConsumerOffset.getConsumer_group(); 249 | // byte[] redisKey = serializer.serialize(redisKeystr); 250 | // Map redisValue = new HashMap(); 251 | // if (kafkaConsumerOffset.getOid() != null) { 252 | // redisValue.put(serializer.serialize("oid"), serializer 253 | // .serialize(kafkaConsumerOffset.getOid() 254 | // .toString())); 255 | // } 256 | // redisValue.put(serializer.serialize("topic"), serializer 257 | // .serialize(kafkaConsumerOffset.getTopic())); 258 | // redisValue.put(serializer.serialize("partition"), 259 | // serializer.serialize(kafkaConsumerOffset 260 | // .getPartition().toString())); 261 | // redisValue.put(serializer.serialize("consumer_group"), 262 | // serializer.serialize(kafkaConsumerOffset 263 | // .getConsumer_group())); 264 | // redisValue.put(serializer.serialize("offset"), serializer 265 | // .serialize(kafkaConsumerOffset.getOffset() 266 | // .toString())); 267 | // redisValue.put(serializer.serialize("last_flush_offset"), 268 | // serializer.serialize(kafkaConsumerOffset 269 | // .getLast_flush_offset().toString())); 270 | // redisValue.put(serializer.serialize("kafka_cluster_name"), 271 | // serializer.serialize(kafkaConsumerOffset 272 | // .getKafka_cluster_name())); 273 | // redisValue.put(serializer.serialize("owner"), serializer 274 | // .serialize(kafkaConsumerOffset.getOwner())); 275 | // redisValue.put(serializer.serialize("update_time"), 276 | // serializer.serialize(Long 277 | // .toString(kafkaConsumerOffset 278 | // .getUpdate_time().getTime()))); 279 | // redisValue.put(serializer.serialize("create_time"), 280 | // serializer.serialize(Long 281 | // .toString(kafkaConsumerOffset 282 | // .getCreate_time().getTime()))); 283 | // redisConnection.hMSet(redisKey, redisValue); 284 | // return true; 285 | // } catch (Exception e) { 286 | // logger.error("write kafkaConsumerOffset" 287 | // + kafkaConsumerOffset.toString() 288 | // + " to redis error, the error is " 289 | // + CommonUtils.getStackTraceAsString(e)); 290 | // return false; 291 | // } 292 | // 293 | // } 294 | // }); 295 | // } 296 | // 297 | // // 去掉synchronized，因为MysqlOffsetPersist的flush和persist里有synchronized方法 298 | // @Override 299 | // public Boolean saveOffsetInBackupExternalStore( 300 | // KafkaConsumerOffset kafkaConsumerOffset) { 301 | // Long lag = kafkaConsumerOffset.getOffset() 302 | // - kafkaConsumerOffset.getLast_flush_offset(); 303 | // if (!lag.equals(0L)) { 304 | // logger.debug("because of the muti-thread, the value is not exactly right, the lag is " 305 | // + lag); 306 | // return saveOffset(kafkaConsumerOffset); 307 | // } 308 | // return true; 309 | // } 310 | // 311 | // @Override 312 | // public KafkaConsumerOffset readOffsetFromBackupExternalStore(String topic, 313 | // int partition) { 314 | // KafkaConsumerOffset kafkaConsumerOffsetFromBackupExternalStore = readOffset( 315 | // topic, partition); 316 | // return kafkaConsumerOffsetFromBackupExternalStore; 317 | // } 318 | // 319 | // // @Override 320 | // // public synchronized Boolean clearOffsetFromBackupExternalStore( 321 | // // String topic, int partition) { 322 | // // return clearOffset(topic, partition); 323 | // // } 324 | // 325 | // @Override 326 | // public KafkaConsumerOffset executeWhenReadNullFromBackupExternalStore( 327 | // String topic, Integer partition) { 328 | // // KafkaConsumerOffset kafkaConsumerOffsetFromBackupExternalStore = 329 | // // null; 330 | // // try { 331 | // // kafkaConsumerOffsetFromBackupExternalStore = (KafkaConsumerOffset) 332 | // // retryerWithResultNull 333 | // // .call(new Callable() { 334 | // // @Override 335 | // // public Object call() throws Exception { 336 | // // return readOffset(topic, partition); 337 | // // } 338 | // // }); 339 | // // } catch (ExecutionException | RetryException e) { 340 | // // logger.error("retry to read kafkaConsumerOffset from redis error, the error is " 341 | // // + CommonUtils.getStackTraceAsString(e)); 342 | // // } 343 | // // return kafkaConsumerOffsetFromBackupExternalStore; 344 | // logger.error("can not read offset from redis! exit the process!"); 345 | // stopEtlWithExceptionFlag = true; 346 | // try { 347 | // Thread.sleep(100000); 348 | // } catch (InterruptedException e) { 349 | // logger.error("------- thread can not sleep ---------------------" 350 | // + e.toString()); 351 | // } 352 | // return null; 353 | // } 354 | // 355 | // @Override 356 | // public KafkaConsumerOffset executeWhenReadNullFromMysql(String topic, 357 | // Integer partition) { 358 | // // KafkaConsumerOffset kafkaConsumerOffset = null; 359 | // // try { 360 | // // kafkaConsumerOffset = (KafkaConsumerOffset) retryerWithResultNull 361 | // // .call(new Callable() { 362 | // // @Override 363 | // // public Object call() throws Exception { 364 | // // return MysqlOffsetManager.getInstance() 365 | // // .readOffsetFromMysql(topic, partition); 366 | // // } 367 | // // }); 368 | // // if (kafkaConsumerOffset == null) { 369 | // // logger.error("the kafkaConsumerOffset read from mysql is null , the topic is " 370 | // // + topic 371 | // // + "the partition is " 372 | // // + partition 373 | // // + "exit the process unclear!"); 374 | // // System.exit(-1); 375 | // // } 376 | // // } catch (ExecutionException | RetryException e) { 377 | // // logger.error("the kafkaConsumerOffset read from mysql is null , the topic is " 378 | // // + topic 379 | // // + "the partition is " 380 | // // + partition 381 | // // + "exit the process unclear! the error is " 382 | // // + CommonUtils.getStackTraceAsString(e)); 383 | // // System.exit(-1); 384 | // // return null; 385 | // // } 386 | // // return kafkaConsumerOffset; 387 | // logger.error("can not read offset from mysql! exit the process!"); 388 | // stopEtlWithExceptionFlag = true; 389 | // try { 390 | // Thread.sleep(100000); 391 | // } catch (InterruptedException e) { 392 | // logger.error("------- thread can not sleep ---------------------" 393 | // + e.toString()); 394 | // } 395 | // return null; 396 | // } 397 | // 398 | // // @Override 399 | // // public Boolean executeWhenNotClearExternalStore(final String topic, 400 | // // final Integer partition) { 401 | // // Boolean flag = false; 402 | // // try { 403 | // // flag = (Boolean) retryerWithResultFails.call(new Callable() { 404 | // // @Override 405 | // // public Object call() throws Exception { 406 | // // return clearOffsetFromBackupExternalStore(topic, partition); 407 | // // } 408 | // // }); 409 | // // if (!flag) { 410 | // // logger.error("retry to clear kafkaConsumerOffset from backup external store error!"); 411 | // // } 412 | // // } catch (ExecutionException | RetryException e) { 413 | // // logger.error("retry to clear kafkaConsumerOffset from backup external store error, the error is " 414 | // // + CommonUtils.getStackTraceAsString(e)); 415 | // // return false; 416 | // // } 417 | // // return flag; 418 | // // } 419 | // 420 | // @Override 421 | // public Boolean executeWhenSaveOffsetFailInMysqlAndExternalStore( 422 | // final KafkaConsumerOffset kafkaConsumerOffset) { 423 | // logger.error("save offset fail in mysql and redis! retry!"); 424 | // Boolean flag = false; 425 | // try { 426 | // flag = (Boolean) retryerWithResultFails.call(new Callable() { 427 | // @Override 428 | // public Object call() throws Exception { 429 | // return MysqlOffsetPersist.getInstance().saveOffset( 430 | // kafkaConsumerOffset); 431 | // } 432 | // }); 433 | // if (!flag) { 434 | // logger.error("save offset fail in mysql and redis! exit the process unclear!"); 435 | // stopEtlWithExceptionFlag = true; 436 | // try { 437 | // Thread.sleep(100000); 438 | // } catch (InterruptedException e) { 439 | // logger.error("------- thread can not sleep ---------------------" 440 | // + e.toString()); 441 | // } 442 | // return null; 443 | // } 444 | // } catch (ExecutionException | RetryException e) { 445 | // logger.error("save offset fail in mysql and redis! exit the process unclear! the error is " 446 | // + CommonUtils.getStackTraceAsString(e)); 447 | // stopEtlWithExceptionFlag = true; 448 | // try { 449 | // Thread.sleep(100000); 450 | // } catch (InterruptedException e2) { 451 | // logger.error("------- thread can not sleep ---------------------" 452 | // + e2.toString()); 453 | // } 454 | // return null; 455 | // } 456 | // return flag; 457 | // } 458 | // 459 | // @Override 460 | // public Boolean backupStoreStateCheck() { 461 | // try { 462 | // return redisKafkaTemplate.execute(new RedisCallback() { 463 | // @Override 464 | // public Boolean doInRedis(RedisConnection redisConnection) { 465 | // try { 466 | // Boolean closed = redisConnection.isClosed(); 467 | // if (!closed) { 468 | // logger.info("redis reconnected!"); 469 | // } else { 470 | // logger.info("redis session closed!"); 471 | // } 472 | // return !closed; 473 | // } catch (Exception e) { 474 | // logger.info("redis connect error, the error is " + e); 475 | // return false; 476 | // } 477 | // } 478 | // }); 479 | // 480 | // } catch (Exception e) { 481 | // logger.error("redis connect error, the error is " + e); 482 | // return false; 483 | // } 484 | // 485 | // } 486 | // 487 | // @Override 488 | // public Boolean updateOwner(KafkaConsumerOffset kafkaConsumerOffset) { 489 | // Date now = new Date(); 490 | // if (kafkaConsumerOffset.getOffset() == 0L) { 491 | // kafkaConsumerOffset.setUpdate_time(now); 492 | // if (kafkaConsumerOffset.getCreate_time() == null) { 493 | // kafkaConsumerOffset.setCreate_time(now); 494 | // } 495 | // } 496 | // return updateOwnerInRedis(kafkaConsumerOffset); 497 | // } 498 | // 499 | // private Boolean updateOwnerInRedis(KafkaConsumerOffset kafkaConsumerOffset) { 500 | // try { 501 | // return redisKafkaTemplate.execute(new RedisCallback() { 502 | // @Override 503 | // public Boolean doInRedis(RedisConnection redisConnection) { 504 | // 505 | // if (kafkaConsumerOffset == null 506 | // || kafkaConsumerOffset.isNull()) { 507 | // logger.error("save offset fail in redis! kafkaConsumerOffset is " 508 | // + kafkaConsumerOffset); 509 | // return true; 510 | // } 511 | // RedisSerializer serializer = redisKafkaTemplate 512 | // .getStringSerializer(); 513 | // String redisKeystr = kafkaConsumerOffset 514 | // .getKafka_cluster_name() 515 | // + "-" 516 | // + kafkaConsumerOffset.getTopic() 517 | // + "-" 518 | // + kafkaConsumerOffset.getPartition() 519 | // + "-" 520 | // + kafkaConsumerOffset.getConsumer_group(); 521 | // byte[] redisKey = serializer.serialize(redisKeystr); 522 | // Map redisValue = new HashMap(); 523 | // if (kafkaConsumerOffset.getOid() != null) { 524 | // redisValue.put(serializer.serialize("oid"), serializer 525 | // .serialize(kafkaConsumerOffset.getOid() 526 | // .toString())); 527 | // } 528 | // redisValue.put(serializer.serialize("topic"), serializer 529 | // .serialize(kafkaConsumerOffset.getTopic())); 530 | // redisValue.put(serializer.serialize("partition"), 531 | // serializer.serialize(kafkaConsumerOffset 532 | // .getPartition().toString())); 533 | // redisValue.put(serializer.serialize("consumer_group"), 534 | // serializer.serialize(kafkaConsumerOffset 535 | // .getConsumer_group())); 536 | // redisValue.put(serializer.serialize("offset"), serializer 537 | // .serialize(kafkaConsumerOffset.getOffset() 538 | // .toString())); 539 | // redisValue.put(serializer.serialize("last_flush_offset"), 540 | // serializer.serialize(kafkaConsumerOffset 541 | // .getLast_flush_offset().toString())); 542 | // redisValue.put(serializer.serialize("kafka_cluster_name"), 543 | // serializer.serialize(kafkaConsumerOffset 544 | // .getKafka_cluster_name())); 545 | // redisValue.put(serializer.serialize("owner"), serializer 546 | // .serialize(kafkaConsumerOffset.getOwner())); 547 | // redisValue.put(serializer.serialize("update_time"), 548 | // serializer.serialize(Long 549 | // .toString(kafkaConsumerOffset 550 | // .getUpdate_time().getTime()))); 551 | // redisValue.put(serializer.serialize("create_time"), 552 | // serializer.serialize(Long 553 | // .toString(kafkaConsumerOffset 554 | // .getCreate_time().getTime()))); 555 | // redisConnection.hMSet(redisKey, redisValue); 556 | // return true; 557 | // 558 | // 559 | // } 560 | // }); 561 | // } catch (Exception e) { 562 | // logger.error("updateOwner in redis error, the error is " + CommonUtils.getStackTraceAsString(e)); 563 | // return false; 564 | // } 565 | // } 566 | // 567 | // @Override 568 | // public void executeWhenSessionTimeout(Integer count) { 569 | // logger.error("session will time out! the count is " + count 570 | // + ", the session time out is " 571 | // + KafkaMysqlOffsetParameter.sessionTimeout + ",exit unclear!"); 572 | // stopEtlWithExceptionFlag = true; 573 | // try { 574 | // Thread.sleep(100000); 575 | // } catch (InterruptedException e) { 576 | // logger.error("------- thread can not sleep ---------------------" 577 | // + e.toString()); 578 | // } 579 | // } 580 | // 581 | // @Override 582 | // public void executeWhenExecuteDataSessionTimeout(KafkaSubscribeConsumeThread kafkaSubscribeConsumeThread) { 583 | // logger.error("session time out!" + ", the session time out is " 584 | // + KafkaMysqlOffsetParameter.sessionTimeout); 585 | // restartKafkaConsumerFlag = true; 586 | // } 587 | // 588 | // @Override 589 | // public void executeWhenOffsetReset() { 590 | // logger.error("kafka consumer offset has no such offset!"); 591 | // try { 592 | // if(lastMailForOffsetReset == null || (lastMailForOffsetReset != null && lastMailForOffsetReset.plusMinutes(3).isBeforeNow())){ 593 | // Do_ta_company_info companyInfo = mapperTaMysql.getCompanyInfo(); 594 | // String mailContent = "公司名称：" + companyInfo.getCompany_name() + "
kafka Offset reset"; 595 | // taCenteredAlertSendService.alertByMail("ta_alert@thinkingdata.cn",null,"etl因为kafka的offset的reset而暂停60秒",mailContent); 596 | // lastMailForOffsetReset = new DateTime(); 597 | // } 598 | // Thread.sleep(3000); 599 | // } catch (Exception e) { 600 | // logger.error("send mail to ta_alert@thinkingdata.cn, " + Throwables.getStackTraceAsString(e)); 601 | // } 602 | // } 603 | // 604 | // @Override 605 | // public void executeWhenException() { 606 | // logger.error("kafka consumer offset reset, exit unclear"); 607 | // try { 608 | // Do_ta_company_info companyInfo = mapperTaMysql.getCompanyInfo(); 609 | // String mailContent = "公司名称：" + companyInfo.getCompany_name() + "
kafka读取异常,请人工介入"; 610 | // taCenteredAlertSendService.alertByMail("ta_alert@thinkingdata.cn",null,"etl因为kafka的offset异常退出",mailContent); 611 | // } catch (Exception e) { 612 | // logger.error("send mail to ta_alert@thinkingdata.cn, " + Throwables.getStackTraceAsString(e)); 613 | // } 614 | // // do not let monitor restart 615 | //// try { 616 | //// setKeyVal(TaConstants.ETL_RESET_MARK, "1"); 617 | //// } catch (ExecutionException | RetryException e) { 618 | //// logger.error("set etl reset mark error, " + e.toString()); 619 | //// } 620 | // stopEtlWithExceptionFlag = true; 621 | // try { 622 | // Thread.sleep(80000); 623 | // } catch (InterruptedException e) { 624 | // logger.error("------- thread can not sleep ---------------------" + e.toString()); 625 | // } 626 | // } 627 | // 628 | //// public Object setKeyVal(String key, String value) throws ExecutionException, RetryException { 629 | //// return retryer.call(new Callable() { 630 | //// @Override 631 | //// public Boolean call() throws Exception { 632 | //// return redisKafkaTemplate.execute((RedisCallback) redisConnection -> { 633 | //// RedisSerializer serializer = redisKafkaTemplate.getStringSerializer(); 634 | //// return redisConnection.set(serializer.serialize(key), serializer.serialize(value)); 635 | //// }); 636 | //// } 637 | //// }); 638 | //// } 639 | // 640 | // 641 | //} 642 | // 643 | // 644 | // 645 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/consumer/persist/StorePersist.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.consumer.persist; 2 | 3 | import cn.thinkingdata.kafka.consumer.dao.KafkaConsumerOffset; 4 | import cn.thinkingdata.kafka.consumer.exception.ExceptionHandler; 5 | 6 | public interface StorePersist extends ExceptionHandler { 7 | 8 | // 从另一个备用存储读取的接口如果读取成功，则把它删除，默认是空 9 | KafkaConsumerOffset readOffsetFromBackupExternalStore(String topic, int partition); 10 | 11 | // 写一个存到备用存储的接口，默认是空 12 | Boolean saveOffsetInBackupExternalStore(KafkaConsumerOffset kafkaConsumerOffset); 13 | 14 | Boolean backupStoreStateCheck(); 15 | 16 | Boolean updateOwner(KafkaConsumerOffset kafkaConsumerOffset); 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/test/ProcessDataThread.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.test; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.util.concurrent.atomic.AtomicLong; 7 | 8 | public class ProcessDataThread implements Runnable { 9 | 10 | private static final Logger logger = LoggerFactory 11 | .getLogger(ProcessDataThread.class); 12 | 13 | public static AtomicLong messageCount = new AtomicLong(0L); 14 | 15 | private final String key; 16 | private final String value; 17 | 18 | public ProcessDataThread(String key, String value) { 19 | this.key = key; 20 | this.value = value; 21 | } 22 | 23 | @Override 24 | public void run() { 25 | logger.info(key + "-----" + value); 26 | long count = messageCount.incrementAndGet(); 27 | logger.info(Thread.currentThread().getName() + "-------" + count); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/test/TestMain.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.test; 2 | 3 | 4 | import cn.thinkingdata.kafka.close.ScanTermMethod; 5 | import cn.thinkingdata.kafka.consumer.KafkaSubscribeConsumer; 6 | import cn.thinkingdata.kafka.consumer.NewIDataLineProcessor; 7 | import org.apache.kafka.clients.consumer.ConsumerRecord; 8 | import org.apache.log4j.PropertyConfigurator; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.IOException; 13 | import java.net.URL; 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | import java.util.concurrent.LinkedBlockingQueue; 17 | import java.util.concurrent.ThreadPoolExecutor; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | public class TestMain { 21 | 22 | private static final Logger logger = LoggerFactory 23 | .getLogger(TestMain.class); 24 | 25 | static String jdbcUrl = "jdbc:mysql://ta1:3306/ta?autoReconnect=true&useUnicode=true"; 26 | // static String dataProcessNum = "3"; 27 | static KafkaSubscribeConsumer consumers; 28 | 29 | 30 | public static void main(String[] args) { 31 | 32 | String brokerList = args[0]; 33 | String kafkaClusterName = args[1]; 34 | String topic = args[2]; 35 | String consumerGroup = args[3]; 36 | String processThreadNum = args[4]; 37 | String flushOffsetSize = args[5]; 38 | String flushInterval = args[6]; 39 | final String dataProcessNum = args[7]; 40 | String maxPartitionFetchBytes = args[8]; 41 | 42 | URL url = TestMain.class.getResource("/log4j.properties"); 43 | PropertyConfigurator.configure(url); 44 | 45 | 46 | NewIDataLineProcessor dataProcessor = new NewIDataLineProcessor() { 47 | 48 | ThreadPoolExecutor executorService = new ThreadPoolExecutor(Integer.parseInt(dataProcessNum), Integer.parseInt(dataProcessNum), 49 | 0L, TimeUnit.MILLISECONDS, 50 | new LinkedBlockingQueue(500));{ 51 | executorService.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); 52 | } 53 | 54 | @Override 55 | public void processData(ConsumerRecord consumerRecord) { 56 | executorService.submit(new ProcessDataThread(consumerRecord.key(), consumerRecord.value())); 57 | } 58 | 59 | @Override 60 | public void finishProcess() { 61 | // 关闭线程池 62 | if (executorService != null) 63 | executorService.shutdown(); 64 | try { 65 | if (!executorService.awaitTermination(5000, TimeUnit.MILLISECONDS)) { 66 | logger.warn("Timed out waiting for data process threads to shut down, exiting uncleanly"); 67 | } 68 | } catch (InterruptedException e) { 69 | logger.warn("Interrupted during shutdown, exiting uncleanly"); 70 | } 71 | } 72 | }; 73 | 74 | Map map = new HashMap<>(); 75 | map.put("jdbc.url", jdbcUrl); 76 | map.put("username", "ta"); 77 | map.put("password", "ThinkingData2018"); 78 | map.put("table.name", "kafka_consumer_offset"); 79 | map.put("broker.list", brokerList); 80 | map.put("kafka.cluster.name", kafkaClusterName); 81 | map.put("topic", topic); 82 | map.put("consumer.group", consumerGroup); 83 | map.put("process.thread.num", processThreadNum); 84 | map.put("flush.offset.size", flushOffsetSize); 85 | map.put("flush.interval", flushInterval); 86 | if (maxPartitionFetchBytes != null) { 87 | map.put("max.partition.fetch.bytes", maxPartitionFetchBytes); 88 | } 89 | consumers = new KafkaSubscribeConsumer(map, dataProcessor, new ScanTermMethod()); 90 | 91 | consumers.run(); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/util/CommonUtils.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.util; 2 | 3 | 4 | import java.io.PrintWriter; 5 | import java.io.StringWriter; 6 | import java.net.InetAddress; 7 | import java.net.UnknownHostException; 8 | 9 | public class CommonUtils { 10 | 11 | public static String getHostNameForLiunx() { 12 | try { 13 | return (InetAddress.getLocalHost()).getHostName(); 14 | } catch (UnknownHostException uhe) { 15 | String host = uhe.getMessage(); // host = "hostname: hostname" 16 | if (host != null) { 17 | int colon = host.indexOf(':'); 18 | if (colon > 0) { 19 | return host.substring(0, colon); 20 | } 21 | } 22 | return "UnknownHost"; 23 | } 24 | } 25 | 26 | public static String getHostName() { 27 | if (System.getenv("COMPUTERNAME") != null) { 28 | return System.getenv("COMPUTERNAME"); 29 | } else { 30 | return getHostNameForLiunx(); 31 | } 32 | } 33 | 34 | public static String getStackTraceAsString(Throwable throwable) { 35 | StringWriter stringWriter = new StringWriter(); 36 | throwable.printStackTrace(new PrintWriter(stringWriter)); 37 | return stringWriter.toString(); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/cn/thinkingdata/kafka/util/RetryerUtil.java: -------------------------------------------------------------------------------- 1 | package cn.thinkingdata.kafka.util; 2 | 3 | import com.github.rholder.retry.Attempt; 4 | import com.github.rholder.retry.RetryListener; 5 | import com.github.rholder.retry.Retryer; 6 | import com.github.rholder.retry.RetryerBuilder; 7 | import com.github.rholder.retry.StopStrategies; 8 | import com.github.rholder.retry.WaitStrategies; 9 | import com.google.common.base.Predicate; 10 | import com.google.common.base.Throwables; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.concurrent.TimeUnit; 15 | 16 | /** 17 | * Created by yangruchen on 2019/1/10. 18 | */ 19 | public class RetryerUtil { 20 | private static final Logger logger = LoggerFactory.getLogger(RetryerUtil.class); 21 | 22 | public static Retryer initRetryerByTimesWithIfResult(int retryTimes,long sleepMilliseconds, Predicate predicate){ 23 | Retryer retryer = RetryerBuilder.newBuilder().retryIfException().retryIfResult(predicate) 24 | .withWaitStrategy(WaitStrategies.fixedWait(sleepMilliseconds,TimeUnit.MILLISECONDS)) 25 | .withStopStrategy(StopStrategies.stopAfterAttempt(retryTimes)) 26 | .withRetryListener(new RetryListener() { 27 | @Override 28 | public void onRetry(Attempt attempt) { 29 | if (attempt.hasException()){ 30 | logger.error(Throwables.getStackTraceAsString(attempt.getExceptionCause())); 31 | } 32 | if(attempt.getAttemptNumber() > 1L){ 33 | logger.info("开始进行失败重试，重试次数：" + attempt.getAttemptNumber() + "， 距离第一次失败时间：" + attempt.getDelaySinceFirstAttempt() + "毫秒"); 34 | } 35 | } 36 | }) 37 | .build(); 38 | return retryer; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | ### set log levels ### 2 | log4j.rootLogger=INFO,infoFile 3 | #log4j.Logger=search,Test 4 | 5 | ### console ### 6 | log4j.appender.stdout = org.apache.log4j.ConsoleAppender 7 | log4j.appender.stdout.Threshold = INFO 8 | log4j.appender.stdout.Target = System.out 9 | log4j.appender.stdout.layout = org.apache.log4j.PatternLayout 10 | log4j.appender.stdout.layout.ConversionPattern =[%p] [%-d{yyyy-MM-dd HH:mm:ss}] %C.%M(%L) | %m%n 11 | 12 | log4j.appender.infoFile = org.apache.log4j.DailyRollingFileAppender 13 | log4j.appender.infoFile.Threshold = INFO 14 | log4j.appender.infoFile.File = ./kafka-consumer.log 15 | log4j.appender.infoFile.DatePattern = '.'yyyy-MM-dd'.log' 16 | log4j.appender.infoFile.Append=true 17 | log4j.appender.infoFile.layout = org.apache.log4j.PatternLayout 18 | log4j.appender.infoFile.layout.ConversionPattern =[%d{yyyy-MM-dd HH:mm:ss,SSS}]-[%p]:%m %x %n 19 | --------------------------------------------------------------------------------