├── .gitignore ├── LICENSE ├── README.md ├── pom.xml └── src ├── main ├── assembly │ ├── dev.xml │ └── release.xml ├── bin │ ├── startup.bat │ ├── startup.sh │ └── stop.sh ├── java │ └── com │ │ └── totoro │ │ └── canal │ │ └── es │ │ ├── TotoroBootStrap.java │ │ ├── TotoroLauncher.java │ │ ├── channel │ │ └── TotoroChannel.java │ │ ├── common │ │ ├── GlobalTask.java │ │ ├── RecycleAble.java │ │ ├── RollBackMonitorFactory.java │ │ ├── TotoroException.java │ │ └── TotoroObjectPool.java │ │ ├── consum │ │ └── es │ │ │ ├── Consumer.java │ │ │ ├── ConsumerTask.java │ │ │ ├── ElasticSearchConsumer.java │ │ │ ├── ElasticsearchMetadata.java │ │ │ ├── ElasticsearchService.java │ │ │ ├── EsColumnHashMap.java │ │ │ ├── EsConf.java │ │ │ ├── EsEntryArrayList.java │ │ │ ├── EsRowDataArrayList.java │ │ │ └── impl │ │ │ └── ElasticsearchServiceImpl.java │ │ ├── select │ │ ├── exception │ │ │ └── SelectException.java │ │ └── selector │ │ │ ├── CanalConf.java │ │ │ ├── SelectorTask.java │ │ │ ├── TotoroSelector.java │ │ │ └── canal │ │ │ └── CanalEmbedSelector.java │ │ ├── transform │ │ ├── EsAdapter.java │ │ ├── MessageFilter.java │ │ ├── MessageFilterChain.java │ │ ├── SimpleEsAdapter.java │ │ ├── SimpleMessageFilter.java │ │ ├── TableFilter.java │ │ ├── TotoroTransForm.java │ │ ├── TransForm.java │ │ └── TransFormTask.java │ │ └── util │ │ └── IPAddressUtil.java └── resources │ ├── canal-es.properties │ ├── logback.xml │ └── totoro-logo.logo └── test └── java └── test └── syn ├── Lock.java ├── LockTest.java └── TestFuture.java /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.ear 17 | *.zip 18 | *.tar.gz 19 | *.rar 20 | 21 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 22 | hs_err_pid* 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![logo.jpg](http://upload-images.jianshu.io/upload_images/4798589-0177ebbf0e0e007e.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240) 2 | ========================= 3 | 一个基于阿里巴巴[Canal](https://github.com/alibaba/canal)，实时同步mysql数据到Elasticsearch的工具。轻量，易于配置，部署简单，支持数据回滚。 4 | 使用 Totoro 可以帮助你轻松的将mysql的数据实时同步到Elasticsearch。Totoro是基于阿里巴巴的canal，是数据库级别的 5 | 监听，对原有项目没有任何侵入，所以你无需更改项目中的任何代码就可以实现实时的数据同步。 6 | 7 | 8 | [![AppVeyor](https://img.shields.io/appveyor/ci/gruntjs/grunt.svg)]() 9 | 10 | 11 | [Elasticsearch](https://www.elastic.co/cn/)是一个分布式搜索服务，提供Restful API，底层基于Lucene， 12 | 采用多shard的方式保证数据安全，并且提供自动resharding的功能，github等大型的站点也采用Elasticsearch作为其搜索服务。 13 | 14 | 对于Elasticsearch，如果要在项目中使用，第一个要解决的问题就是，如何将数据库中的数据同步到Elasticsearch，下面常见的几种中方案 15 | 16 | * 修改代码，修改插入数据库的代码，同时插入Elasticsearch 17 | * 修改代码，修改插入数据库的代码，同时放入消息队列，在消息队列的另一端进行插入Elasticsearch 18 | * 使用 Elasticsearch技术栈的[Logstash](https://www.elastic.co/cn/products/logstash)，编写sql语句定时执行搜集数据，并插入到Elasticsearch 19 | 20 | 在上诉方案中，一三都有明显缺陷，第二种方案是目前采用最多的，但还是会对原有项目代码有侵入，并且要引入消息队列。 21 | 如果你不想引入消息队列并且想获得一个开箱即用的同步中间件，那么Totoro会令你非常满意 22 | 。 23 | 24 | Totoro的方案是，基于阿里巴巴开源数据库中间件canal，监听mysql数据库，并且在mysql数据库数据发生变化的时候 25 | 发送消息给Totoro，Totoro会将数据同步到Elasticsearch，如果消费出错支持回滚，保证数据被正确消费。 26 | 27 | ![](http://upload-images.jianshu.io/upload_images/4798589-78d5777d4b1128e6.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240) 28 | 29 | 30 | 工作原理 31 | ====================== 32 | 33 | totoro主要分为4个模块: select、Transformation、consumer、channel 34 | * select 负责从canal中拉取数据 35 | * Transformation 负责将select生产的数据，进行过滤、处理、转换 36 | * consumer 负责消费数据，在这里就是将数据同步到elasticsearch 37 | * channel 是以上3个模块链接者，也是数据在totoro中的容器。select将拉取的数据放入channel，Transformation监听到select放入的数据，进行处理，处理完再放回channel，等待consumer去消费 38 | ![](http://upload-images.jianshu.io/upload_images/4798589-b373bb768f382564.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240) 39 | select、Transformation、consumer分别是3个任务，并行执行。为保证数据消费顺序与ack（每次消费一条数据要向canal进行ack）顺序， 40 | 其中select task与consumer task 都分别只有一条线程。而Transformation task具有多条线程。 41 | 42 | 43 | QuickStart 44 | ====================== 45 | 1.安装Canal 46 | 47 | 有关于canal的安装，请参见canal的文档，里面有很详细说明 [Canal QuickStart](https://github.com/alibaba/canal/wiki/QuickStart) 48 | 49 | 50 | 2.安装Totoro 51 | ``` 52 | git clone git@github.com:zhongchengxcr/canal-elasticsearch.git 53 | cd canal-elasticsearch 54 | mvn clean compile package -P dev -Dmaven.test.skip=true 55 | 56 | ``` 57 | 编译完成后，进入/target/totoro目录，可以看到如下结构： 58 | ```$xslt 59 | -rw-r--r-- 1 zhongcheng staff 2214 12 21 17:08 README.md 60 | drwxr-xr-x 5 zhongcheng staff 160 12 21 17:22 bin 61 | drwxr-xr-x 5 zhongcheng staff 160 12 21 17:08 conf 62 | drwxr-xr-x 73 zhongcheng staff 2336 12 21 17:08 lib 63 | drwxr-xr-x 4 zhongcheng staff 128 12 21 17:08 logs 64 | ``` 65 | 66 | 3.配置修改 67 | ```$xslt 68 | vi conf/canal-es.properties 69 | 70 | ###一下配置需要根据自己的业务进行修改 71 | 72 | # ----------------------- canal 相关配置 ----------------------------- 73 | #canal的实例名字 74 | totoro.canal.destination=totoro 75 | #cananl 服务端的模式，单机：single ，集群：cluster/ 76 | totoro.canal.mode=sign 77 | #canal 地址，包括端口号 78 | totoro.canal.address=127.0.0.1:11111 79 | #过滤表达式 80 | totoro.canal.filter.patten= 81 | #如果canal模式是集群的话，则需要填写zk地址 82 | totoro.canal.zk.address= 83 | totoro.canal.username= 84 | totoro.canal.password= 85 | 86 | #此项必须配置，不配置会导致 totoro 启动不了， totoro 只会处理在此配置的表 87 | #没有在此配置的表，totoro 将会忽略，不会进行同步，格式 "database.table.id" 多个使用 "，"分割 88 | #database代表数据库，table 代表 数据库中的表，id代表 table中的 id 89 | #totoro 会默认将 database 作为 es中的index，table作为es中的type ，使用db中的id作为es的id 90 | totoro.canal.table.accept=demo.cc.id 91 | 92 | 93 | # ----------------------- elasticsearch 相关配置 ----------------------------- 94 | totoro.es.address=127.0.0.1:9300 95 | totoro.es.cluster.name=my-elasticsearch 96 | totoro.es.username= 97 | totoro.es.password= 98 | # ----------------------- totoro 相关配置 ---------------------------- 99 | #处理信息转换的线程数量 默认 3个 ， 不要配置太大，2-4 之间吧，取决于业务情况，太大并不会增加性能，反而会增加上下文切换的开销 100 | totoro.cannal.trans.thread.nums=3 101 | 102 | ``` 103 | 4.准备启动 104 | 105 | ```$xslt 106 | cd ../bin 107 | ./startup.sh 108 | 109 | 110 | _____ _ 111 | |_ _|___ | |_ ___ _ __ ___ 112 | | | / _ \ | __|/ _ \ | '__|/ _ \ 113 | | || (_) || |_| (_) || | | (_) | 114 | |_| \___/ \__|\___/ |_| \___/ 115 | [Totoro 1.0-SNAPSHOT，Build 2017/12/20，Author:zhongcheng_m@yeah.net] 116 | 117 | cd to /Users/zhongcheng/IdeaProjects/canal-elasticsearch/target/totoro/bin for workaround relative path 118 | LOG CONFIGURATION : /Users/zhongcheng/IdeaProjects/canal-elasticsearch/target/totoro/bin/../conf/logback.xml 119 | sync conf : /Users/zhongcheng/IdeaProjects/canal-elasticsearch/target/totoro/bin/../conf/canal-es.properties 120 | cd to /Users/zhongcheng/IdeaProjects/canal-elasticsearch/target/totoro/bin for continue 121 | 122 | ``` 123 | 124 | 5.查看日志 125 | 126 | ```$xslt 127 | cd ../logs 128 | tail -100f totoro.log 129 | 130 | [2017-12-21 17:58:27.699] [INFO] [main] [c.t.c.e.s.s.canal.CanalEmbedSelector] --- TotoroSelector init start ， conf :CanalConf{mode=SIGN， destination='totoro'， filterPatten=''， address='127.0.0.1:11111'， zkAddress=''， userName=''， accept='demo.cc.id'} 131 | [2017-12-21 17:58:27.719] [INFO] [main] [c.t.c.e.s.s.canal.CanalEmbedSelector] --- TotoroSelector init complete ....... 132 | [2017-12-21 17:58:27.737] [INFO] [main] [c.t.c.e.select.selector.SelectorTask] --- Selector task init ....... 133 | [2017-12-21 17:58:27.737] [INFO] [main] [c.t.c.e.select.selector.SelectorTask] --- Selector task complete ....... 134 | [2017-12-21 17:58:27.748] [INFO] [main] [c.t.c.e.transform.MessageFilterChain] --- TableFilter has benn registered to message filter chain 135 | [2017-12-21 17:58:27.748] [INFO] [main] [c.t.c.e.transform.MessageFilterChain] --- SimpleMessageFilter has benn registered to message filter chain 136 | [2017-12-21 17:58:27.749] [INFO] [main] [c.t.c.es.transform.SimpleEsAdapter] --- Add accept :demo.cc.id 137 | [2017-12-21 17:58:27.750] [INFO] [main] [c.t.canal.es.transform.TransFormTask] --- TransFormTask init start ....... 138 | [2017-12-21 17:58:27.758] [INFO] [main] [c.t.canal.es.transform.TransFormTask] --- TransFormTask init complete ....... 139 | [2017-12-21 17:58:30.571] [INFO] [main] [c.t.c.e.c.e.i.ElasticsearchServiceImpl] --- Complete the connection to elasticsearch 140 | [2017-12-21 17:58:30.572] [INFO] [main] [c.t.canal.es.consum.es.ConsumerTask] --- Consumer task init start ....... 141 | [2017-12-21 17:58:30.573] [INFO] [main] [c.t.canal.es.consum.es.ConsumerTask] --- Consumer task init complete....... 142 | [2017-12-21 17:58:30.573] [INFO] [main] [com.totoro.canal.es.TotoroLauncher] --- Totoro init complete ....... 143 | [2017-12-21 17:58:30.574] [INFO] [taskName = TransFormTask] [c.t.canal.es.transform.TransFormTask] --- TransFormTask start ....... 144 | [2017-12-21 17:58:30.574] [INFO] [taskName = ConsumerTask] [c.t.canal.es.consum.es.ConsumerTask] --- ConsumerTask start ....... 145 | [2017-12-21 17:58:30.656] [INFO] [taskName = SelectorTask] [c.t.c.e.select.selector.SelectorTask] --- Selector task start ....... 146 | 147 | ``` 148 | 149 | 6.关闭 150 | ```$xslt 151 | cd ../bin 152 | ./stop.sh 153 | ``` 154 | 155 | 存在的问题 156 | ====================== 157 | * 内存抖动与GC时间过高(已解决,使用对象池化) 158 | 159 | * Elasticsearch 客户端 transport 的 cpu与内存消耗比较多，经测试 发现 transport 启动了 20+ 线程 160 | 161 | * 数据转换的灵活性不够 162 | 163 | * 单一节点，不支持多节点部署 164 | 165 | 166 | 计划 167 | ====================== 168 | * 0.1.0 169 | #####将selector部分抽出单独的模块,分别支持canal,各种mq,以及自定义的selector 170 | 171 | 欢迎有想法的朋友一起参与，讨论 172 | QQ群:688734361 173 | ![](http://upload-images.jianshu.io/upload_images/4798589-a34789352b17055f.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240) 174 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.totoro.canal 8 | canal-elasticsearch 9 | 0.0.1 10 | 11 | 12 | 13 | org.apache.maven.plugins 14 | maven-compiler-plugin 15 | 16 | 1.8 17 | 1.8 18 | 19 | 20 | 21 | 22 | 23 | org.apache.maven.plugins 24 | maven-assembly-plugin 25 | 26 | 2.2.1 27 | 28 | 29 | assemble 30 | 31 | single 32 | 33 | package 34 | 35 | 36 | 37 | false 38 | false 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | dev 50 | 51 | true 52 | 53 | env 54 | !release 55 | 56 | 57 | 58 | 59 | 60 | 61 | maven-assembly-plugin 62 | 63 | 64 | 65 | ${basedir}/src/main/assembly/dev.xml 66 | 67 | totoro 68 | ${project.build.directory} 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | release 78 | 79 | 80 | env 81 | release 82 | 83 | 84 | 85 | 86 | 87 | 88 | maven-assembly-plugin 89 | 90 | 91 | 92 | ${basedir}/src/main/assembly/release.xml 93 | 94 | 95 | ${project.artifactId}-${project.version} 96 | 97 | ${project.parent.build.directory} 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 5.1.1 107 | UTF-8 108 | UTF-8 109 | 1.8 110 | 1.0.24 111 | 2.7 112 | 113 | 114 | 115 | 116 | 117 | com.alibaba.otter 118 | canal.client 119 | ${com.alibaba.otter.canal.version} 120 | 121 | 122 | 123 | org.elasticsearch.client 124 | transport 125 | ${org.elasticsearch.version} 126 | 127 | 128 | org.elasticsearch 129 | elasticsearch 130 | ${org.elasticsearch.version} 131 | 132 | 133 | 134 | org.apache.logging.log4j 135 | log4j-to-slf4j 136 | ${log4j-to-slf4j} 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | -------------------------------------------------------------------------------- /src/main/assembly/dev.xml: -------------------------------------------------------------------------------- 1 | 4 | dist 5 | 6 | dir 7 | 8 | false 9 | 10 | 11 | . 12 | / 13 | 14 | README* 15 | 16 | 17 | 18 | ./src/main/bin 19 | bin 20 | 21 | **/* 22 | 23 | 0755 24 | 25 | 26 | ./src/main/conf 27 | /conf 28 | 29 | **/* 30 | 31 | 32 | 33 | ./src/main/resources 34 | /conf 35 | 36 | **/*.properties 37 | logback.xml 38 | *.logo 39 | 40 | 41 | 42 | 43 | 44 | target 45 | logs 46 | 47 | **/* 48 | 49 | 50 | 51 | 52 | 53 | lib 54 | 55 | junit:junit 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /src/main/assembly/release.xml: -------------------------------------------------------------------------------- 1 | 3 | dist 4 | 5 | tar.gz 6 | 7 | false 8 | 9 | 10 | . 11 | / 12 | 13 | README* 14 | 15 | 16 | 17 | ./src/main/bin 18 | bin 19 | 20 | **/* 21 | 22 | 0755 23 | 24 | 25 | ./src/main/conf 26 | /conf 27 | 28 | **/* 29 | 30 | 31 | 32 | ./src/main/resources 33 | /conf 34 | 35 | **/*.properties 36 | logback.xml 37 | *.logo 38 | 39 | 40 | 41 | target 42 | logs 43 | 44 | **/* 45 | 46 | 47 | 48 | 49 | 50 | lib 51 | 52 | junit:junit 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /src/main/bin/startup.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | @if not "%ECHO%" == "" echo %ECHO% 3 | @if "%OS%" == "Windows_NT" setlocal 4 | 5 | set ENV_PATH=.\ 6 | if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0% 7 | 8 | set conf_dir=%ENV_PATH%\..\conf 9 | set sync_conf=%conf_dir%\canal-es.properties 10 | set logback_configurationFile=%conf_dir%\logback.xml 11 | 12 | set CLASSPATH=%conf_dir% 13 | set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% 14 | 15 | set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m 16 | set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 17 | set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n 18 | set CANAL_OPTS= -Dloader.path=%conf_dir% -DappName=totoro-canal-elasticsearch -Dlogback.configurationFile="%logback_configurationFile%" -Dsync.conf="%sync_conf%" 19 | 20 | set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS% 21 | 22 | set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" com.totoro.canal.es.TotoroLauncher 23 | echo start cmd : %CMD_STR% 24 | 25 | java %JAVA_OPTS% -classpath "%CLASSPATH%" -jar %ENV_PATH%\..\lib\canal-elasticsearch-*.jar -------------------------------------------------------------------------------- /src/main/bin/startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | current_path=`pwd` 4 | case "`uname`" in 5 | Linux) 6 | bin_abs_path=$(readlink -f $(dirname $0)) 7 | ;; 8 | *) 9 | bin_abs_path=`cd $(dirname $0); pwd` 10 | ;; 11 | esac 12 | base=${bin_abs_path}/.. 13 | sync_conf=$base/conf/canal-es.properties 14 | loader_path=$base/conf 15 | logback_configurationFile=$base/conf/logback.xml 16 | totoro_logo=$base/conf/totoro-logo.logo 17 | export LANG=en_US.UTF-8 18 | export BASE=$base 19 | 20 | cat $totoro_logo 21 | 22 | if [ -f $base/bin/totoro.pid ] ; then 23 | echo "found totoro.pid , Please run stop.sh first ,then startup.sh" 2>&2 24 | exit 1 25 | fi 26 | 27 | if [ ! -d $base/logs/totoro ] ; then 28 | mkdir -p $base/logs/totoro 29 | fi 30 | 31 | ## set java path 32 | if [ -z "$JAVA" ] ; then 33 | JAVA=$(which java) 34 | fi 35 | 36 | if [ -z "$JAVA" ]; then 37 | echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.8) in your PATH." 2>&2 38 | exit 1 39 | fi 40 | 41 | case "$#" 42 | in 43 | 0 ) 44 | ;; 45 | 1 ) 46 | var=$* 47 | if [ -f $var ] ; then 48 | sync_conf=$var 49 | else 50 | echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN." 51 | exit 52 | fi;; 53 | 2 ) 54 | var=$1 55 | if [ -f $var ] ; then 56 | sync_conf=$var 57 | else 58 | if [ "$1" = "debug" ]; then 59 | DEBUG_PORT=$2 60 | DEBUG_SUSPEND="n" 61 | JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND" 62 | fi 63 | fi;; 64 | * ) 65 | echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN." 66 | exit;; 67 | esac 68 | 69 | str=`file -L $JAVA | grep 64-bit` 70 | if [ -n "$str" ]; then 71 | JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError" 72 | else 73 | JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m " 74 | fi 75 | 76 | JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8" 77 | CANAL_OPTS="-Dloader.path=$loader_path -DappName=totoro-canal-elasticsearch -Dlogback.configurationFile=$logback_configurationFile" 78 | MAIN_JAR=$base/lib/canal-elasticsearch-*.jar 79 | 80 | if [ -e $sync_conf -a -e $logback_configurationFile ] 81 | then 82 | 83 | for i in $base/lib/*; 84 | do CLASSPATH=$i:"$CLASSPATH"; 85 | done 86 | CLASSPATH="$base/conf:$CLASSPATH"; 87 | 88 | echo "cd to $bin_abs_path for workaround relative path" 89 | cd $bin_abs_path 90 | 91 | echo LOG CONFIGURATION : $logback_configurationFile 92 | echo sync conf : $sync_conf 93 | ##echo CLASSPATH :$CLASSPATH 94 | $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.totoro.canal.es.TotoroLauncher 1>>$base/logs/totoro.log 2>&1 & 95 | 96 | echo $! > $base/bin/totoro.pid 97 | 98 | echo "cd to $current_path for continue" 99 | cd $current_path 100 | else 101 | echo "totoro conf("$sync_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!" 102 | fi 103 | -------------------------------------------------------------------------------- /src/main/bin/stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cygwin=false; 4 | linux=false; 5 | case "`uname`" in 6 | CYGWIN*) 7 | cygwin=true 8 | ;; 9 | Linux*) 10 | linux=true 11 | ;; 12 | esac 13 | 14 | get_pid() { 15 | STR=$1 16 | PID=$2 17 | if $cygwin; then 18 | JAVA_CMD="$JAVA_HOME\bin\java" 19 | JAVA_CMD=`cygpath --path --unix $JAVA_CMD` 20 | JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'` 21 | else 22 | if $linux; then 23 | if [ ! -z "$PID" ]; then 24 | JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` 25 | else 26 | JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` 27 | fi 28 | else 29 | if [ ! -z "$PID" ]; then 30 | JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` 31 | else 32 | JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'` 33 | fi 34 | fi 35 | fi 36 | echo $JAVA_PID; 37 | } 38 | 39 | base=`dirname $0`/.. 40 | pidfile=$base/bin/totoro.pid 41 | if [ ! -f "$pidfile" ];then 42 | echo "totoro-canal-elasticsearch is not running. exists" 43 | exit 44 | fi 45 | 46 | pid=`cat $pidfile` 47 | if [ "$pid" == "" ] ; then 48 | pid=`get_pid "appName=totoro-canal-elasticsearch"` 49 | fi 50 | 51 | echo -e "`hostname`: stopping totoro-canal-elasticsearch $pid ... " 52 | kill $pid 53 | 54 | LOOPS=0 55 | while (true); 56 | do 57 | gpid=`get_pid "appName=totoro-canal-elasticsearch" "$pid"` 58 | if [ "$gpid" == "" ] ; then 59 | echo "Oook! cost:$LOOPS" 60 | `rm $pidfile` 61 | break; 62 | fi 63 | let LOOPS=LOOPS+1 64 | sleep 1 65 | done -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/TotoroBootStrap.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es; 2 | 3 | import com.totoro.canal.es.channel.TotoroChannel; 4 | import com.totoro.canal.es.common.GlobalTask; 5 | import com.totoro.canal.es.consum.es.*; 6 | import com.totoro.canal.es.consum.es.impl.ElasticsearchServiceImpl; 7 | import com.totoro.canal.es.select.selector.CanalConf; 8 | import com.totoro.canal.es.select.selector.SelectorTask; 9 | import com.totoro.canal.es.select.selector.TotoroSelector; 10 | import com.totoro.canal.es.select.selector.canal.CanalEmbedSelector; 11 | import com.totoro.canal.es.transform.*; 12 | import org.apache.commons.lang.ClassUtils; 13 | import org.apache.commons.lang.StringUtils; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | import java.net.UnknownHostException; 18 | import java.util.Map; 19 | import java.util.Properties; 20 | import java.util.concurrent.ConcurrentHashMap; 21 | import java.util.concurrent.ExecutionException; 22 | 23 | /** 24 | * 说明 .
25 | *

26 | *

27 | * Copyright: Copyright (c) 2017/11/19 下午6:00 28 | *

29 | * Company: xxx 30 | *

31 | * 32 | * @author zhongcheng_m@yeah.net 33 | * @version 1.0.0 34 | */ 35 | public class TotoroBootStrap { 36 | 37 | 38 | private static final Logger logger = LoggerFactory.getLogger(TotoroLauncher.class); 39 | 40 | private volatile boolean running = false; 41 | 42 | private TotoroChannel channel; 43 | 44 | private TotoroSelector totoroSelector; 45 | 46 | private ElasticsearchService elasticsearchService; 47 | 48 | private Map taskMap = new ConcurrentHashMap<>(); 49 | 50 | public TotoroBootStrap(final Properties conf) throws UnknownHostException { 51 | 52 | CanalConf canalConf = getCanalConf(conf); 53 | EsConf esConf = getEsConf(conf); 54 | int transThreadNum = getTransThreadNum(conf); 55 | //totoroSelector需要提前初始化好 56 | totoroSelector = new CanalEmbedSelector(canalConf); 57 | //初始化 Channel 58 | initChannel(); 59 | //初始化 SelectorTask 60 | SelectorTask selectorTask = initSelectorTask(); 61 | //初始化 TransFormTask 62 | TransFormTask transFormTask = initTransFormTask(canalConf, transThreadNum); 63 | //初始化 ConsumerTask 64 | ConsumerTask consumerTask = initConsumerTask(esConf); 65 | 66 | taskMap.put(ClassUtils.getShortClassName(SelectorTask.class), selectorTask); 67 | taskMap.put(ClassUtils.getShortClassName(TransFormTask.class), transFormTask); 68 | taskMap.put(ClassUtils.getShortClassName(ConsumerTask.class), consumerTask); 69 | 70 | 71 | logger.info("Totoro init complete ......."); 72 | } 73 | 74 | 75 | private SelectorTask initSelectorTask() { 76 | return new SelectorTask(totoroSelector, channel, this); 77 | } 78 | 79 | private TransFormTask initTransFormTask(CanalConf canalConf, int transThreadNum) { 80 | MessageFilter tableFilter = new TableFilter(canalConf); 81 | MessageFilter simpleFilter = new SimpleMessageFilter(); 82 | 83 | MessageFilterChain messageFilterChain = MessageFilterChain.getInstance(); 84 | messageFilterChain.register(tableFilter); 85 | messageFilterChain.register(simpleFilter); 86 | 87 | EsAdapter esAdapter = new SimpleEsAdapter(canalConf); 88 | 89 | return new TransFormTask(channel, esAdapter, transThreadNum); 90 | } 91 | 92 | private ConsumerTask initConsumerTask(EsConf esConf) throws UnknownHostException { 93 | 94 | elasticsearchService = new ElasticsearchServiceImpl(esConf); 95 | Consumer consumer = new ElasticSearchConsumer(elasticsearchService); 96 | ConsumerTask consumerTask = new ConsumerTask(channel); 97 | consumerTask.register(consumer); 98 | return consumerTask; 99 | } 100 | 101 | 102 | private void initChannel() { 103 | channel = new TotoroChannel(totoroSelector); 104 | } 105 | 106 | 107 | public void start() throws InterruptedException, ExecutionException { 108 | //主线程所在 109 | running = true; 110 | taskMap.forEach((key, value) -> value.start()); 111 | } 112 | 113 | public void stop() { 114 | taskMap.forEach((key, value) -> value.shutdown()); 115 | taskMap.clear(); 116 | channel.close(); 117 | totoroSelector.stop(); 118 | elasticsearchService.close(); 119 | } 120 | 121 | 122 | private EsConf getEsConf(Properties conf) { 123 | 124 | String address = conf.getProperty("totoro.es.address"); 125 | String clusterName = conf.getProperty("totoro.es.cluster.name"); 126 | String username = conf.getProperty("totoro.es.username"); 127 | String password = conf.getProperty("totoro.es.password"); 128 | 129 | return new EsConf() 130 | .setAddress(address) 131 | .setClusterName(clusterName) 132 | .setUsername(username) 133 | .setPassword(password) 134 | .builder(); 135 | } 136 | 137 | 138 | private CanalConf getCanalConf(Properties conf) { 139 | 140 | String address = conf.getProperty("totoro.canal.address"); 141 | String zkAddress = conf.getProperty("totoro.canal.zk.address"); 142 | String username = conf.getProperty("totoro.canal.username"); 143 | String password = conf.getProperty("totoro.canal.password"); 144 | String mode = conf.getProperty("totoro.canal.mode"); 145 | String destination = conf.getProperty("totoro.canal.destination"); 146 | String filterPatten = conf.getProperty("totoro.canal.filter.patten"); 147 | String accept = conf.getProperty("totoro.canal.table.accept"); 148 | 149 | return new CanalConf() 150 | .setAddress(address) 151 | .setZkAddress(zkAddress) 152 | .setUserName(username) 153 | .setPassWord(password) 154 | .setMode(mode) 155 | .setDestination(destination) 156 | .setFilterPatten(filterPatten) 157 | .setAccept(accept) 158 | .builder(); 159 | } 160 | 161 | 162 | private int getTransThreadNum(Properties conf) { 163 | String numStr = conf.getProperty("totoro.cannal.trans.thread.nums"); 164 | 165 | if (numStr == null || StringUtils.isEmpty(numStr.trim())) { 166 | return 3; 167 | } 168 | return Integer.valueOf(numStr); 169 | } 170 | 171 | 172 | public Map getTaskMap() { 173 | return taskMap; 174 | } 175 | 176 | 177 | } 178 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/TotoroLauncher.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es; 2 | 3 | import com.google.common.io.CharStreams; 4 | import org.apache.commons.lang.StringUtils; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.core.io.ClassPathResource; 8 | import org.springframework.core.io.Resource; 9 | 10 | import java.io.FileInputStream; 11 | import java.io.IOException; 12 | import java.io.InputStreamReader; 13 | import java.io.Reader; 14 | import java.util.Properties; 15 | import java.util.concurrent.ExecutionException; 16 | 17 | /** 18 | *

19 | * Copyright: Copyright (c) 20 | *

21 | * Company: xx 22 | *

23 | * 24 | * @author zhongcheng_m@yeah.net 25 | * @version 1.0.0 26 | */ 27 | public class TotoroLauncher { 28 | 29 | private static final Logger logger = LoggerFactory.getLogger(TotoroLauncher.class); 30 | 31 | private static final String CLASSPATH_URL_PREFIX = "classpath:"; 32 | 33 | 34 | static { 35 | System.out.println(""); 36 | System.out.println(" _____ _"); 37 | System.out.println("|_ _|___ | |_ ___ _ __ ___ "); 38 | System.out.println(" | | / _ \\ | __|/ _ \\ | '__|/ _ \\ "); 39 | System.out.println(" | || (_) || |_| (_) || | | (_) |"); 40 | System.out.println(" |_| \\___/ \\__|\\___/ |_| \\___/"); 41 | System.out.println("[Totoro 1.0-SNAPSHOT,Build 2017/12/20,Author:zhongcheng_m@yeah.net]"); 42 | System.out.println(""); 43 | } 44 | 45 | public static void main(String[] args) throws InterruptedException, IOException, ExecutionException { 46 | 47 | setGlobalUncaughtExceptionHandler(); 48 | 49 | String conf = System.getProperty("canal-es.properties", "classpath:canal-es.properties"); 50 | 51 | Properties properties = getProperties(conf); 52 | 53 | TotoroBootStrap canalScheduler = new TotoroBootStrap(properties); 54 | canalScheduler.start(); 55 | 56 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 57 | try { 58 | logger.info("## stop the totoro server"); 59 | canalScheduler.stop(); 60 | } catch (Throwable e) { 61 | logger.warn("##something goes wrong when stopping totoro Server:", e); 62 | } finally { 63 | logger.info("## totoro server is down."); 64 | } 65 | })); 66 | 67 | } 68 | 69 | private static Properties getProperties(String conf) throws IOException { 70 | Properties properties = new Properties(); 71 | 72 | if (conf.startsWith(CLASSPATH_URL_PREFIX)) { 73 | conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX); 74 | properties.load(TotoroLauncher.class.getClassLoader().getResourceAsStream(conf)); 75 | } else { 76 | properties.load(new FileInputStream(conf)); 77 | } 78 | return properties; 79 | } 80 | 81 | 82 | private static void setGlobalUncaughtExceptionHandler() { 83 | Thread.setDefaultUncaughtExceptionHandler((t, e) -> logger.error("UnCaughtException", e)); 84 | } 85 | 86 | 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/channel/TotoroChannel.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.channel; 2 | 3 | import com.alibaba.otter.canal.common.utils.BooleanMutex; 4 | import com.alibaba.otter.canal.protocol.Message; 5 | import com.totoro.canal.es.common.RollBackMonitorFactory; 6 | import com.totoro.canal.es.consum.es.ElasticsearchMetadata; 7 | import com.totoro.canal.es.select.selector.TotoroSelector; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.concurrent.Future; 12 | import java.util.concurrent.LinkedBlockingQueue; 13 | 14 | /** 15 | *

16 | * Copyright: Copyright (c) 17 | *

18 | * Company: xx 19 | *

20 | * 21 | * @author zhongcheng_m@yeah.net 22 | * @version 1.0.0 23 | */ 24 | public class TotoroChannel { 25 | 26 | private BooleanMutex rollBack = RollBackMonitorFactory.getBooleanMutex(); 27 | 28 | private Logger logger = LoggerFactory.getLogger(TotoroChannel.class); 29 | 30 | private LinkedBlockingQueue selectorMessageQueue = new LinkedBlockingQueue<>(5); 31 | 32 | private LinkedBlockingQueue> transFormFuture = new LinkedBlockingQueue<>(5); 33 | 34 | 35 | private TotoroSelector totoroSelector; 36 | 37 | public TotoroChannel(TotoroSelector totoroSelector) { 38 | this.totoroSelector = totoroSelector; 39 | } 40 | 41 | public void ack(Long batchId) { 42 | totoroSelector.ack(batchId); 43 | } 44 | 45 | /** 46 | * 处于回滚状态下，拒绝所有put的消息 47 | */ 48 | 49 | public void clearMessage() { 50 | selectorMessageQueue.clear(); 51 | transFormFuture.clear(); 52 | } 53 | 54 | 55 | public void putMessage(Message e) throws InterruptedException { 56 | if (rollBack.state() == true) { 57 | selectorMessageQueue.put(e); 58 | } else { 59 | logger.info("The rollback happened =============> discard message , batchId :{}", e.getId()); 60 | } 61 | } 62 | 63 | public Message takeMessage() throws InterruptedException { 64 | return selectorMessageQueue.take(); 65 | } 66 | 67 | public void putFuture(Future future) throws InterruptedException { 68 | if (rollBack.state() == true) { 69 | transFormFuture.put(future); 70 | } else { 71 | future.cancel(true); 72 | logger.info("The rollback happened =============> try cancel future "); 73 | } 74 | } 75 | 76 | public Future take() throws InterruptedException { 77 | return transFormFuture.take(); 78 | } 79 | 80 | 81 | public void close() { 82 | Object[] tuple2Arr = transFormFuture.toArray(); 83 | for (Object obj : tuple2Arr) { 84 | Future future = (Future) obj; 85 | future.cancel(true); 86 | } 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/common/GlobalTask.java: -------------------------------------------------------------------------------- 1 | 2 | package com.totoro.canal.es.common; 3 | 4 | import org.apache.commons.lang.ClassUtils; 5 | import org.apache.commons.lang.exception.ExceptionUtils; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.ArrayList; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.concurrent.ExecutorService; 14 | import java.util.concurrent.Future; 15 | 16 | /** 17 | * mainstem,select,extract,transform,load parent Thread. 18 | * 19 | * @author xiaoqing.zhouxq 2011-8-23 上午10:38:14 20 | */ 21 | public abstract class GlobalTask extends Thread { 22 | 23 | protected final Logger logger = LoggerFactory.getLogger(this.getClass()); 24 | 25 | protected volatile boolean running = false; 26 | 27 | protected ExecutorService executorService; 28 | 29 | protected Map pendingFuture; 30 | 31 | 32 | public GlobalTask() { 33 | setName(createTaskName(ClassUtils.getShortClassName(this.getClass()))); 34 | pendingFuture = new HashMap<>(); 35 | } 36 | 37 | public void shutdown() { 38 | running = false; 39 | interrupt(); 40 | List cancelFutures = new ArrayList<>(); 41 | for (Map.Entry entry : pendingFuture.entrySet()) { 42 | if (!entry.getValue().isDone()) { 43 | logger.warn("WARN ## Task future processId[{}] canceled!", entry.getKey()); 44 | cancelFutures.add(entry.getValue()); 45 | } 46 | } 47 | 48 | for (Future future : cancelFutures) { 49 | future.cancel(true); 50 | } 51 | pendingFuture.clear(); 52 | 53 | if (executorService != null) { 54 | executorService.shutdown(); 55 | } 56 | } 57 | 58 | 59 | protected String createTaskName(String taskName) { 60 | return new StringBuilder().append("taskName = ").append(taskName).toString(); 61 | } 62 | 63 | 64 | protected boolean isInterrupt(Throwable e) { 65 | if (!running) { 66 | return true; 67 | } 68 | 69 | if (ExceptionUtils.getRootCause(e) instanceof InterruptedException) { 70 | return true; 71 | } 72 | 73 | return false; 74 | 75 | } 76 | 77 | // ====================== setter / getter ========================= 78 | 79 | public void setExecutorService(ExecutorService executorService) { 80 | this.executorService = executorService; 81 | } 82 | 83 | 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/common/RecycleAble.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.common; 2 | 3 | /** 4 | * 说明 .
5 | *

6 | *

7 | * Copyright: Copyright (c) 2017/12/22 下午12:30 8 | *

9 | * Company: xxx 10 | *

11 | * 12 | * @author zhongcheng_m@yeah.net 13 | * @version 1.0.0 14 | */ 15 | public interface RecycleAble { 16 | 17 | boolean recycle(); 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/common/RollBackMonitorFactory.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.common; 2 | 3 | import com.alibaba.otter.canal.common.utils.BooleanMutex; 4 | 5 | /** 6 | * 说明 .
7 | *

8 | *

9 | * Copyright: Copyright (c) 2017/12/05 上午10:04 10 | *

11 | * Company: xxx 12 | *

13 | * 14 | * @author zhongcheng_m@yeah.net 15 | * @version 1.0.0 16 | */ 17 | public class RollBackMonitorFactory { 18 | 19 | private static BooleanMutex booleanMutex = new BooleanMutex(false); 20 | 21 | public static BooleanMutex getBooleanMutex() { 22 | return booleanMutex; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/common/TotoroException.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.common; 2 | 3 | /** 4 | * 说明 .
5 | *

6 | *

7 | * Copyright: Copyright (c) 2017/12/01 下午12:53 8 | *

9 | * Company: xxx 10 | *

11 | * 12 | * @author zhongcheng_m@yeah.net 13 | * @version 1.0.0 14 | */ 15 | public class TotoroException extends RuntimeException { 16 | 17 | private String errMsg; 18 | 19 | public TotoroException(String errMsg) { 20 | super(errMsg); 21 | this.errMsg = errMsg; 22 | } 23 | 24 | public String getErrMsg() { 25 | return errMsg; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/common/TotoroObjectPool.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.common; 2 | 3 | import com.totoro.canal.es.consum.es.ElasticsearchMetadata; 4 | import com.totoro.canal.es.consum.es.EsColumnHashMap; 5 | import com.totoro.canal.es.consum.es.EsEntryArrayList; 6 | import com.totoro.canal.es.consum.es.EsRowDataArrayList; 7 | import com.totoro.canal.es.transform.TotoroTransForm; 8 | import com.totoro.canal.es.transform.TransForm; 9 | import io.netty.util.Recycler; 10 | 11 | /** 12 | * 说明 .
13 | *

14 | *

15 | * Copyright: Copyright (c) 2017/12/22 下午12:49 16 | *

17 | * Company: xxx 18 | *

19 | * 20 | * @author zhongcheng_m@yeah.net 21 | * @version 1.0.0 22 | */ 23 | public class TotoroObjectPool { 24 | 25 | public static ElasticsearchMetadata esMetadata() { 26 | return ELASTICSEARCH_METADATA_RECYCLER.get(); 27 | } 28 | 29 | public static ElasticsearchMetadata.EsEntry esEntry() { 30 | return ELASTICSEARCH_ENTRY_RECYCLER.get(); 31 | } 32 | 33 | public static ElasticsearchMetadata.EsRowData esRowData() { 34 | return ELASTICSEARCH_ROWDATA_RECYCLER.get(); 35 | } 36 | 37 | static { 38 | System.setProperty("io.netty.recycler.ratio", "1"); 39 | } 40 | 41 | 42 | public static void main(String[] args) { 43 | 44 | 45 | System.out.printf(System.getProperty("io.netty.recycler.ratio")); 46 | 47 | ElasticsearchMetadata.EsRowData esRowData = TotoroObjectPool.esRowData(); 48 | ElasticsearchMetadata.EsRowData esRowData1 = TotoroObjectPool.esRowData(); 49 | ElasticsearchMetadata.EsRowData esRowData6 = TotoroObjectPool.esRowData(); 50 | ElasticsearchMetadata.EsRowData esRowData5 = TotoroObjectPool.esRowData(); 51 | 52 | 53 | System.out.println(esRowData); 54 | System.out.println(esRowData1); 55 | System.out.println(esRowData6); 56 | System.out.println(esRowData5); 57 | 58 | esRowData.recycle(); 59 | esRowData1.recycle(); 60 | esRowData6.recycle(); 61 | esRowData5.recycle(); 62 | 63 | ElasticsearchMetadata.EsRowData esRowData7 = TotoroObjectPool.esRowData(); 64 | ElasticsearchMetadata.EsRowData esRowData8 = TotoroObjectPool.esRowData(); 65 | ElasticsearchMetadata.EsRowData esRowData9 = TotoroObjectPool.esRowData(); 66 | ElasticsearchMetadata.EsRowData esRowData0 = TotoroObjectPool.esRowData(); 67 | 68 | ElasticsearchMetadata.EsRowData esRowData12 = TotoroObjectPool.esRowData(); 69 | ElasticsearchMetadata.EsRowData esRowData121 = TotoroObjectPool.esRowData(); 70 | 71 | 72 | System.out.println(esRowData7); 73 | System.out.println(esRowData8); 74 | esRowData7.recycle(); 75 | esRowData8.recycle(); 76 | 77 | 78 | System.out.println(esRowData9); 79 | System.out.println(esRowData0); 80 | esRowData9.recycle(); 81 | esRowData0.recycle(); 82 | 83 | 84 | // new Thread(()->{ 85 | // 86 | // ElasticsearchMetadata.EsRowData esRowData3 = TotoroObjectPool.esRowData(); 87 | // ElasticsearchMetadata.EsRowData esRowData4 = TotoroObjectPool.esRowData(); 88 | // System.out.println(esRowData3); 89 | // System.out.println(esRowData4); 90 | // 91 | // esRowData3.recycle(); 92 | // esRowData4.recycle(); 93 | // 94 | // try { 95 | // Thread.sleep(1000L); 96 | // } catch (InterruptedException e) { 97 | // e.printStackTrace(); 98 | // } 99 | // 100 | // }).start(); 101 | // 102 | // new Thread(()->{ 103 | // 104 | // ElasticsearchMetadata.EsRowData esRowData3 = TotoroObjectPool.esRowData(); 105 | // ElasticsearchMetadata.EsRowData esRowData4 = TotoroObjectPool.esRowData(); 106 | // System.out.println(esRowData3); 107 | // System.out.println(esRowData4); 108 | // 109 | // esRowData3.recycle(); 110 | // esRowData4.recycle(); 111 | // 112 | // try { 113 | // Thread.sleep(1000L); 114 | // } catch (InterruptedException e) { 115 | // e.printStackTrace(); 116 | // } 117 | // }).start(); 118 | // new Thread(()->{ 119 | // 120 | // ElasticsearchMetadata.EsRowData esRowData3 = TotoroObjectPool.esRowData(); 121 | // ElasticsearchMetadata.EsRowData esRowData4 = TotoroObjectPool.esRowData(); 122 | // System.out.println(esRowData3); 123 | // System.out.println(esRowData4); 124 | // esRowData3.recycle(); 125 | // esRowData4.recycle(); 126 | // 127 | // try { 128 | // Thread.sleep(1000L); 129 | // } catch (InterruptedException e) { 130 | // e.printStackTrace(); 131 | // } 132 | // 133 | // }).start(); 134 | // new Thread(()->{ 135 | // 136 | // ElasticsearchMetadata.EsRowData esRowData3 = TotoroObjectPool.esRowData(); 137 | // ElasticsearchMetadata.EsRowData esRowData4 = TotoroObjectPool.esRowData(); 138 | // System.out.println(esRowData3); 139 | // System.out.println(esRowData4); 140 | // esRowData3.recycle(); 141 | // esRowData4.recycle(); 142 | // 143 | // try { 144 | // Thread.sleep(1000L); 145 | // } catch (InterruptedException e) { 146 | // e.printStackTrace(); 147 | // } 148 | // 149 | // }).start(); 150 | // new Thread(()->{ 151 | // 152 | // ElasticsearchMetadata.EsRowData esRowData3 = TotoroObjectPool.esRowData(); 153 | // ElasticsearchMetadata.EsRowData esRowData4 = TotoroObjectPool.esRowData(); 154 | // System.out.println(esRowData3); 155 | // System.out.println(esRowData4); 156 | // esRowData3.recycle(); 157 | // esRowData4.recycle(); 158 | // 159 | // try { 160 | // Thread.sleep(1000L); 161 | // } catch (InterruptedException e) { 162 | // e.printStackTrace(); 163 | // } 164 | // 165 | // }).start(); 166 | 167 | } 168 | 169 | public static EsColumnHashMap esColumnHashMap() { 170 | return ELASTICSEARCH_COLUMNMAP_RECYCLER.get(); 171 | } 172 | 173 | public static EsEntryArrayList esEntryArrayList() { 174 | return ELASTICSEARCH_ENTRY_LIST__RECYCLER.get(); 175 | } 176 | 177 | public static EsRowDataArrayList esRowDataArrayList() { 178 | return ELASTICSEARCH_ROW_DATA__RECYCLER.get(); 179 | } 180 | 181 | public static TransForm transForm() { 182 | return TRANS_FORM_RECYCLER.get(); 183 | } 184 | 185 | 186 | private static final Recycler ELASTICSEARCH_METADATA_RECYCLER = 187 | new Recycler() { 188 | @Override 189 | protected ElasticsearchMetadata newObject(Handle handle) { 190 | return new ElasticsearchMetadata(handle); 191 | } 192 | }; 193 | 194 | 195 | private static final Recycler ELASTICSEARCH_ENTRY_RECYCLER = 196 | new Recycler() { 197 | @Override 198 | protected ElasticsearchMetadata.EsEntry newObject(Handle handle) { 199 | return new ElasticsearchMetadata.EsEntry(handle); 200 | } 201 | }; 202 | 203 | private static final Recycler ELASTICSEARCH_ROWDATA_RECYCLER = 204 | new Recycler() { 205 | @Override 206 | protected ElasticsearchMetadata.EsRowData newObject(Handle handle) { 207 | return new ElasticsearchMetadata.EsRowData(handle); 208 | } 209 | }; 210 | 211 | 212 | private static final Recycler ELASTICSEARCH_COLUMNMAP_RECYCLER = 213 | new Recycler() { 214 | @Override 215 | protected EsColumnHashMap newObject(Handle handle) { 216 | return new EsColumnHashMap(handle); 217 | } 218 | }; 219 | 220 | 221 | private static final Recycler TRANS_FORM_RECYCLER = 222 | new Recycler() { 223 | @Override 224 | protected TransForm newObject(Handle handle) { 225 | return new TotoroTransForm(handle); 226 | } 227 | }; 228 | 229 | private static final Recycler ELASTICSEARCH_ENTRY_LIST__RECYCLER = 230 | new Recycler() { 231 | @Override 232 | protected EsEntryArrayList newObject(Handle handle) { 233 | return new EsEntryArrayList(handle); 234 | } 235 | }; 236 | 237 | private static final Recycler ELASTICSEARCH_ROW_DATA__RECYCLER = 238 | new Recycler() { 239 | @Override 240 | protected EsRowDataArrayList newObject(Handle handle) { 241 | return new EsRowDataArrayList(handle); 242 | } 243 | }; 244 | 245 | 246 | } 247 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/consum/es/Consumer.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.consum.es; 2 | 3 | public interface Consumer { 4 | 5 | void consume(ElasticsearchMetadata object); 6 | 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/consum/es/ConsumerTask.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.consum.es; 2 | 3 | import com.alibaba.otter.canal.common.utils.BooleanMutex; 4 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 5 | import com.totoro.canal.es.channel.TotoroChannel; 6 | import com.totoro.canal.es.common.GlobalTask; 7 | import com.totoro.canal.es.common.RollBackMonitorFactory; 8 | 9 | import java.util.Set; 10 | import java.util.concurrent.*; 11 | 12 | /** 13 | * 说明 .
14 | *

15 | *

16 | * Copyright: Copyright (c) 2017/12/01 下午12:49 17 | *

18 | * Company: xxx 19 | *

20 | * 21 | * @author zhongcheng_m@yeah.net 22 | * @version 1.0.0 23 | */ 24 | public class ConsumerTask extends GlobalTask { 25 | 26 | private final Set consumers = new ConcurrentHashMap().keySet(true); 27 | 28 | private TotoroChannel channel; 29 | 30 | private BooleanMutex rollBack = RollBackMonitorFactory.getBooleanMutex(); 31 | 32 | public ConsumerTask(TotoroChannel totoroChannel) { 33 | logger.info("Consumer task init start ......."); 34 | this.channel = totoroChannel; 35 | ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("esload-pool-%d").build(); 36 | executorService = Executors.newSingleThreadExecutor(threadFactory); 37 | 38 | logger.info("Consumer task init complete......."); 39 | } 40 | 41 | @Override 42 | public void run() { 43 | logger.info("ConsumerTask start ......."); 44 | running = true; 45 | while (running) { 46 | try { 47 | rollBack.get(); 48 | } catch (InterruptedException e) { 49 | logger.error("Consumer task has benn interrupted "); 50 | running = false; 51 | break; 52 | } 53 | ElasticsearchMetadata elasticsearchMetadata = null; 54 | try { 55 | Future future = channel.take(); 56 | 57 | 58 | elasticsearchMetadata = future.get(); 59 | logger.info("Consumer message start =====> {}", elasticsearchMetadata.getBatchId()); 60 | 61 | ElasticsearchMetadata finalElasticsearchMetadata = elasticsearchMetadata; 62 | 63 | consumers.forEach(consumer -> consumer.consume(finalElasticsearchMetadata)); 64 | 65 | channel.ack(elasticsearchMetadata.getBatchId()); 66 | 67 | logger.info("Consumer message ack =====> {}", elasticsearchMetadata.getBatchId()); 68 | 69 | } catch (Exception e) { 70 | 71 | if (e instanceof InterruptedException) { 72 | logger.error("Trans form thread has been interrupted ", e); 73 | } else if (e instanceof ExecutionException) { 74 | logger.error("Trans form callable exception ", e); 75 | } else { 76 | logger.error("Consumer catch unknow exception ", e); 77 | } 78 | 79 | logger.error("Exception occurred , Call roll back!"); 80 | rollBack.set(false); //回滚 81 | try { 82 | Thread.sleep(500); 83 | } catch (InterruptedException e1) { 84 | logger.error("Consumer task has benn interrupted "); 85 | running = false; 86 | break; 87 | } 88 | } finally { 89 | if (elasticsearchMetadata != null) { 90 | elasticsearchMetadata.recycle(); 91 | } 92 | 93 | } 94 | } 95 | } 96 | 97 | public void register(Consumer consumer) { 98 | consumers.add(consumer); 99 | } 100 | 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/consum/es/ElasticSearchConsumer.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.consum.es; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * 说明 .
10 | *

11 | *

12 | * Copyright: Copyright (c) 2017/11/19 下午6:51 13 | *

14 | * Company: xxx 15 | *

16 | * 17 | * @author zhongcheng_m@yeah.net 18 | * @version 1.0.0 19 | */ 20 | public class ElasticSearchConsumer implements Consumer { 21 | 22 | 23 | private ElasticsearchService elasticsearchService; 24 | 25 | public ElasticSearchConsumer(ElasticsearchService elasticsearchService) { 26 | this.elasticsearchService = elasticsearchService; 27 | } 28 | 29 | @Override 30 | public void consume(ElasticsearchMetadata metadata) { 31 | 32 | List esEntries = metadata.getEsEntries(); 33 | if (esEntries != null && esEntries.size() > 0) { 34 | 35 | esEntries.forEach(esEntry -> { 36 | int eventType = esEntry.getEventType(); 37 | String index = esEntry.getIndex(); 38 | String type = esEntry.getType(); 39 | List esRowDatas = esEntry.getEsRowDatas(); 40 | 41 | if (ElasticsearchMetadata.INSERT == eventType) { 42 | elasticsearchService.insertById(index, type, esRowDatas); 43 | } else if (ElasticsearchMetadata.DELETE == eventType) { 44 | 45 | elasticsearchService.deleteById(index, type, esRowDatas); 46 | } else if (ElasticsearchMetadata.UPDATE == eventType) { 47 | 48 | elasticsearchService.update(index, type, esRowDatas); 49 | } 50 | 51 | }); 52 | } 53 | } 54 | 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/consum/es/ElasticsearchMetadata.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.consum.es; 2 | 3 | import com.google.common.base.Joiner; 4 | import com.totoro.canal.es.common.RecycleAble; 5 | import com.totoro.canal.es.transform.TransForm; 6 | import io.netty.util.Recycler; 7 | 8 | import java.util.Map; 9 | 10 | /** 11 | * 说明 .
12 | *

13 | *

14 | * Copyright: Copyright (c) 2017/11/19 下午6:32 15 | *

16 | * Company: xxx 17 | *

18 | * 19 | * @author zhongcheng_m@yeah.net 20 | * @version 1.0.0 21 | */ 22 | public class ElasticsearchMetadata implements RecycleAble { 23 | 24 | public final static int INSERT = 1; 25 | 26 | public final static int DELETE = 2; 27 | 28 | public final static int UPDATE = 3; 29 | 30 | private Long batchId; 31 | 32 | private EsEntryArrayList esEntries; 33 | 34 | private final Recycler.Handle handle; 35 | 36 | private TransForm transForm; 37 | 38 | public ElasticsearchMetadata(Recycler.Handle handle) { 39 | this.handle = handle; 40 | } 41 | 42 | @Override 43 | public boolean recycle() { 44 | batchId = null; 45 | if (esEntries != null) { 46 | esEntries.recycle(); 47 | } 48 | 49 | if(transForm!=null){ 50 | transForm.recycle(); 51 | } 52 | 53 | handle.recycle(this); 54 | return true; 55 | } 56 | 57 | public static class EsEntry implements RecycleAble { 58 | 59 | private String index; 60 | 61 | private String type; 62 | 63 | private int eventType; 64 | 65 | private EsRowDataArrayList esRowDatas; 66 | 67 | private final Recycler.Handle handle; 68 | 69 | @Override 70 | public boolean recycle() { 71 | index = null; 72 | type = null; 73 | eventType = 0; 74 | if (esRowDatas != null) { 75 | esRowDatas.recycle(); 76 | } 77 | handle.recycle(this); 78 | return true; 79 | } 80 | 81 | 82 | public EsEntry(Recycler.Handle handle) { 83 | this.handle = handle; 84 | } 85 | 86 | public String getIndex() { 87 | return index; 88 | } 89 | 90 | public EsEntry setIndex(String index) { 91 | this.index = index; 92 | return this; 93 | } 94 | 95 | public String getType() { 96 | return type; 97 | } 98 | 99 | public EsEntry setType(String type) { 100 | this.type = type; 101 | return this; 102 | } 103 | 104 | public EsRowDataArrayList getEsRowDatas() { 105 | return esRowDatas; 106 | } 107 | 108 | public EsEntry setEsRowDatas(EsRowDataArrayList esRowDatas) { 109 | this.esRowDatas = esRowDatas; 110 | return this; 111 | } 112 | 113 | public int getEventType() { 114 | return eventType; 115 | } 116 | 117 | public EsEntry setEventType(int eventType) { 118 | this.eventType = eventType; 119 | return this; 120 | } 121 | 122 | 123 | @Override 124 | public String toString() { 125 | final StringBuilder sb = new StringBuilder("EsEntry{"); 126 | sb.append("index='").append(index).append('\''); 127 | sb.append(", type='").append(type).append('\''); 128 | sb.append(", eventType=").append(getEventTypeName(eventType)); 129 | sb.append(", esRowDatas=").append(Joiner.on(",").join(esRowDatas)); 130 | sb.append('}'); 131 | return sb.toString(); 132 | } 133 | 134 | 135 | public static String getEventTypeName(int eventType) { 136 | if (eventType == INSERT) { 137 | return "INSERT"; 138 | } else if (eventType == DELETE) { 139 | return "DELETE"; 140 | } else if (eventType == UPDATE) { 141 | return "UPDATE"; 142 | } else { 143 | return "UNKNOW_TYPE"; 144 | } 145 | } 146 | 147 | 148 | } 149 | 150 | public static class EsRowData implements RecycleAble { 151 | public String idColumn; 152 | 153 | public EsColumnHashMap rowData; 154 | 155 | private final Recycler.Handle handle; 156 | 157 | public EsRowData(Recycler.Handle handle) { 158 | this.handle = handle; 159 | } 160 | 161 | @Override 162 | public boolean recycle() { 163 | idColumn = null; 164 | 165 | if(rowData!=null){ 166 | rowData.recycle(); 167 | } 168 | handle.recycle(this); 169 | return true; 170 | } 171 | 172 | 173 | public String getIdColumn() { 174 | return idColumn; 175 | } 176 | 177 | public EsRowData setIdColumn(String idColumn) { 178 | this.idColumn = idColumn; 179 | return this; 180 | } 181 | 182 | public Map getRowData() { 183 | return rowData; 184 | } 185 | 186 | public EsRowData setRowData(EsColumnHashMap rowData) { 187 | this.rowData = rowData; 188 | return this; 189 | } 190 | 191 | @Override 192 | public String toString() { 193 | final StringBuilder sb = new StringBuilder("EsRowData{"); 194 | sb.append("idColumn='").append(idColumn).append('\''); 195 | sb.append(", rowData=").append(rowData); 196 | sb.append('}'); 197 | return sb.toString(); 198 | } 199 | 200 | 201 | } 202 | 203 | 204 | public Long getBatchId() { 205 | return batchId; 206 | } 207 | 208 | public ElasticsearchMetadata setBatchId(Long batchId) { 209 | this.batchId = batchId; 210 | return this; 211 | } 212 | 213 | 214 | public EsEntryArrayList getEsEntries() { 215 | return esEntries; 216 | } 217 | 218 | public ElasticsearchMetadata setEsEntries(EsEntryArrayList esEntries) { 219 | this.esEntries = esEntries; 220 | return this; 221 | } 222 | 223 | @Override 224 | public String toString() { 225 | final StringBuilder sb = new StringBuilder("ElasticsearchMetadata{"); 226 | sb.append("batchId=").append(batchId); 227 | sb.append(", esEntries=").append(Joiner.on(",").join(esEntries)); 228 | sb.append('}'); 229 | return sb.toString(); 230 | } 231 | 232 | public ElasticsearchMetadata setTransForm(TransForm transForm) { 233 | this.transForm = transForm; 234 | return this; 235 | } 236 | 237 | public TransForm getTransForm() { 238 | return transForm; 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/consum/es/ElasticsearchService.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.consum.es; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * @author zhongcheng 7 | */ 8 | public interface ElasticsearchService { 9 | 10 | void insertById(String index, String type, List esRowDataList); 11 | 12 | void update(String index, String type, List esRowDataList); 13 | 14 | void deleteById(String index, String type, List esRowDataList); 15 | 16 | void close(); 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/consum/es/EsColumnHashMap.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.consum.es; 2 | 3 | import com.totoro.canal.es.common.RecycleAble; 4 | import io.netty.util.Recycler; 5 | 6 | import java.util.HashMap; 7 | 8 | /** 9 | * 说明 .
10 | *

11 | *

12 | * Copyright: Copyright (c) 2017/12/22 下午1:00 13 | *

14 | * Company: xxx 15 | *

16 | * 17 | * @author zhongcheng_m@yeah.net 18 | * @version 1.0.0 19 | */ 20 | public class EsColumnHashMap extends HashMap implements RecycleAble { 21 | 22 | 23 | private final Recycler.Handle handle; 24 | 25 | 26 | public EsColumnHashMap(Recycler.Handle handle) { 27 | this.handle = handle; 28 | } 29 | 30 | 31 | @Override 32 | public boolean recycle() { 33 | this.clear(); 34 | handle.recycle(this); 35 | return true; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/consum/es/EsConf.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.consum.es; 2 | 3 | import com.google.common.base.Preconditions; 4 | import com.totoro.canal.es.util.IPAddressUtil; 5 | import org.apache.commons.lang.StringUtils; 6 | 7 | import java.io.Serializable; 8 | 9 | /** 10 | * 说明 .
11 | *

12 | *

13 | * Copyright: Copyright (c) 2017/12/19 下午4:39 14 | *

15 | * Company: xxx 16 | *

17 | * 18 | * @author zhongcheng_m@yeah.net 19 | * @version 1.0.0 20 | */ 21 | public class EsConf implements Serializable { 22 | 23 | 24 | private String address; 25 | 26 | private String username; 27 | 28 | private String password; 29 | 30 | private String clusterName; 31 | 32 | 33 | public String getAddress() { 34 | return address; 35 | } 36 | 37 | public EsConf setAddress(String address) { 38 | this.address = address; 39 | return this; 40 | } 41 | 42 | public String getUsername() { 43 | return username; 44 | } 45 | 46 | public EsConf setUsername(String username) { 47 | this.username = username; 48 | return this; 49 | } 50 | 51 | public String getPassword() { 52 | return password; 53 | } 54 | 55 | public EsConf setPassword(String password) { 56 | this.password = password; 57 | return this; 58 | } 59 | 60 | public String getClusterName() { 61 | return clusterName; 62 | } 63 | 64 | public EsConf setClusterName(String clusterName) { 65 | this.clusterName = clusterName; 66 | return this; 67 | } 68 | 69 | 70 | public EsConf builder() { 71 | Preconditions.checkArgument(!StringUtils.isEmpty(address)); 72 | Preconditions.checkArgument(!StringUtils.isEmpty(clusterName)); 73 | Preconditions.checkArgument(IPAddressUtil.isAddress(address)); 74 | return this; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/consum/es/EsEntryArrayList.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.consum.es; 2 | 3 | import com.totoro.canal.es.common.RecycleAble; 4 | import io.netty.util.Recycler; 5 | 6 | import java.util.ArrayList; 7 | 8 | /** 9 | * 说明 .
10 | *

11 | *

12 | * Copyright: Copyright (c) 2017/12/22 下午2:21 13 | *

14 | * Company: xxx 15 | *

16 | * 17 | * @author zhongcheng_m@yeah.net 18 | * @version 1.0.0 19 | */ 20 | public class EsEntryArrayList extends ArrayList implements RecycleAble { 21 | 22 | private final Recycler.Handle handle; 23 | 24 | 25 | public EsEntryArrayList(Recycler.Handle handle) { 26 | this.handle = handle; 27 | } 28 | 29 | @Override 30 | public boolean recycle() { 31 | 32 | if (this.size() > 0) { 33 | this.forEach(ElasticsearchMetadata.EsEntry::recycle); 34 | } 35 | 36 | this.clear(); 37 | handle.recycle(this); 38 | return true; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/consum/es/EsRowDataArrayList.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.consum.es; 2 | 3 | import com.totoro.canal.es.common.RecycleAble; 4 | import io.netty.util.Recycler; 5 | 6 | import java.util.ArrayList; 7 | 8 | /** 9 | * 说明 .
10 | *

11 | *

12 | * Copyright: Copyright (c) 2017/12/22 下午2:25 13 | *

14 | * Company: xxx 15 | *

16 | * 17 | * @author zhongcheng_m@yeah.net 18 | * @version 1.0.0 19 | */ 20 | public class EsRowDataArrayList extends ArrayList implements RecycleAble { 21 | private final Recycler.Handle handle; 22 | 23 | public EsRowDataArrayList(Recycler.Handle handle) { 24 | this.handle = handle; 25 | } 26 | 27 | @Override 28 | public boolean recycle() { 29 | if (this.size() > 0) { 30 | this.forEach(ElasticsearchMetadata.EsRowData::recycle); 31 | } 32 | this.clear(); 33 | handle.recycle(this); 34 | return true; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/consum/es/impl/ElasticsearchServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.consum.es.impl; 2 | 3 | 4 | import com.totoro.canal.es.consum.es.ElasticsearchMetadata; 5 | import com.totoro.canal.es.consum.es.ElasticsearchService; 6 | import com.totoro.canal.es.consum.es.EsConf; 7 | import org.elasticsearch.client.transport.TransportClient; 8 | import org.elasticsearch.common.recycler.Recycler; 9 | import org.elasticsearch.common.settings.Settings; 10 | import org.elasticsearch.common.transport.InetSocketTransportAddress; 11 | import org.elasticsearch.transport.client.PreBuiltTransportClient; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.net.InetAddress; 16 | import java.net.UnknownHostException; 17 | import java.util.List; 18 | import java.util.Map; 19 | 20 | /** 21 | * 说明 .
22 | *

23 | *

24 | * Copyright: Copyright (c) 2017/11/19 下午6:53 25 | *

26 | * Company: xxx 27 | *

28 | * 29 | * @author zhongcheng_m@yeah.net 30 | * @version 1.0.0 31 | */ 32 | public class ElasticsearchServiceImpl implements ElasticsearchService { 33 | 34 | private Logger logger = LoggerFactory.getLogger(getClass()); 35 | 36 | private TransportClient transportClient; 37 | 38 | 39 | public ElasticsearchServiceImpl(EsConf esConf) throws UnknownHostException { 40 | 41 | String clusterName = esConf.getClusterName(); 42 | String address = esConf.getAddress(); 43 | String[] hostPort = address.split(":"); 44 | 45 | logger.info("Connect to elasticsearch {}:{}", clusterName, address); 46 | 47 | Settings settings = Settings.builder().put("cluster.name", clusterName) 48 | .put("client.transport.sniff", true) 49 | .build(); 50 | transportClient = new PreBuiltTransportClient(settings) 51 | .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(hostPort[0]), Integer.valueOf(hostPort[1]))); 52 | 53 | 54 | logger.info("Complete the connection to elasticsearch"); 55 | } 56 | 57 | @Override 58 | public void insertById(final String index, final String type, final List esRowDataList) { 59 | esRowDataList.forEach(esRowData -> { 60 | String idColumn = esRowData.getIdColumn(); 61 | Map dataMap = esRowData.getRowData(); 62 | String id = (String) esRowData.getRowData().get(idColumn); 63 | transportClient.prepareIndex(index, type, id).setSource(dataMap).get(); 64 | logger.info("Insert into elasticsearch ====> {} ", index + "." + type + "." + id); 65 | }); 66 | 67 | } 68 | 69 | @Override 70 | public void update(String index, String type, List esRowDataList) { 71 | esRowDataList.forEach(esRowData -> { 72 | String idColumn = esRowData.getIdColumn(); 73 | Map dataMap = esRowData.getRowData(); 74 | String id = (String) esRowData.getRowData().get(idColumn); 75 | transportClient.prepareIndex(index, type, id).setSource(dataMap).get(); 76 | logger.info("Update into elasticsearch ====> {} ", index + "." + type + "." + id); 77 | }); 78 | } 79 | 80 | @Override 81 | public void deleteById(String index, String type, List esRowDataList) { 82 | esRowDataList.forEach(esRowData -> { 83 | String idColumn = esRowData.getIdColumn(); 84 | String id = (String) esRowData.getRowData().get(idColumn); 85 | transportClient.prepareDelete(index, type, id).get(); 86 | logger.info("Delete into elasticsearch ====> {} ", index + "." + type + "." + id); 87 | 88 | }); 89 | 90 | } 91 | 92 | @Override 93 | public void close() { 94 | transportClient.close(); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/select/exception/SelectException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2010-2101 Alibaba Group Holding Limited. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.totoro.canal.es.select.exception; 18 | 19 | /** 20 | * 标题、简要说明.
21 | * 类详细说明. 22 | *

23 | * Copyright: Copyright (c) 24 | *

25 | * Company: xx 26 | *

27 | * 28 | * @author zhongcheng_m@yeah.net 29 | * @version 1.0.0 30 | */ 31 | public class SelectException extends RuntimeException { 32 | 33 | /** 34 | * 35 | */ 36 | private static final long serialVersionUID = 1L; 37 | 38 | public SelectException(String cause){ 39 | super(cause); 40 | } 41 | 42 | public SelectException(Throwable t){ 43 | super(t); 44 | } 45 | 46 | public SelectException(String cause, Throwable t){ 47 | super(cause, t); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/select/selector/CanalConf.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.select.selector; 2 | 3 | import com.google.common.base.Preconditions; 4 | import com.totoro.canal.es.common.TotoroException; 5 | import com.totoro.canal.es.select.selector.canal.CanalEmbedSelector; 6 | import com.totoro.canal.es.util.IPAddressUtil; 7 | import org.apache.commons.lang.StringUtils; 8 | 9 | import java.io.Serializable; 10 | 11 | /** 12 | * 说明 .
13 | *

14 | *

15 | * Copyright: Copyright (c) 2017/12/04 上午10:50 16 | *

17 | * Company: xxx 18 | *

19 | * 20 | * @author zhongcheng_m@yeah.net 21 | * @version 1.0.0 22 | */ 23 | public class CanalConf implements Serializable { 24 | 25 | private CanalEmbedSelector.Mode mode = CanalEmbedSelector.Mode.SIGN; 26 | 27 | //#canal的实例名字 28 | private String destination; 29 | 30 | private String filterPatten = ""; 31 | 32 | //canal 地址 33 | private String address; 34 | 35 | private String zkAddress; 36 | 37 | private String userName; 38 | 39 | private String passWord; 40 | 41 | private String accept; 42 | 43 | 44 | public CanalEmbedSelector.Mode getMode() { 45 | return mode; 46 | } 47 | 48 | public CanalConf setMode(String mode) { 49 | if (!StringUtils.isEmpty(mode)) { 50 | mode = mode.toUpperCase(); 51 | try { 52 | this.mode = CanalEmbedSelector.Mode.valueOf(mode); 53 | } catch (IllegalArgumentException e) { 54 | throw new TotoroException("no match patten,mode:" + mode); 55 | } 56 | } 57 | return this; 58 | } 59 | 60 | public String getDestination() { 61 | return destination; 62 | } 63 | 64 | public CanalConf setDestination(String destination) { 65 | this.destination = destination; 66 | return this; 67 | } 68 | 69 | public String getAddress() { 70 | return address; 71 | } 72 | 73 | public CanalConf setAddress(String address) { 74 | this.address = address; 75 | return this; 76 | } 77 | 78 | public String getFilterPatten() { 79 | return filterPatten; 80 | } 81 | 82 | public CanalConf setFilterPatten(String filterPatten) { 83 | if (!StringUtils.isEmpty(filterPatten)) { 84 | this.filterPatten = filterPatten; 85 | } 86 | 87 | return this; 88 | } 89 | 90 | public String getZkAddress() { 91 | return zkAddress; 92 | } 93 | 94 | public CanalConf setZkAddress(String zkAddress) { 95 | this.zkAddress = zkAddress; 96 | return this; 97 | } 98 | 99 | public String getUserName() { 100 | return userName; 101 | } 102 | 103 | public CanalConf setUserName(String userName) { 104 | this.userName = userName; 105 | return this; 106 | } 107 | 108 | public String getPassWord() { 109 | return passWord; 110 | } 111 | 112 | public CanalConf setPassWord(String passWord) { 113 | this.passWord = passWord; 114 | return this; 115 | } 116 | 117 | 118 | public String getAccept() { 119 | return accept; 120 | } 121 | 122 | public CanalConf setAccept(String accept) { 123 | this.accept = accept; 124 | return this; 125 | } 126 | 127 | public CanalConf builder() { 128 | 129 | Preconditions.checkArgument(StringUtils.isNotEmpty(destination), "Illegal destination , destination can't be empty"); 130 | Preconditions.checkArgument(StringUtils.isNotEmpty(accept), "Illegal accept , accept can't be empty"); 131 | 132 | if (CanalEmbedSelector.Mode.SIGN.equals(mode)) { 133 | Preconditions.checkArgument(IPAddressUtil.isAddress(address), "Illegal address : %s", address); 134 | 135 | } else if (CanalEmbedSelector.Mode.CLUSTER.equals(mode)) { 136 | Preconditions.checkArgument(IPAddressUtil.isAddress(address), "Illegal zkAddress : %s", zkAddress); 137 | } 138 | 139 | return this; 140 | } 141 | 142 | 143 | @Override 144 | public String toString() { 145 | final StringBuilder sb = new StringBuilder("CanalConf{"); 146 | sb.append("mode=").append(mode); 147 | sb.append(", destination='").append(destination).append('\''); 148 | sb.append(", filterPatten='").append(filterPatten).append('\''); 149 | sb.append(", address='").append(address).append('\''); 150 | sb.append(", zkAddress='").append(zkAddress).append('\''); 151 | sb.append(", userName='").append(userName).append('\''); 152 | sb.append(", accept='").append(accept).append('\''); 153 | sb.append('}'); 154 | return sb.toString(); 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/select/selector/SelectorTask.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.select.selector; 2 | 3 | import com.alibaba.otter.canal.common.utils.BooleanMutex; 4 | import com.alibaba.otter.canal.protocol.CanalEntry; 5 | import com.alibaba.otter.canal.protocol.Message; 6 | import com.google.protobuf.InvalidProtocolBufferException; 7 | import com.totoro.canal.es.TotoroBootStrap; 8 | import com.totoro.canal.es.channel.TotoroChannel; 9 | import com.totoro.canal.es.common.RollBackMonitorFactory; 10 | import com.totoro.canal.es.common.GlobalTask; 11 | import org.apache.commons.lang.SystemUtils; 12 | import org.springframework.util.CollectionUtils; 13 | 14 | import java.text.SimpleDateFormat; 15 | import java.util.Date; 16 | import java.util.List; 17 | 18 | /** 19 | * 说明 .
20 | *

21 | *

22 | * Copyright: Copyright (c) 2017/12/01 下午4:42 23 | *

24 | * Company: xxx 25 | *

26 | * 27 | * @author zhongcheng_m@yeah.net 28 | * @version 1.0.0 29 | */ 30 | public class SelectorTask extends GlobalTask { 31 | 32 | 33 | private TotoroSelector totoroSelector; 34 | 35 | private TotoroChannel channel; 36 | 37 | private BooleanMutex rollBack = RollBackMonitorFactory.getBooleanMutex(); 38 | 39 | private static String context_format = null; 40 | 41 | private static String row_format = null; 42 | 43 | private static String transaction_format = null; 44 | 45 | private static final String SEP = SystemUtils.LINE_SEPARATOR; 46 | 47 | private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; 48 | 49 | private SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT); 50 | 51 | static { 52 | context_format = SEP + "****************************************************" + SEP; 53 | context_format += "* Batch Id: [{}] ,count : [{}] , memsize : [{}] , Time : {}" + SEP; 54 | context_format += "* Start : [{}] " + SEP; 55 | context_format += "* End : [{}] " + SEP; 56 | context_format += "****************************************************" + SEP; 57 | 58 | row_format = SEP 59 | + "----------------> binlog[{}:{}] , name[{},{}] , eventType : {} , executeTime : {} , delay : {}ms" 60 | + SEP; 61 | 62 | transaction_format = SEP + "================> binlog[{}:{}] , executeTime : {} , delay : {}ms" + SEP; 63 | 64 | } 65 | 66 | /** 67 | * 是否耦合度过高？？ 68 | */ 69 | 70 | public SelectorTask(TotoroSelector totoroSelector, TotoroChannel channel, TotoroBootStrap canalScheduler) { 71 | logger.info("Selector task init ......."); 72 | this.totoroSelector = totoroSelector; 73 | this.channel = channel; 74 | logger.info("Selector task complete ......."); 75 | } 76 | 77 | 78 | @Override 79 | public void run() { 80 | running = true; 81 | 82 | totoroSelector.start(); 83 | totoroSelector.rollback(); 84 | 85 | logger.info("Selector task start ......."); 86 | Message message; 87 | rollBack.set(true); 88 | while (running) { 89 | try { 90 | //出现回滚立即停止 91 | message = totoroSelector.selector(); 92 | 93 | 94 | /** 95 | * 当前处理回滚的调度模型，可以保证在消费端出错的时候、正确处理回滚，并正确应答和继续消费数据。 96 | * 97 | * 当发生回滚时，首先 consumer task 会将 rollback 设置为true ,自己停止工作，等待唤醒 98 | * 然后 trans task 也会同样挂起 99 | * channel会拒绝接受 message 和 future ,对于已经提交的 future 会尝试取消 100 | * 101 | * 到此 除了 selector task 以外 的所有线程 全部尽最大努力去停止处理消息，但注意此时还没有回滚 102 | * 103 | * 因为 selector 是循环获取数据，每次循环都会判断 rollback 状态，一旦发现rollback状态，跳出循环 104 | * 返回到 selector task里面 ，task会感知到回滚状态 ，清空渠道中的消息 ，并回滚 至最后一个未应答的 105 | * 消费点，然后丢弃本条消息，重新获取一次消息（回滚的消息） 106 | * 当上面所有工作 都做完了，便完成了回滚 ，selector task 改变回滚状态，重新正常工作 107 | * 108 | * 粗略测试结果 ： 40000条数据，单机测试，当 batchId 能被2整除的时候回滚 109 | * 在不真正消费数据的前提下（消费端直接应答），处理性能非常好 110 | * 111 | */ 112 | if (rollBack.state() == false) { 113 | totoroSelector.rollback(); 114 | logger.info("The rollback happened =============> discard message , batchId ：{}", message.getId()); 115 | //丢弃刚才的消息 116 | message = totoroSelector.selector(); 117 | channel.clearMessage(); 118 | rollBack.set(true); 119 | } 120 | 121 | long batchId = message.getId(); 122 | int size = message.getEntries().size(); 123 | if (batchId == -1 || size == 0) { 124 | message = null;//help gc 125 | } else { 126 | logger.info("Put message into channel =====> batchId :{}", message.getId()); 127 | 128 | if (logger.isDebugEnabled()) { 129 | printSummary(message, batchId, size); 130 | printEntry(message.getEntries()); 131 | } 132 | //将消息放入管道 133 | channel.putMessage(message); 134 | } 135 | } catch (InterruptedException e) { 136 | logger.error("Selector task has been interrupted ", e); 137 | running = false; 138 | break; 139 | } 140 | } 141 | } 142 | 143 | 144 | private void printSummary(Message message, long batchId, int size) { 145 | long memsize = 0; 146 | for (CanalEntry.Entry entry : message.getEntries()) { 147 | memsize += entry.getHeader().getEventLength(); 148 | } 149 | 150 | String startPosition = null; 151 | String endPosition = null; 152 | if (!CollectionUtils.isEmpty(message.getEntries())) { 153 | startPosition = buildPositionForDump(message.getEntries().get(0)); 154 | endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1)); 155 | } 156 | 157 | logger.info(context_format, new Object[]{batchId, size, memsize, format.format(new Date()), startPosition, 158 | endPosition}); 159 | } 160 | 161 | protected String buildPositionForDump(CanalEntry.Entry entry) { 162 | long time = entry.getHeader().getExecuteTime(); 163 | Date date = new Date(time); 164 | return entry.getHeader().getLogfileName() + ":" + entry.getHeader().getLogfileOffset() + ":" 165 | + entry.getHeader().getExecuteTime() + "(" + format.format(date) + ")"; 166 | } 167 | 168 | protected void printEntry(List entrys) { 169 | for (CanalEntry.Entry entry : entrys) { 170 | long executeTime = entry.getHeader().getExecuteTime(); 171 | long delayTime = System.currentTimeMillis() - executeTime; 172 | 173 | if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) { 174 | if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN) { 175 | CanalEntry.TransactionBegin begin = null; 176 | try { 177 | begin = CanalEntry.TransactionBegin.parseFrom(entry.getStoreValue()); 178 | } catch (InvalidProtocolBufferException e) { 179 | throw new RuntimeException("parse event has an error , data:" + entry.toString(), e); 180 | } 181 | // 打印事务头信息，执行的线程id，事务耗时 182 | logger.info(transaction_format, 183 | new Object[]{entry.getHeader().getLogfileName(), 184 | String.valueOf(entry.getHeader().getLogfileOffset()), 185 | String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime)}); 186 | logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId()); 187 | } else if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) { 188 | CanalEntry.TransactionEnd end = null; 189 | try { 190 | end = CanalEntry.TransactionEnd.parseFrom(entry.getStoreValue()); 191 | } catch (InvalidProtocolBufferException e) { 192 | throw new RuntimeException("parse event has an error , data:" + entry.toString(), e); 193 | } 194 | // 打印事务提交信息，事务id 195 | logger.info("----------------\n"); 196 | logger.info(" END ----> transaction id: {}", end.getTransactionId()); 197 | logger.info(transaction_format, 198 | new Object[]{entry.getHeader().getLogfileName(), 199 | String.valueOf(entry.getHeader().getLogfileOffset()), 200 | String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime)}); 201 | } 202 | 203 | continue; 204 | } 205 | 206 | if (entry.getEntryType() == CanalEntry.EntryType.ROWDATA) { 207 | CanalEntry.RowChange rowChage = null; 208 | try { 209 | rowChage = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); 210 | } catch (Exception e) { 211 | throw new RuntimeException("parse event has an error , data:" + entry.toString(), e); 212 | } 213 | 214 | CanalEntry.EventType eventType = rowChage.getEventType(); 215 | 216 | logger.info(row_format, 217 | new Object[]{entry.getHeader().getLogfileName(), 218 | String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(), 219 | entry.getHeader().getTableName(), eventType, 220 | String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime)}); 221 | 222 | if (eventType == CanalEntry.EventType.QUERY || rowChage.getIsDdl()) { 223 | logger.info(" sql ----> " + rowChage.getSql() + SEP); 224 | continue; 225 | } 226 | 227 | for (CanalEntry.RowData rowData : rowChage.getRowDatasList()) { 228 | if (eventType == CanalEntry.EventType.DELETE) { 229 | printColumn(rowData.getBeforeColumnsList()); 230 | } else if (eventType == CanalEntry.EventType.INSERT) { 231 | printColumn(rowData.getAfterColumnsList()); 232 | } else { 233 | printColumn(rowData.getAfterColumnsList()); 234 | } 235 | } 236 | } 237 | } 238 | } 239 | 240 | protected void printColumn(List columns) { 241 | for (CanalEntry.Column column : columns) { 242 | StringBuilder builder = new StringBuilder(); 243 | builder.append(column.getName() + " : " + column.getValue()); 244 | builder.append(" type=" + column.getMysqlType()); 245 | if (column.getUpdated()) { 246 | builder.append(" update=" + column.getUpdated()); 247 | } 248 | builder.append(SEP); 249 | logger.info(builder.toString()); 250 | } 251 | } 252 | 253 | } 254 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/select/selector/TotoroSelector.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.select.selector; 2 | 3 | import com.alibaba.otter.canal.protocol.Message; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | *

9 | * Copyright: Copyright (c) 10 | *

11 | * Company: xx 12 | *

13 | * 14 | * @author zhongcheng_m@yeah.net 15 | * @version 1.0.0 16 | */ 17 | public interface TotoroSelector { 18 | 19 | /** 20 | */ 21 | void start(); 22 | 23 | /** 24 | */ 25 | boolean isStart(); 26 | 27 | /** 28 | */ 29 | void stop(); 30 | 31 | /** 32 | */ 33 | Message selector() throws InterruptedException; 34 | 35 | 36 | /** 37 | */ 38 | Long lastEntryTime(); 39 | 40 | /** 41 | */ 42 | List unAckBatchs(); 43 | 44 | /** 45 | */ 46 | void rollback(Long batchId); 47 | 48 | /** 49 | */ 50 | void rollback(); 51 | 52 | /** 53 | */ 54 | void ack(Long batchId); 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/select/selector/canal/CanalEmbedSelector.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.select.selector.canal; 2 | 3 | import com.alibaba.otter.canal.client.CanalConnector; 4 | import com.alibaba.otter.canal.client.CanalConnectors; 5 | import com.alibaba.otter.canal.common.utils.AddressUtils; 6 | import com.alibaba.otter.canal.common.utils.BooleanMutex; 7 | import com.alibaba.otter.canal.protocol.Message; 8 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 9 | import com.totoro.canal.es.common.RollBackMonitorFactory; 10 | import com.totoro.canal.es.common.TotoroException; 11 | import com.totoro.canal.es.select.selector.CanalConf; 12 | import com.totoro.canal.es.select.selector.TotoroSelector; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | import sun.net.util.IPAddressUtil; 16 | 17 | import java.net.InetSocketAddress; 18 | import java.net.SocketAddress; 19 | import java.util.List; 20 | import java.util.Properties; 21 | import java.util.concurrent.TimeUnit; 22 | import java.util.concurrent.locks.LockSupport; 23 | 24 | /** 25 | *

26 | * Copyright: Copyright (c) 27 | *

28 | * Company: xx 29 | *

30 | * 31 | * @author zhongcheng_m@yeah.net 32 | * @version 1.0.0 33 | */ 34 | public class CanalEmbedSelector implements TotoroSelector { 35 | 36 | private static final Logger logger = LoggerFactory.getLogger(CanalEmbedSelector.class); 37 | 38 | private volatile boolean running = false; 39 | 40 | private CanalConnector connector; // instance client 41 | 42 | private String destination; 43 | 44 | private static final int maxEmptyTimes = 10; 45 | 46 | private int batchSize = 5 * 1024; 47 | 48 | private String filterPatten; 49 | 50 | private long batchTimeout = -1L; 51 | 52 | private Mode mode; 53 | 54 | private BooleanMutex rollBack = RollBackMonitorFactory.getBooleanMutex(); 55 | 56 | public enum Mode { 57 | SIGN, CLUSTER 58 | } 59 | 60 | public CanalEmbedSelector(CanalConf conf) { 61 | 62 | logger.info("TotoroSelector init start , conf :{}", conf.toString()); 63 | 64 | this.mode = conf.getMode(); 65 | this.destination = conf.getDestination(); 66 | this.filterPatten = conf.getFilterPatten(); 67 | 68 | String userName = conf.getUserName(); 69 | String passWord = conf.getPassWord(); 70 | 71 | if (Mode.SIGN.equals(mode)) { 72 | String address = conf.getAddress(); 73 | 74 | String[] hostPort = address.split(":"); 75 | 76 | String ip = hostPort[0]; 77 | Integer port = Integer.valueOf(hostPort[1]); 78 | 79 | SocketAddress socketAddress = new InetSocketAddress(ip, port); 80 | 81 | connector = CanalConnectors.newSingleConnector(socketAddress, 82 | destination, 83 | userName, 84 | passWord); 85 | 86 | } else if (Mode.CLUSTER.equals(mode)) { 87 | String zkAddress = conf.getZkAddress(); 88 | connector = CanalConnectors.newClusterConnector(zkAddress, destination, userName, passWord); 89 | } else { 90 | throw new TotoroException("Invalid mode"); 91 | } 92 | logger.info("TotoroSelector init complete ......."); 93 | } 94 | 95 | 96 | @Override 97 | public void start() { 98 | if (running) { 99 | return; 100 | } 101 | connector.connect(); 102 | connector.subscribe(filterPatten); 103 | running = true; 104 | } 105 | 106 | @Override 107 | public boolean isStart() { 108 | return running; 109 | } 110 | 111 | @Override 112 | public void stop() { 113 | connector.disconnect(); 114 | } 115 | 116 | @Override 117 | public Message selector() throws InterruptedException { 118 | if (!running) { 119 | throw new RuntimeException("CanalEmbedSelector has benn not start"); 120 | } 121 | 122 | Message message = null; 123 | int emptyTimes = 0; 124 | 125 | if (batchTimeout < 0) { 126 | while (running) { 127 | message = connector.getWithoutAck(batchSize); 128 | 129 | if (message == null || message.getId() == -1L) { 130 | if (rollBack.state() == false) { 131 | break; 132 | } else { 133 | applyWait(emptyTimes++); 134 | } 135 | 136 | } else { 137 | break; 138 | } 139 | } 140 | if (!running) { 141 | throw new InterruptedException(); 142 | } 143 | } else { 144 | 145 | while (running) { 146 | message = connector.getWithoutAck(batchSize, batchTimeout, TimeUnit.SECONDS); 147 | if (message == null || message.getId() == -1L) { 148 | continue; 149 | } else { 150 | break; 151 | } 152 | } 153 | if (!running) { 154 | throw new InterruptedException(); 155 | } 156 | } 157 | 158 | return message; 159 | } 160 | 161 | @Override 162 | public Long lastEntryTime() { 163 | return null; 164 | } 165 | 166 | @Override 167 | public List unAckBatchs() { 168 | return null; 169 | } 170 | 171 | @Override 172 | public void rollback(Long batchId) { 173 | connector.rollback(batchId); 174 | } 175 | 176 | @Override 177 | public void rollback() { 178 | connector.rollback(); 179 | } 180 | 181 | @Override 182 | public void ack(Long batchId) { 183 | connector.ack(batchId); 184 | } 185 | 186 | 187 | private void applyWait(int emptyTimes) throws InterruptedException { 188 | int newEmptyTimes = emptyTimes > maxEmptyTimes ? maxEmptyTimes : emptyTimes; 189 | if (emptyTimes >= 3) { 190 | Thread.yield(); 191 | LockSupport.parkNanos(10000 * 1000L * newEmptyTimes); 192 | } 193 | } 194 | 195 | 196 | } 197 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/transform/EsAdapter.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.transform; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * 说明 .
9 | *

10 | *

11 | * Copyright: Copyright (c) 2017/12/19 上午9:49 12 | *

13 | * Company: xxx 14 | *

15 | * 16 | * @author zhongcheng_m@yeah.net 17 | * @version 1.0.0 18 | */ 19 | public interface EsAdapter { 20 | 21 | String getEsIdColumn(String database, String table); 22 | 23 | int getEsEventType(CanalEntry.EventType eventType); 24 | 25 | List getColumnList(int esEventType, CanalEntry.RowData rowData); 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/transform/MessageFilter.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.transform; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry; 4 | 5 | /** 6 | * 说明 .
7 | *

8 | *

9 | * Copyright: Copyright (c) 2017/12/18 下午3:06 10 | *

11 | * Company: xxx 12 | *

13 | * 14 | * @author zhongcheng_m@yeah.net 15 | * @version 1.0.0 16 | */ 17 | public interface MessageFilter { 18 | 19 | boolean filter(CanalEntry.Entry entry); 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/transform/MessageFilterChain.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.transform; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.concurrent.locks.ReentrantReadWriteLock; 10 | 11 | /** 12 | * 说明 .
13 | *

14 | *

15 | * Copyright: Copyright (c) 2017/12/18 下午3:07 16 | *

17 | * Company: xxx 18 | *

19 | * 20 | * @author zhongcheng_m@yeah.net 21 | * @version 1.0.0 22 | */ 23 | public class MessageFilterChain implements MessageFilter { 24 | 25 | private Logger logger = LoggerFactory.getLogger(getClass()); 26 | 27 | private MessageFilterChain() { 28 | 29 | } 30 | 31 | private List messageFilterList = new ArrayList<>(); 32 | 33 | private ReentrantReadWriteLock reentrantReadWriteLock = new ReentrantReadWriteLock(); 34 | 35 | private ReentrantReadWriteLock.ReadLock readLock = reentrantReadWriteLock.readLock(); 36 | 37 | private ReentrantReadWriteLock.WriteLock writeLock = reentrantReadWriteLock.writeLock(); 38 | 39 | private static volatile MessageFilterChain messageFilterChain = null; 40 | 41 | 42 | public static MessageFilterChain getInstance() { 43 | if (messageFilterChain == null) { 44 | synchronized (MessageFilterChain.class) { 45 | if (messageFilterChain == null) { 46 | messageFilterChain = new MessageFilterChain(); 47 | } 48 | } 49 | } 50 | return messageFilterChain; 51 | } 52 | 53 | @Override 54 | public boolean filter(CanalEntry.Entry entry) { 55 | try { 56 | readLock.lock(); 57 | boolean accept = true; 58 | for (MessageFilter filter : messageFilterList) { 59 | accept &= filter.filter(entry); 60 | } 61 | return accept; 62 | } finally { 63 | readLock.unlock(); 64 | } 65 | 66 | } 67 | 68 | 69 | public void register(MessageFilter messageFilter) { 70 | try { 71 | writeLock.lock(); 72 | messageFilterList.add(messageFilter); 73 | logger.info(messageFilter.getClass().getSimpleName() + " has benn registered to message filter chain "); 74 | } finally { 75 | writeLock.unlock(); 76 | } 77 | } 78 | 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/transform/SimpleEsAdapter.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.transform; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry; 4 | import com.totoro.canal.es.consum.es.ElasticsearchMetadata; 5 | import com.totoro.canal.es.select.selector.CanalConf; 6 | import org.apache.commons.lang.StringUtils; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.concurrent.ConcurrentHashMap; 13 | 14 | /** 15 | * 说明 .
16 | *

17 | *

18 | * Copyright: Copyright (c) 2017/12/19 上午9:37 19 | *

20 | * Company: xxx 21 | *

22 | * 23 | * @author zhongcheng_m@yeah.net 24 | * @version 1.0.0 25 | */ 26 | public class SimpleEsAdapter implements EsAdapter { 27 | 28 | private Logger logger = LoggerFactory.getLogger(getClass()); 29 | 30 | private static Map eventTypePair = new ConcurrentHashMap<>(); 31 | 32 | private static final String DELIMITER = ","; 33 | 34 | private static final String CONNECTOR = "\\."; 35 | 36 | private static final String CONNECTOR_TEP = "."; 37 | 38 | /** 39 | * ConcurrentHashMap其实可以考虑换成普通 hasmap 40 | * 因为正常情况下不会出现并发写 导致的扩容死循环问题 41 | * 但考虑到以后可能会增加功能而导致并发，所以选择ConcurrentHashMap 42 | * 并且目前大部分是读操作，性能不会相差太多 43 | */ 44 | private static Map idPair = new ConcurrentHashMap<>(); 45 | 46 | public SimpleEsAdapter(CanalConf canalConf) { 47 | 48 | String accept = canalConf.getAccept(); 49 | String[] acceptArr = accept.split(DELIMITER); 50 | for (String str : acceptArr) { 51 | String[] strArr = str.split(CONNECTOR); 52 | if (strArr.length == 3) { 53 | String dataBaseTable = StringUtils.substringBeforeLast(str, CONNECTOR_TEP); 54 | String idColumn = StringUtils.substringAfterLast(str, CONNECTOR_TEP); 55 | idPair.put(dataBaseTable, idColumn); 56 | 57 | logger.info("Add accept :{}", str); 58 | } 59 | } 60 | 61 | } 62 | 63 | static { 64 | eventTypePair.put(CanalEntry.EventType.INSERT, ElasticsearchMetadata.INSERT); 65 | eventTypePair.put(CanalEntry.EventType.UPDATE, ElasticsearchMetadata.UPDATE); 66 | eventTypePair.put(CanalEntry.EventType.DELETE, ElasticsearchMetadata.DELETE); 67 | } 68 | 69 | 70 | @Override 71 | public String getEsIdColumn(String database, String table) { 72 | return idPair.get(database + CONNECTOR_TEP + table); 73 | } 74 | 75 | @Override 76 | public int getEsEventType(CanalEntry.EventType eventType) { 77 | if (eventTypePair.containsKey(eventType)) { 78 | return eventTypePair.get(eventType); 79 | } else { 80 | throw new RuntimeException(); //一般情况下不会发生，因为在filter时已经做过判断 81 | } 82 | } 83 | 84 | @Override 85 | public List getColumnList(int esEventType, CanalEntry.RowData rowData) { 86 | 87 | List columnList; 88 | if (esEventType == ElasticsearchMetadata.DELETE) { 89 | columnList = rowData.getBeforeColumnsList(); 90 | } else { 91 | columnList = rowData.getAfterColumnsList(); 92 | } 93 | 94 | return columnList; 95 | 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/transform/SimpleMessageFilter.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.transform; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry; 4 | 5 | import java.util.HashSet; 6 | import java.util.Set; 7 | 8 | /** 9 | * 说明 .
10 | *

11 | *

12 | * Copyright: Copyright (c) 2017/12/18 下午3:31 13 | *

14 | * Company: xxx 15 | *

16 | * 17 | * @author zhongcheng_m@yeah.net 18 | * @version 1.0.0 19 | */ 20 | public class SimpleMessageFilter implements MessageFilter { 21 | 22 | private static final Set acceptEventType = new HashSet<>(); 23 | 24 | 25 | static { 26 | acceptEventType.add(CanalEntry.EventType.INSERT); 27 | acceptEventType.add(CanalEntry.EventType.DELETE); 28 | acceptEventType.add(CanalEntry.EventType.UPDATE); 29 | } 30 | 31 | @Override 32 | public boolean filter(CanalEntry.Entry entry) { 33 | 34 | //过滤掉事物头尾等 非 row data 的 entry 35 | boolean rowData = entry.getEntryType() == CanalEntry.EntryType.ROWDATA; 36 | 37 | if (!rowData) { 38 | return false; 39 | } 40 | 41 | //只保存 insert update delete 类型的 时间 42 | boolean eventType = filterEventType(entry.getHeader().getEventType()); 43 | 44 | if (!eventType) { 45 | return false; 46 | } 47 | 48 | return true; 49 | } 50 | 51 | 52 | private boolean filterEventType(CanalEntry.EventType eventType) { 53 | return acceptEventType.contains(eventType); 54 | } 55 | } -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/transform/TableFilter.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.transform; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry; 4 | import com.totoro.canal.es.select.selector.CanalConf; 5 | import org.apache.commons.lang.StringUtils; 6 | 7 | import java.util.HashSet; 8 | import java.util.Set; 9 | 10 | /** 11 | * 说明 .
12 | *

13 | *

14 | * Copyright: Copyright (c) 2017/12/18 下午3:29 15 | *

16 | * Company: xxx 17 | *

18 | * 19 | * @author zhongcheng_m@yeah.net 20 | * @version 1.0.0 21 | */ 22 | public class TableFilter implements MessageFilter { 23 | 24 | private static final String DELIMITER = ","; 25 | 26 | private static final String CONNECTOR = "\\."; 27 | 28 | private static final String CONNECTOR_TEP = "."; 29 | 30 | private Set acceptTable = new HashSet<>(); 31 | 32 | public TableFilter(CanalConf canalConf) { 33 | String accept = canalConf.getAccept(); 34 | String[] acceptArr = accept.split(DELIMITER); 35 | for (String str : acceptArr) { 36 | String[] strArr = str.split(CONNECTOR); 37 | if (strArr.length == 2) { 38 | acceptTable.add(str); 39 | } else if (strArr.length == 3) { 40 | acceptTable.add(StringUtils.substringBeforeLast(str, CONNECTOR_TEP)); 41 | } 42 | } 43 | } 44 | 45 | 46 | @Override 47 | public boolean filter(CanalEntry.Entry entry) { 48 | String database = entry.getHeader().getSchemaName(); 49 | String table = entry.getHeader().getTableName(); 50 | return acceptTable.contains(database + CONNECTOR_TEP + table); 51 | } 52 | 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/transform/TotoroTransForm.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.transform; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry; 4 | import com.alibaba.otter.canal.protocol.Message; 5 | import com.google.protobuf.InvalidProtocolBufferException; 6 | import com.totoro.canal.es.common.TotoroObjectPool; 7 | import com.totoro.canal.es.consum.es.ElasticsearchMetadata; 8 | import com.totoro.canal.es.consum.es.EsColumnHashMap; 9 | import com.totoro.canal.es.consum.es.EsEntryArrayList; 10 | import com.totoro.canal.es.consum.es.EsRowDataArrayList; 11 | import io.netty.util.Recycler; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.*; 16 | import java.util.concurrent.Callable; 17 | import java.util.stream.Collectors; 18 | 19 | /** 20 | * 说明 .
21 | *

22 | *

23 | * Copyright: Copyright (c) 2017/11/19 下午6:44 24 | *

25 | * Company: xxx 26 | *

27 | * 28 | * @author zhongcheng_m@yeah.net 29 | * @version 1.0.0 30 | */ 31 | public class TotoroTransForm implements TransForm, Callable { 32 | 33 | private Logger logger = LoggerFactory.getLogger(getClass()); 34 | 35 | private Message message; 36 | 37 | private MessageFilter messageFilterChain = MessageFilterChain.getInstance(); 38 | 39 | private EsAdapter esAdapter; 40 | 41 | public TotoroTransForm(Recycler.Handle handle) { 42 | this.handle = handle; 43 | } 44 | 45 | private final Recycler.Handle handle; 46 | 47 | 48 | @Override 49 | public boolean recycle() { 50 | esAdapter = null; 51 | message = null; 52 | handle.recycle(this); 53 | return true; 54 | } 55 | 56 | @Override 57 | public ElasticsearchMetadata call() throws Exception { 58 | return trans(message); 59 | } 60 | 61 | @Override 62 | public ElasticsearchMetadata trans(Message message) { 63 | List entries = message.getEntries(); 64 | ElasticsearchMetadata elasticsearchMetadata = TotoroObjectPool.esMetadata(); 65 | elasticsearchMetadata.setBatchId(message.getId()); 66 | elasticsearchMetadata.setTransForm(this); 67 | if (entries != null && entries.size() > 0) { 68 | EsEntryArrayList esEntryList = TotoroObjectPool.esEntryArrayList(); 69 | 70 | entries.forEach(entry -> { 71 | if (messageFilterChain.filter(entry)) { 72 | try { 73 | ElasticsearchMetadata.EsEntry esEntry = getElasticsearchMetadata(entry); 74 | esEntryList.add(esEntry); 75 | } catch (InvalidProtocolBufferException e) { 76 | e.printStackTrace(); 77 | } 78 | } 79 | }); 80 | elasticsearchMetadata.setEsEntries(esEntryList); 81 | } 82 | 83 | logger.info("Trans form complete message id =====> {}", message.getId()); 84 | 85 | if (logger.isDebugEnabled()) { 86 | logger.debug("Trans form complete elasticsearch metadata =====> {}", elasticsearchMetadata.toString()); 87 | } 88 | return elasticsearchMetadata; 89 | } 90 | 91 | private ElasticsearchMetadata.EsEntry getElasticsearchMetadata(CanalEntry.Entry entry) throws InvalidProtocolBufferException { 92 | 93 | final String database = entry.getHeader().getSchemaName(); // => index 94 | final String table = entry.getHeader().getTableName();// => type 95 | final CanalEntry.RowChange change = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); 96 | final List rowDataList = change.getRowDatasList(); 97 | 98 | CanalEntry.EventType eventType = entry.getHeader().getEventType(); 99 | final int esEventType = esAdapter.getEsEventType(eventType); 100 | 101 | 102 | EsRowDataArrayList esRowDataList = TotoroObjectPool.esRowDataArrayList(); 103 | 104 | for (CanalEntry.RowData rowData : rowDataList) { 105 | List columnList = esAdapter.getColumnList(esEventType, rowData); 106 | ElasticsearchMetadata.EsRowData esRowData = TotoroObjectPool.esRowData(); 107 | EsColumnHashMap columnMap = TotoroObjectPool.esColumnHashMap(); 108 | columnList.forEach(column -> columnMap.put(column.getName(), column.getValue())); 109 | esRowData.setRowData(columnMap); 110 | esRowData.setIdColumn(esAdapter.getEsIdColumn(database, table));//获取es对应的id Column 111 | esRowDataList.add(esRowData); 112 | } 113 | 114 | ElasticsearchMetadata.EsEntry esEntry = TotoroObjectPool.esEntry(); 115 | 116 | esEntry.setIndex(database) 117 | .setType(table) 118 | .setEsRowDatas(esRowDataList) 119 | .setEventType(esEventType); 120 | 121 | return esEntry; 122 | } 123 | 124 | 125 | public TotoroTransForm setMessage(Message message) { 126 | this.message = message; 127 | return this; 128 | } 129 | 130 | public TotoroTransForm setEsAdapter(EsAdapter esAdapter) { 131 | this.esAdapter = esAdapter; 132 | return this; 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/transform/TransForm.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.transform; 2 | 3 | import com.totoro.canal.es.common.RecycleAble; 4 | 5 | /** 6 | * 7 | * @param 8 | * @param 9 | * @author zhongcheng 10 | */ 11 | public interface TransForm extends RecycleAble { 12 | 13 | O trans(I input); 14 | 15 | 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/transform/TransFormTask.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.transform; 2 | 3 | import com.alibaba.otter.canal.common.utils.BooleanMutex; 4 | import com.alibaba.otter.canal.protocol.Message; 5 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 6 | import com.totoro.canal.es.channel.TotoroChannel; 7 | import com.totoro.canal.es.common.GlobalTask; 8 | import com.totoro.canal.es.common.RollBackMonitorFactory; 9 | import com.totoro.canal.es.common.TotoroObjectPool; 10 | import com.totoro.canal.es.consum.es.ElasticsearchMetadata; 11 | 12 | import java.util.concurrent.Executors; 13 | import java.util.concurrent.Future; 14 | import java.util.concurrent.ThreadFactory; 15 | 16 | /** 17 | * 说明 .
18 | *

19 | *

20 | * Copyright: Copyright (c) 2017/12/01 下午1:27 21 | *

22 | * Company: xxx 23 | *

24 | * 25 | * @author zhongcheng_m@yeah.net 26 | * @version 1.0.0 27 | */ 28 | public class TransFormTask extends GlobalTask { 29 | 30 | private TotoroChannel channel; 31 | 32 | private BooleanMutex booleanMutex = RollBackMonitorFactory.getBooleanMutex(); 33 | 34 | private EsAdapter esAdapter; 35 | 36 | public TransFormTask(TotoroChannel channel, EsAdapter esAdapter, int threadNum) { 37 | 38 | logger.info("TransFormTask init start ......."); 39 | 40 | this.channel = channel; 41 | ThreadFactory threadFactory = new ThreadFactoryBuilder() 42 | .setNameFormat("trans-pool-%d") 43 | .build(); 44 | executorService = Executors.newFixedThreadPool(threadNum, threadFactory); 45 | this.esAdapter = esAdapter; 46 | 47 | logger.info("TransFormTask init complete ......."); 48 | } 49 | 50 | 51 | @Override 52 | public void run() { 53 | logger.info("TransFormTask start ......."); 54 | running = true; 55 | while (running) { 56 | try { 57 | booleanMutex.get(); 58 | } catch (InterruptedException e) { 59 | logger.error("TransFormTask task has been interrupted ", e); 60 | running = false; 61 | break; 62 | } 63 | try { 64 | Message message = channel.takeMessage(); 65 | logger.info("Transform message =====> {}", message.getId()); 66 | TotoroTransForm transForm = getTotoroTransForm(message); 67 | Future future = executorService.submit(transForm); 68 | channel.putFuture(future); 69 | } catch (InterruptedException e) { 70 | logger.error("TransFormTask task has been interrupted ", e); 71 | running = false; 72 | break; 73 | } 74 | } 75 | } 76 | 77 | private TotoroTransForm getTotoroTransForm(Message message) { 78 | TotoroTransForm transForm = (TotoroTransForm) TotoroObjectPool.transForm(); 79 | transForm.setEsAdapter(esAdapter); 80 | transForm.setMessage(message); 81 | return transForm; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/com/totoro/canal/es/util/IPAddressUtil.java: -------------------------------------------------------------------------------- 1 | package com.totoro.canal.es.util; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | 5 | /** 6 | * 说明 .
7 | *

8 | *

9 | * Copyright: Copyright (c) 2017/12/04 下午12:46 10 | *

11 | * Company: xxx 12 | *

13 | * 14 | * @author zhongcheng_m@yeah.net 15 | * @version 1.0.0 16 | */ 17 | public class IPAddressUtil extends sun.net.util.IPAddressUtil { 18 | public static boolean isAddress(String address) { 19 | if (!StringUtils.isEmpty(address)) { 20 | String[] hostPort = address.trim().split(":"); 21 | if (hostPort.length == 2) { 22 | String ip = hostPort[0]; 23 | String portStr = hostPort[1]; 24 | if (sun.net.util.IPAddressUtil.isIPv4LiteralAddress(ip) && StringUtils.isNumeric(portStr)) { 25 | Integer port = Integer.valueOf(portStr); 26 | return 0 < port && port < 65535; 27 | } 28 | } 29 | } 30 | return false; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/resources/canal-es.properties: -------------------------------------------------------------------------------- 1 | # ----------------------- canal 相关配置 ----------------------------- 2 | #canal的实例名字 3 | totoro.canal.destination=totoro 4 | #cananl 服务端的模式，单机：single ，集群：cluster/ 5 | totoro.canal.mode=sign 6 | #canal 地址,包括端口号 7 | totoro.canal.address=127.0.0.1:11111 8 | #过滤表达式 9 | totoro.canal.filter.patten= 10 | #如果canal模式是集群的话，则需要填写zk地址 11 | totoro.canal.zk.address= 12 | totoro.canal.username= 13 | totoro.canal.password= 14 | 15 | #此项必须配置，不配置会导致 totoro 启动不了, totoro 只会处理在此配置的表 16 | #没有在此配置的表，totoro 将会忽略，不会进行同步，格式 "database.table.id" 多个使用 ","分割 17 | #database代表数据库，table 代表 数据库中的表，id代表 table中的 id 18 | #totoro 会默认将 database 作为 es中的index，table作为es中的type ，使用db中的id作为es的id 19 | totoro.canal.table.accept=demo.cc.id 20 | 21 | 22 | # ----------------------- elasticsearch 相关配置 ----------------------------- 23 | totoro.es.address=127.0.0.1:9300 24 | totoro.es.cluster.name=my-elasticsearch 25 | totoro.es.username= 26 | totoro.es.password= 27 | # ----------------------- totoro 相关配置 ---------------------------- 28 | #处理信息转换的线程数量 默认 3个 , 不要配置太大，2-4 之间吧，取决于业务情况，太大并不会增加性能，反而会增加上下文切换的开销 29 | totoro.cannal.trans.thread.nums=3 30 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | [%d{yyyy-MM-dd HH:mm:ss.SSS}] [%level] [%thread] [%logger{36}] --- %msg%n 6 | 7 | 8 | 9 | ../logs/totoro.log 10 | 11 | 12 | ../logs/totoro.log.%d{yyyy-MM-dd} 13 | 300 14 | 15 | 16 | [%d{yyyy-MM-dd HH:mm:ss.SSS}] [%level] [%thread] [%logger{36}] --- %msg%n 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/main/resources/totoro-logo.logo: -------------------------------------------------------------------------------- 1 | 2 | [ _____ _ 3 | |_ _|___ | |_ ___ _ __ ___ 4 | | | / _ \ | __|/ _ \ | '__|/ _ \ 5 | | || (_) || |_| (_) || | | (_) | 6 | |_| \___/ \__|\___/ |_| \___/ 7 | [Totoro 0.0.1,Build 2017/12/20,Author:zhongcheng_m@yeah.net] 8 | 9 | -------------------------------------------------------------------------------- /src/test/java/test/syn/Lock.java: -------------------------------------------------------------------------------- 1 | package test.syn; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.concurrent.ExecutionException; 6 | import java.util.concurrent.ExecutorService; 7 | import java.util.concurrent.Executors; 8 | import java.util.concurrent.locks.LockSupport; 9 | 10 | /** 11 | * 说明 .
12 | *

13 | *

14 | * Copyright: Copyright (c) 2017/12/04 下午2:19 15 | *

16 | * Company: xxx 17 | *

18 | * 19 | * @author zhongcheng_m@yeah.net 20 | * @version 1.0.0 21 | */ 22 | public class Lock { 23 | 24 | public static void get() { 25 | LockSupport.park(); 26 | } 27 | 28 | 29 | public static void main(String[] args) throws ExecutionException, InterruptedException { 30 | 31 | ExecutorService executorService = Executors.newFixedThreadPool(2); 32 | 33 | 34 | TestFuture testFuture = new TestFuture(() -> { 35 | Thread.sleep(4000); 36 | return "zhong"; 37 | }); 38 | 39 | executorService.submit(testFuture); 40 | 41 | System.out.printf("-------"); 42 | 43 | System.out.printf(testFuture.get()); 44 | 45 | System.out.printf("asdasda"); 46 | 47 | } 48 | 49 | 50 | public static class Student { 51 | 52 | 53 | public Student(String name, int age) { 54 | this.name = name; 55 | this.age = age; 56 | } 57 | 58 | private String name; 59 | 60 | private int age; 61 | 62 | public String getName() { 63 | return name; 64 | } 65 | 66 | public Student setName(String name) { 67 | this.name = name; 68 | return this; 69 | } 70 | 71 | public int getAge() { 72 | return age; 73 | } 74 | 75 | public Student setAge(int age) { 76 | this.age = age; 77 | return this; 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/test/java/test/syn/LockTest.java: -------------------------------------------------------------------------------- 1 | package test.syn; 2 | 3 | import com.alibaba.otter.canal.client.CanalConnector; 4 | import com.alibaba.otter.canal.client.CanalConnectors; 5 | import com.alibaba.otter.canal.protocol.Message; 6 | 7 | import java.net.InetSocketAddress; 8 | import java.util.concurrent.locks.LockSupport; 9 | 10 | /** 11 | * 说明 .
12 | *

13 | *

14 | * Copyright: Copyright (c) 2017/12/04 下午2:13 15 | *

16 | * Company: xxx 17 | *

18 | * 19 | * @author zhongcheng_m@yeah.net 20 | * @version 1.0.0 21 | */ 22 | public class LockTest { 23 | 24 | static boolean running = true; 25 | 26 | private static final int maxEmptyTimes = 10; 27 | 28 | public static void main(String[] args) throws InterruptedException { 29 | 30 | CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress("127.0.0.1", 11111), 31 | "totoro", 32 | "", 33 | ""); 34 | 35 | connector.connect(); 36 | connector.subscribe(); 37 | int emptyTimes = 0; 38 | 39 | while (running) { 40 | Message message = connector.getWithoutAck(5 * 1024); 41 | 42 | if (message == null || message.getId() == -1L) { 43 | applyWait(emptyTimes++); 44 | } else { 45 | //logger.info(message.toString()); 46 | long messageId = message.getId(); 47 | System.out.println("消息id：" + messageId); 48 | Thread.sleep(1000); 49 | 50 | connector.rollback(); 51 | } 52 | 53 | 54 | } 55 | 56 | } 57 | 58 | private static void applyWait(int emptyTimes) throws InterruptedException { 59 | int newEmptyTimes = emptyTimes > maxEmptyTimes ? maxEmptyTimes : emptyTimes; 60 | if (emptyTimes >= 3) { 61 | Thread.yield(); 62 | LockSupport.parkNanos(10000 * 1000L * newEmptyTimes); 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/test/java/test/syn/TestFuture.java: -------------------------------------------------------------------------------- 1 | package test.syn; 2 | 3 | import java.util.concurrent.Callable; 4 | import java.util.concurrent.FutureTask; 5 | 6 | /** 7 | * 说明 .
8 | *

9 | *

10 | * Copyright: Copyright (c) 2017/12/22 下午2:45 11 | *

12 | * Company: xxx 13 | *

14 | * 15 | * @author zhongcheng_m@yeah.net 16 | * @version 1.0.0 17 | */ 18 | public class TestFuture extends FutureTask { 19 | 20 | 21 | public TestFuture(Callable callable) { 22 | super(callable); 23 | } 24 | 25 | 26 | } 27 | --------------------------------------------------------------------------------