├── README.md ├── bin ├── SysConfig.properties ├── start.sh └── syncClient.jar └── src ├── META-INF └── MANIFEST.MF ├── SysConfig.properties ├── lib ├── canal.client-1.0.24.jar ├── canal.common-1.0.24.jar ├── canal.protocol-1.0.24.jar ├── commons-beanutils-1.8.2.jar ├── commons-codec-1.9.jar ├── commons-collections-3.2.jar ├── commons-io-2.4.jar ├── commons-lang-2.6.jar ├── commons-lang3-3.7.jar ├── commons-logging-1.0.4.jar ├── commons-pool-1.5.6.jar ├── commons-pool2-2.5.0.jar ├── elasticsearch-rest-client-6.3.2.jar ├── fastjson-1.2.28.jar ├── httpasyncclient-4.1.2.jar ├── httpclient-4.5.2.jar ├── httpcore-4.4.5.jar ├── httpcore-nio-4.4.5.jar ├── jedis-2.9.0.jar ├── kafka-clients-1.0.0.jar ├── protobuf-java-2.6.1.jar ├── slf4j-api-1.7.12.jar ├── ssdb.jar ├── xmemcached-2.4.5.jar ├── zkclient-0.10.jar └── zookeeper-3.4.5.jar └── src └── com └── sync ├── common ├── CanalData.java ├── EsApi.java ├── GetProperties.java ├── HttpClient.java ├── HttpmqApi.java ├── MemApi.java ├── ReadProperties.java ├── RedisApi.java ├── SsdbApi.java ├── TargetData.java ├── Tool.java └── WriteLog.java └── process ├── Cache.java ├── ElasticSearch.java ├── Httpmq.java ├── Kafka.java ├── Prop.java ├── Redis.java ├── Ssdb.java └── task.java /README.md: -------------------------------------------------------------------------------- 1 | ### **syncClient** 2 | 3 | >   syncClient，数据实时同步中间件（同步mysql到kafka、redis、elasticsearch、httpmq）！ 4 | 5 | 本项目使用canal，将mysql的表数据实时同步到kafka、redis、elasticsearch、httpmq；   6 | 7 | 基本原理： 8 | canal解析binlog的数据，由syncClient订阅，然后实时推送到kafka或者redis、elasticsearch、httpmq、ssdb；如果kafka、redis、es、httpmq服务异常，syncClient会回滚操作；canal、kafka、redis、es、httpmq的异常退出，都不会影响数据的传输； 9 | 10 | 11 | --- 12 | 13 | **目录：** 14 | bin：已编译二进制项目，可以直接使用； 15 | src：源代码； 16 | 17 | --- 18 | 19 | **配置说明：** 20 | 21 | #common 22 | system.debug=1 # 是否开始调试：1未开启，0为关闭（线上运行请关闭） 23 | 24 | #canal server 25 | canal.ip=127.0.0.1 # canal 服务端 ip; 26 | canal.port=11111 # canal 服务端 端口：默认11111; 27 | canal.destination=one # canal 服务端项目（destinations），多个用逗号分隔，如：redis,kafka; 28 | canal.username= # canal 用户名：默认为空; 29 | canal.password= # canal 密码：默认为空; 30 | 31 | #redis plugin 32 | redis.target_type=redis # 同步插件类型 kafka or redis、elasticsearch、httpmq; 33 | redis.target_ip= # redis服务端 ip; 34 | redis.target_port= # redis端口：默认6379; 35 | redis.target_deep= # 同步到redis的队列名称规则; 36 | redis.target_filter_api= # rest api地址，配置后会根据api返回的数据过滤同步数据; 37 | 38 | #kafka plugin 39 | kafka.target_type=kafka # 同步插件类型 kafka; 40 | kafka.target_ip= # kafka服务端 ip; 41 | kafka.target_port= # kafka端口：默认9092; 42 | kafka.target_deep= # 同步到kafka的集合名称规则; 43 | kafka.target_filter_api= # rest api地址，配置后会根据api返回的数据过滤同步数据; 44 | 45 | #elasticsearch plugin 46 | es.target_type=elasticsearch # 同步插件类型elasticsearch; 47 | es.target_ip=10.5.3.66 # es服务端 ip; 48 | es.target_port= # es端口：默认9200; 49 | es.target_deep= # 同步到es的index名称规则; 50 | es.target_filter_api= # rest api地址，配置后会根据api返回的数据过滤同步数据; 51 | 52 | #httpmq plugin 53 | httpmq.target_type=httpmq # 同步插件类型 httpmq; 54 | httpmq.target_ip=10.5.3.66 # httpmq服务端 ip; 55 | httpmq.target_port=1218 # httpmq端口：默认 1218 56 | httpmq.target_deep= # 同步到httpmq的队列名称规则; 57 | httpmq.target_filter_api= # rest api地址，配置后会根据api返回的数据过滤同步数据; 58 | 59 | #cache plugin 60 | cache.target_type=cache # 缓存同步插件 61 | cache.target_plugin=memcached # 缓存同步类型：暂支持redis、memcached缓存服务器; 62 | cache.target_ip=127.0.0.1 # 缓存服务器ip; 63 | cache.target_port=11211 # 缓存服务器端口; 64 | cache.target_filter_api= # rest api地址，配置后会根据api返回的数据过滤同步数据; 65 | cache.target_version_sign= # 缓存key前缀 66 | 67 | #target_deep参数影响topic规则，默认值1： 68 | 1、sync_{项目名}_{db}_{table}; 69 | 2、sync_{项目名}_{db}; 70 | 3、sync_{项目名}; 71 | 4、sync_{db}_{table}; 72 | 73 | --- 74 | 75 | **使用场景(基于日志增量订阅&消费支持的业务)：** 76 | 77 | 数据库镜像 78 | 数据库实时备份 79 | 多级索引 (分库索引) 80 | search build 81 | 业务cache刷新 82 | 数据变化等重要业务消息 83 | 84 | **Kafka：** 85 | 86 | Topic规则：对应配置项目target_deep指定的规则，比如：target_deep=4，数据库的每个表有单独的topic，如数据库admin的user表，对应的kafka主题名为：sync_admin_user   87 | Topic数据字段： 88 | 89 | 插入数据同步格式： 90 | { 91 | "head": { 92 | "binlog_pos": 53036, 93 | "type": "INSERT", 94 | "binlog_file": "mysql-bin.000173", 95 | "db": "sdsw", 96 | "table": "sys_log" 97 | }, 98 | "after": [ 99 | { 100 | "log_id": "1", 101 | }, 102 | { 103 | "log_ip": "27.17.47.100", 104 | }, 105 | { 106 | "log_addtime": "1494204717", 107 | } 108 | ] 109 | } 110 | 111 | 修改数据同步格式： 112 | { 113 | "head": { 114 | "binlog_pos": 53036, 115 | "type": "UPDATE", 116 | "binlog_file": "mysql-bin.000173", 117 | "db": "sdsw", 118 | "table": "sys_log" 119 | }, 120 | "before": [ 121 | { 122 | "log_id": "1", 123 | }, 124 | { 125 | "log_ip": "27.17.47.100", 126 | }, 127 | { 128 | "log_addtime": "1494204717", 129 | } 130 | ], 131 | "after": [ 132 | { 133 | "log_id": "1", 134 | }, 135 | { 136 | "log_ip": "27.17.47.1", 137 | }, 138 | { 139 | "log_addtime": "1494204717", 140 | } 141 | ] 142 | } 143 | 144 | 删除数据同步格式： 145 | { 146 | "head": { 147 | "binlog_pos": 53036, 148 | "type": "DELETE", 149 | "binlog_file": "mysql-bin.000173", 150 | "db": "sdsw", 151 | "table": "sys_log" 152 | }, 153 | "before": [ 154 | { 155 | "log_id": "1", 156 | }, 157 | { 158 | "log_ip": "27.17.47.1", 159 | }, 160 | { 161 | "log_addtime": "1494204717", 162 | } 163 | ] 164 | } 165 | 166 | head.type 类型：INSERT（插入）、UPDATE（修改）、DELETE（删除）； 167 | 168 | head.db 数据库； 169 | 170 | head.table 数据库表； 171 | 172 | head.binlog_pos 日志位置； 173 | 174 | head.binlog_file 日志文件；   175 | 176 | before： UPDATE（修改前）、DELETE（删除前）的数据；   177 | 178 | after： INSERT（插入后）、UPDATE（修改后）的数据； 179 | 180 | 181 | **Redis：** 182 | 183 | List规则：对应配置项目target_deep指定的规则，比如：target_deep=4，数据库的每个表有单独的list，如数据库admin的user表，对应的redis list名为：sync_admin_user   184 | 185 | **Elasticsearch** 186 | 187 | 规则：数据库的每个表有单独的Elasticsearch index，如数据库admin的user表，对应的es index名为：sync_admin_user, index type 为default; 188 | 189 | Elasticsearch同步数据的head中有id字段； 190 | 191 | Mysql 同步到 Elasticsearch注意事项： 192 | 193 | 1、表需要有一个唯一id主键； 194 | 2、表时间字段datetime会转为es的时间字段，其他字段对应es的文本类型； 195 | 3、主键、时间字段禁止修改，其他字段尽量提前规划好； 196 | 197 | **Httpmq：** 198 | 199 | List规则：对应配置项目target_deep指定的规则，比如：target_deep=4，数据库的每个表有单独的list，如数据库admin的user表，对应的redis list名为：sync_admin_user   200 | 201 | 202 | **Cache：** 203 | 204 | 缓存同步插件：原理是根据数据库变更同步更新表及字段的版本号，业务sdk根据版本号变化判断是否需要更新数据。同步开发了缓存配置管理中心、缓存版本调用sdk（未开源）； 205 | 206 | 207 | Flag Counter 208 | 209 | -------------------------------------------------------------------------------- /bin/SysConfig.properties: -------------------------------------------------------------------------------- 1 | #common 2 | system.debug=1 3 | 4 | #canal server 5 | canal.ip=127.0.0.1 6 | canal.port=11111 7 | canal.destination=httpmq 8 | canal.username= 9 | canal.password= 10 | 11 | #redis plugin 12 | redis.target_type=redis 13 | redis.target_ip=127.0.0.1 14 | redis.target_port=6397 15 | redis.target_filter_api= 16 | redis.target_deep=2 17 | 18 | #elasticsearch plugin 19 | es.target_type=elasticsearch 20 | es.target_ip=127.0.0.1 21 | es.target_port=9200 22 | es.target_filter_api= 23 | es.target_deep=2 24 | 25 | #httpmq plugin 26 | httpmq.target_type=httpmq 27 | httpmq.target_ip=127.0.0.1 28 | httpmq.target_port=1218 29 | httpmq.target_filter_api= 30 | httpmq.target_deep=2 31 | 32 | #kafka plugin 33 | kafka.target_type=kafka 34 | kafka.target_ip=127.0.0.1 35 | kafka.target_port=9092 36 | kafka.target_filter_api= 37 | kafka.target_deep=2 38 | 39 | #cache plugin 40 | cache.target_type=cache 41 | cache.target_plugin=memcached 42 | cache.target_ip=127.0.0.1 43 | cache.target_port=11211 44 | cache.target_filter_api= 45 | cache.target_version_sign=database: -------------------------------------------------------------------------------- /bin/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | java -jar syncClient.jar & -------------------------------------------------------------------------------- /bin/syncClient.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/bin/syncClient.jar -------------------------------------------------------------------------------- /src/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Main-Class: com.sync.process.task 3 | -------------------------------------------------------------------------------- /src/SysConfig.properties: -------------------------------------------------------------------------------- 1 | #common 2 | system.debug=1 3 | 4 | #canal server 5 | canal.ip=127.0.0.1 6 | canal.port=11111 7 | canal.destination=cache 8 | canal.username= 9 | canal.password= 10 | 11 | #redis plugin 12 | redis.target_type=redis 13 | redis.target_ip=127.0.0.1 14 | redis.target_port=6397 15 | redis.target_deep=3 16 | 17 | #elasticsearch plugin 18 | es.target_type=elasticsearch 19 | es.target_ip=127.0.0.1 20 | es.target_port=9200 21 | es.target_deep=3 22 | 23 | #httpmq plugin 24 | httpmq.target_type=httpmq 25 | httpmq.target_ip=127.0.0.1 26 | httpmq.target_port=1218 27 | httpmq.target_deep=3 28 | 29 | #kafka plugin 30 | httpmq.target_type=kafka 31 | httpmq.target_ip=127.0.0.1 32 | httpmq.target_port=9092 33 | httpmq.target_deep=3 34 | 35 | #cache plugin 36 | cache.target_type=cache 37 | cache.target_plugin=memcached 38 | cache.target_ip=127.0.0.1 39 | cache.target_port=11211 40 | cache.target_field_filter= 41 | cache.target_version_sign=database: -------------------------------------------------------------------------------- /src/lib/canal.client-1.0.24.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/canal.client-1.0.24.jar -------------------------------------------------------------------------------- /src/lib/canal.common-1.0.24.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/canal.common-1.0.24.jar -------------------------------------------------------------------------------- /src/lib/canal.protocol-1.0.24.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/canal.protocol-1.0.24.jar -------------------------------------------------------------------------------- /src/lib/commons-beanutils-1.8.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/commons-beanutils-1.8.2.jar -------------------------------------------------------------------------------- /src/lib/commons-codec-1.9.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/commons-codec-1.9.jar -------------------------------------------------------------------------------- /src/lib/commons-collections-3.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/commons-collections-3.2.jar -------------------------------------------------------------------------------- /src/lib/commons-io-2.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/commons-io-2.4.jar -------------------------------------------------------------------------------- /src/lib/commons-lang-2.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/commons-lang-2.6.jar -------------------------------------------------------------------------------- /src/lib/commons-lang3-3.7.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/commons-lang3-3.7.jar -------------------------------------------------------------------------------- /src/lib/commons-logging-1.0.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/commons-logging-1.0.4.jar -------------------------------------------------------------------------------- /src/lib/commons-pool-1.5.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/commons-pool-1.5.6.jar -------------------------------------------------------------------------------- /src/lib/commons-pool2-2.5.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/commons-pool2-2.5.0.jar -------------------------------------------------------------------------------- /src/lib/elasticsearch-rest-client-6.3.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/elasticsearch-rest-client-6.3.2.jar -------------------------------------------------------------------------------- /src/lib/fastjson-1.2.28.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/fastjson-1.2.28.jar -------------------------------------------------------------------------------- /src/lib/httpasyncclient-4.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/httpasyncclient-4.1.2.jar -------------------------------------------------------------------------------- /src/lib/httpclient-4.5.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/httpclient-4.5.2.jar -------------------------------------------------------------------------------- /src/lib/httpcore-4.4.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/httpcore-4.4.5.jar -------------------------------------------------------------------------------- /src/lib/httpcore-nio-4.4.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/httpcore-nio-4.4.5.jar -------------------------------------------------------------------------------- /src/lib/jedis-2.9.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/jedis-2.9.0.jar -------------------------------------------------------------------------------- /src/lib/kafka-clients-1.0.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/kafka-clients-1.0.0.jar -------------------------------------------------------------------------------- /src/lib/protobuf-java-2.6.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/protobuf-java-2.6.1.jar -------------------------------------------------------------------------------- /src/lib/slf4j-api-1.7.12.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/slf4j-api-1.7.12.jar -------------------------------------------------------------------------------- /src/lib/ssdb.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/ssdb.jar -------------------------------------------------------------------------------- /src/lib/xmemcached-2.4.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/xmemcached-2.4.5.jar -------------------------------------------------------------------------------- /src/lib/zkclient-0.10.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/zkclient-0.10.jar -------------------------------------------------------------------------------- /src/lib/zookeeper-3.4.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/lib/zookeeper-3.4.5.jar -------------------------------------------------------------------------------- /src/src/com/sync/common/CanalData.java: -------------------------------------------------------------------------------- 1 | package com.sync.common; 2 | 3 | /** 4 | * CanalData 5 | * 6 | * @author sasou web:http://www.php-gene.com/ 7 | * @version 1.0.0 8 | */ 9 | public final class CanalData { 10 | public String ip = "127.0.0.1"; 11 | public int port = 11111; 12 | public String[] destination = null; 13 | public String username = ""; 14 | public String password = ""; 15 | 16 | /** 17 | * @return the ip 18 | */ 19 | public String getIp() { 20 | return ip; 21 | } 22 | 23 | /** 24 | * @param ip 25 | * the ip to set 26 | */ 27 | public void setIp(String ip) { 28 | this.ip = ip; 29 | } 30 | 31 | /** 32 | * @return the port 33 | */ 34 | public int getPort() { 35 | return port; 36 | } 37 | 38 | /** 39 | * @param port 40 | * the port to set 41 | */ 42 | public void setPort(int port) { 43 | this.port = port; 44 | } 45 | 46 | /** 47 | * @return the destination 48 | */ 49 | public String[] getDestination() { 50 | return destination; 51 | } 52 | 53 | /** 54 | * @param destination 55 | * the destination to set 56 | */ 57 | public void setDestination(String destination) { 58 | this.destination = destination.split(","); 59 | } 60 | 61 | /** 62 | * @return the username 63 | */ 64 | public String getUsername() { 65 | return username; 66 | } 67 | 68 | /** 69 | * @param username 70 | * the username to set 71 | */ 72 | public void setUsername(String username) { 73 | this.username = username; 74 | } 75 | 76 | /** 77 | * @return the password 78 | */ 79 | public String getPassword() { 80 | return password; 81 | } 82 | 83 | /** 84 | * @param password 85 | * the password to set 86 | */ 87 | public void setPassword(String password) { 88 | this.password = password; 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/src/com/sync/common/EsApi.java: -------------------------------------------------------------------------------- 1 | package com.sync.common; 2 | 3 | import java.io.IOException; 4 | import java.util.Collections; 5 | import java.util.Map; 6 | 7 | import org.apache.http.HttpEntity; 8 | import org.apache.http.HttpHost; 9 | import org.apache.http.entity.ContentType; 10 | import org.apache.http.nio.entity.NStringEntity; 11 | import org.apache.http.util.EntityUtils; 12 | import org.elasticsearch.client.*; 13 | 14 | import com.alibaba.fastjson.JSON; 15 | 16 | 17 | /** 18 | * EsApi 19 | * 20 | * @author sasou web:http://www.php-gene.com/ 21 | * @version 1.0.0 22 | */ 23 | public final class EsApi { 24 | 25 | private String canal_destination = null; 26 | private static RestClient rs = null; 27 | 28 | public EsApi(String name) throws Exception { 29 | canal_destination = name; 30 | try { 31 | rs = RestClient.builder(new HttpHost(GetProperties.target.get(canal_destination).ip, GetProperties.target.get(canal_destination).port, "http")).build(); 32 | } catch (Exception e) { 33 | rs.close(); 34 | throw new Exception("elasticsearch link fail", e); 35 | } 36 | } 37 | 38 | /** 39 | * @param index 40 | * @param content 41 | * @throws Exception 42 | */ 43 | public boolean sync(String index, String content) throws Exception { 44 | Map data = jsonToMap(content); 45 | Map head = jsonToMap((String) data.get("head").toString()); 46 | String type = (String) head.get("type").toString(); 47 | String id = (String) head.get("id").toString(); 48 | String text = ""; 49 | switch(type) { 50 | case "INSERT": 51 | text = (String) data.get("after").toString(); 52 | if (!"".equals(text)) { 53 | try { 54 | return insert(index, "default", id, text); 55 | } catch (Exception e) { 56 | throw new Exception("elasticsearch insert fail", e); 57 | } 58 | } 59 | break; 60 | case "UPDATE": 61 | text = (String) data.get("after").toString(); 62 | if (!"".equals(id)) { 63 | try { 64 | return update(index, "default", id, text); 65 | } catch (Exception e) { 66 | throw new Exception("elasticsearch update fail", e); 67 | } 68 | } 69 | break; 70 | case "DELETE": 71 | if (!"".equals(id)) { 72 | try { 73 | return delete(index, "default", id); 74 | } catch (Exception e) { 75 | 76 | } 77 | } 78 | break; 79 | } 80 | return true; 81 | } 82 | 83 | /** 84 | * json string to map 85 | * @param jsonObj 86 | * @return 87 | */ 88 | public static Map jsonToMap(String jsonObj) { 89 | @SuppressWarnings("unchecked") 90 | Map maps = (Map) JSON.parse((String) jsonObj); 91 | return maps; 92 | } 93 | 94 | /** 95 | * @param index 96 | * @param type 97 | * @param id 98 | * @param content 99 | * @throws Exception 100 | */ 101 | public boolean insert(String index, String type, String id, String content) throws Exception { 102 | Map params = Collections.emptyMap(); 103 | HttpEntity entity = new NStringEntity(content, ContentType.APPLICATION_JSON); 104 | Response response = rs.performRequest("PUT", "/" + index + "/" + type + "/" + id, params, entity); 105 | String ret = (String) EntityUtils.toString(response.getEntity()); 106 | return ret.contains("created") || ret.contains("updated"); 107 | } 108 | 109 | /** 110 | * @param index 111 | * @param type 112 | * @param id 113 | * @param content 114 | * @throws Exception 115 | */ 116 | public boolean update(String index, String type, String id, String content) throws Exception { 117 | Map params = Collections.emptyMap(); 118 | HttpEntity entity = new NStringEntity(content, ContentType.APPLICATION_JSON); 119 | Response response = rs.performRequest("PUT", "/" + index + "/" + type + "/" + id, params, entity); 120 | String ret = (String) EntityUtils.toString(response.getEntity()); 121 | return ret.contains("created") || ret.contains("updated"); 122 | } 123 | 124 | /** 125 | * @param index 126 | * @param type 127 | * @param id 128 | * @throws Exception 129 | */ 130 | public boolean delete(String index, String type, String id) throws Exception { 131 | Map params = Collections.emptyMap(); 132 | Response response = rs.performRequest("DELETE", "/"+ index + "/" + type + "/" + id, params); 133 | String ret = (String) EntityUtils.toString(response.getEntity()); 134 | return ret.contains("not_found") || ret.contains("deleted"); 135 | } 136 | 137 | /** 138 | * @param index 139 | * @param type 140 | * @param id 141 | * @throws IOException 142 | */ 143 | public String get(String index, String type, String id) throws Exception { 144 | try { 145 | Response response = rs.performRequest("GET", "/" + index + "/" + type + "/" + id, Collections.singletonMap("pretty", "true")); 146 | return EntityUtils.toString(response.getEntity()); 147 | } catch (Exception e) { 148 | e.printStackTrace(); 149 | } 150 | return ""; 151 | } 152 | 153 | /** 154 | * @param index 155 | * @throws IOException 156 | */ 157 | public boolean index(String index) throws Exception { 158 | try { 159 | Response response = rs.performRequest("HEAD", index, Collections.emptyMap()); 160 | return response.getStatusLine().getReasonPhrase().equals("OK"); 161 | } catch (Exception e) { 162 | 163 | } 164 | return false; 165 | } 166 | 167 | /** 168 | * @param index 169 | * @throws IOException 170 | */ 171 | public String getMappings(String index) throws Exception { 172 | try { 173 | Response response = rs.performRequest("GET", "/" + index + "/_mappings", Collections.emptyMap()); 174 | return EntityUtils.toString(response.getEntity()); 175 | } catch (Exception e) { 176 | 177 | } 178 | return ""; 179 | } 180 | 181 | /** 182 | * @param index 183 | * @throws IOException 184 | */ 185 | public String setMappings(String index) throws Exception { 186 | try { 187 | HttpEntity entity = new NStringEntity("{\"mappings\":{\"default\" :{\"properties\":{\"@timestamp\":{\"type\" : \"date\"}}}}}", ContentType.APPLICATION_JSON); 188 | Response response = rs.performRequest("PUT", "/" + index, Collections.emptyMap(), entity); 189 | return EntityUtils.toString(response.getEntity()); 190 | } catch (Exception e) { 191 | e.printStackTrace(); 192 | } 193 | return ""; 194 | } 195 | 196 | } 197 | -------------------------------------------------------------------------------- /src/src/com/sync/common/GetProperties.java: -------------------------------------------------------------------------------- 1 | package com.sync.common; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.Properties; 6 | 7 | import com.alibaba.fastjson.JSONObject; 8 | import com.sync.common.ReadProperties; 9 | 10 | /** 11 | * GetProperties 12 | * 13 | * @author sasou web:http://www.php-gene.com/ 14 | * @version 1.0.0 15 | */ 16 | 17 | public class GetProperties { 18 | // debug 19 | public static int system_debug = 0; 20 | // canal 21 | public static CanalData canal = new CanalData(); 22 | 23 | // target 24 | public static Map target = new HashMap(); 25 | 26 | @SuppressWarnings("unchecked") 27 | public static boolean update() { 28 | // read config 29 | Properties prop = ReadProperties.readProperties(); 30 | String tmp = ""; 31 | 32 | // debug 33 | tmp = prop.getProperty("system.debug", ""); 34 | if (!"".equals(tmp)) { 35 | system_debug = Integer.parseInt(tmp); 36 | } 37 | 38 | // canal 39 | tmp = prop.getProperty("canal.ip", ""); 40 | if (!"".equals(tmp)) { 41 | canal.setIp(tmp); 42 | } 43 | 44 | tmp = prop.getProperty("canal.port", ""); 45 | if (!"".equals(tmp)) { 46 | canal.setPort(Integer.parseInt(tmp)); 47 | } 48 | canal.setDestination(prop.getProperty("canal.destination", "")); 49 | canal.setUsername(prop.getProperty("canal.username", "")); 50 | canal.setPassword(prop.getProperty("canal.password", "")); 51 | 52 | // target 53 | if (canal.destination != null) { 54 | TargetData target_tmp = null; 55 | int num = canal.destination.length; 56 | if (num > 0) { 57 | for (int i = 0; i < num; i++) { 58 | if (target.containsKey(canal.destination[i])) { 59 | target_tmp = target.get(canal.destination[i]); 60 | } else { 61 | target_tmp = new TargetData(); 62 | } 63 | 64 | tmp = prop.getProperty(canal.destination[i] + ".target_type", ""); 65 | if (!"".equals(tmp)) { 66 | target_tmp.setType(tmp); 67 | } 68 | tmp = prop.getProperty(canal.destination[i] + ".target_ip", ""); 69 | if (!"".equals(tmp)) { 70 | target_tmp.setIp(tmp); 71 | } 72 | if ("kafka".equals(target_tmp.type)) { 73 | target_tmp.setPort(9092); 74 | } 75 | if ("redis".equals(target_tmp.type)) { 76 | target_tmp.setPort(6379); 77 | } 78 | if ("elasticsearch".equals(target_tmp.type)) { 79 | target_tmp.setPort(9200); 80 | } 81 | if ("httpmq".equals(target_tmp.type)) { 82 | target_tmp.setPort(1218); 83 | } 84 | tmp = prop.getProperty(canal.destination[i] + ".target_port", ""); 85 | if (!"".equals(tmp)) { 86 | target_tmp.setPort(Integer.parseInt(tmp)); 87 | } 88 | tmp = prop.getProperty(canal.destination[i] + ".target_deep", ""); 89 | if (!"".equals(tmp)) { 90 | target_tmp.setDeep(Integer.parseInt(tmp)); 91 | } 92 | tmp = prop.getProperty(canal.destination[i] + ".target_filter_api", ""); 93 | target_tmp.setFilter(tmp); 94 | if (!"".equals(tmp)) { 95 | try { 96 | String json = HttpClient.sendGet(tmp, ""); 97 | @SuppressWarnings("rawtypes") 98 | Map filterMap = new HashMap(); 99 | syncCache(json, filterMap); 100 | target_tmp.setFilterMap(filterMap); 101 | } catch (Exception e) { 102 | System.out.println(e); 103 | } 104 | } 105 | 106 | if ("cache".equals(target_tmp.type)) { 107 | target_tmp.setPlugin(prop.getProperty(canal.destination[i] + ".target_plugin", "")); 108 | target_tmp.setSign(prop.getProperty(canal.destination[i] + ".target_version_sign", "")); 109 | } 110 | target.put(canal.destination[i], target_tmp); 111 | } 112 | } 113 | } 114 | prop.clear(); 115 | return true; 116 | } 117 | 118 | public static String get(String key) { 119 | // read config 120 | Properties prop = ReadProperties.readProperties(); 121 | return prop.getProperty(key, ""); 122 | } 123 | 124 | 125 | @SuppressWarnings({ "rawtypes", "unchecked" }) 126 | public static void syncCache(String jsonStr, Map cacheMap) { 127 | JSONObject jsonobject = JSONObject.parseObject(jsonStr); 128 | JSONObject db = null; 129 | try { 130 | db = (JSONObject) jsonobject.get("data"); 131 | } catch (Exception e) { 132 | return; 133 | } 134 | 135 | for (String dbName : db.keySet()){ 136 | JSONObject table = (JSONObject) db.get(dbName); 137 | for (String tableName : table.keySet()){ 138 | String key = dbName + "." + tableName; 139 | Map tableMap = null; 140 | if (cacheMap.containsKey(key)) { 141 | tableMap = (Map) cacheMap.get(key); 142 | } else { 143 | tableMap = new HashMap(); 144 | cacheMap.put(key, tableMap); 145 | } 146 | JSONObject field = (JSONObject) table.get(tableName); 147 | for (String fieldName : field.keySet()){ 148 | if (!tableMap.containsKey(fieldName)) { 149 | tableMap.put(fieldName, ""); 150 | } 151 | } 152 | } 153 | } 154 | } 155 | 156 | } -------------------------------------------------------------------------------- /src/src/com/sync/common/HttpClient.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sasou/syncClient/cf0b975b07d82642d7e1c2a83b5509ad29feb2f1/src/src/com/sync/common/HttpClient.java -------------------------------------------------------------------------------- /src/src/com/sync/common/HttpmqApi.java: -------------------------------------------------------------------------------- 1 | package com.sync.common; 2 | 3 | public class HttpmqApi { 4 | 5 | private String canal_destination = null; 6 | private String url = null; 7 | 8 | public HttpmqApi(String name) { 9 | canal_destination = name; 10 | url = "http://" + GetProperties.target.get(canal_destination).ip + ":" + GetProperties.target.get(canal_destination).port + "?"; 11 | } 12 | 13 | /** 14 | * @param index 15 | * @param content 16 | * @throws Exception 17 | */ 18 | public boolean put(String type, String content) throws Exception { 19 | String ret = null; 20 | ret = HttpClient.sendPost(url + "charset=utf-8&opt=put&name=" + type, content); 21 | if (ret.indexOf("HTTPMQ_PUT_OK") != -1) { 22 | return true; 23 | } 24 | return false; 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/src/com/sync/common/MemApi.java: -------------------------------------------------------------------------------- 1 | package com.sync.common; 2 | 3 | import java.io.IOException; 4 | import java.util.concurrent.TimeoutException; 5 | 6 | import net.rubyeye.xmemcached.MemcachedClient; 7 | import net.rubyeye.xmemcached.MemcachedClientBuilder; 8 | import net.rubyeye.xmemcached.XMemcachedClientBuilder; 9 | import net.rubyeye.xmemcached.exception.MemcachedException; 10 | import net.rubyeye.xmemcached.utils.AddrUtil; 11 | 12 | /** 13 | * MemApi 14 | * 15 | * @author sasou web:http://www.php-gene.com/ 16 | * @version 1.0.0 17 | */ 18 | 19 | public class MemApi { 20 | 21 | /** 22 | * MemCachedClient 23 | */ 24 | protected static MemcachedClient memCachedClient = null; 25 | 26 | 27 | 28 | /** 29 | * MemApi 30 | */ 31 | public MemApi(String name) 32 | { 33 | if (memCachedClient == null) { 34 | MemcachedClientBuilder builder = new XMemcachedClientBuilder(AddrUtil.getAddresses(GetProperties.target.get(name).ip + ":" + GetProperties.target.get(name).port)); 35 | try { 36 | memCachedClient = builder.build(); 37 | builder.setConnectionPoolSize(5); 38 | memCachedClient.setEnableHeartBeat(false); 39 | } catch (IOException e) { 40 | 41 | } 42 | } 43 | } 44 | 45 | /** 46 | * @param key 47 | * @param value 48 | * @return boolean 49 | * @throws Exception 50 | */ 51 | public boolean set(String key, Object value) throws Exception { 52 | boolean blag = false; 53 | try { 54 | blag = memCachedClient.set(key, 0, value); 55 | } catch (MemcachedException e) { 56 | throw new Exception("MemcachedClient operation fail"); 57 | } catch (TimeoutException e) { 58 | throw new Exception("MemcachedClient operation timeout"); 59 | } catch (InterruptedException e) { 60 | // ignore 61 | } 62 | return blag; 63 | } 64 | 65 | /** 66 | * @param key 67 | * @param value 68 | * @param expiry 69 | * @return boolean 70 | * @throws Exception 71 | */ 72 | public boolean set(String key, Object value, int expiry) throws Exception{ 73 | boolean blag = false; 74 | try { 75 | blag = memCachedClient.set(key, expiry, value); 76 | } catch (MemcachedException e) { 77 | throw new Exception("MemcachedClient operation fail"); 78 | } catch (TimeoutException e) { 79 | throw new Exception("MemcachedClient operation timeout"); 80 | } catch (InterruptedException e) { 81 | // ignore 82 | } 83 | return blag; 84 | } 85 | 86 | /** 87 | * @param key 88 | * @param value 89 | * @return boolean 90 | * @throws Exception 91 | */ 92 | public boolean add(String key, Object value) throws Exception { 93 | boolean blag = false; 94 | try { 95 | if (get(key) != null) { 96 | return blag; 97 | } else { 98 | blag = memCachedClient.add(key, 0, value); 99 | } 100 | } catch (MemcachedException e) { 101 | throw new Exception("MemcachedClient operation fail"); 102 | } catch (TimeoutException e) { 103 | throw new Exception("MemcachedClient operation timeout"); 104 | } catch (InterruptedException e) { 105 | // ignore 106 | } 107 | return blag; 108 | } 109 | 110 | /** 111 | * @param key 112 | * @param value 113 | * @return boolean 114 | * @throws Exception 115 | */ 116 | public boolean replace(String key, Object value) throws Exception { 117 | boolean blag = false; 118 | try { 119 | blag = memCachedClient.replace(key, 0, value); 120 | } catch (MemcachedException e) { 121 | throw new Exception("MemcachedClient operation fail"); 122 | } catch (TimeoutException e) { 123 | throw new Exception("MemcachedClient operation timeout"); 124 | } catch (InterruptedException e) { 125 | // ignore 126 | } 127 | return blag; 128 | } 129 | 130 | /** 131 | * @param key 132 | * @param value 133 | * @param expiry 134 | * @return boolean 135 | * @throws Exception 136 | */ 137 | public boolean replace(String key, Object value, int expiry) throws Exception { 138 | boolean blag = false; 139 | try { 140 | blag = memCachedClient.replace(key, expiry, value); 141 | } catch (MemcachedException e) { 142 | throw new Exception("MemcachedClient operation fail"); 143 | } catch (TimeoutException e) { 144 | throw new Exception("MemcachedClient operation timeout"); 145 | } catch (InterruptedException e) { 146 | // ignore 147 | } 148 | return blag; 149 | } 150 | 151 | 152 | /** 153 | * 154 | * @param key 155 | * @return boolean 156 | * @throws Exception 157 | */ 158 | public String get(String key) throws Exception { 159 | String blag = ""; 160 | try { 161 | blag = memCachedClient.get(key).toString(); 162 | } catch (MemcachedException e) { 163 | throw new Exception("MemcachedClient operation fail"); 164 | } catch (TimeoutException e) { 165 | throw new Exception("MemcachedClient operation timeout"); 166 | } catch (InterruptedException e) { 167 | // ignore 168 | } 169 | return blag; 170 | } 171 | 172 | /** 173 | * incr 174 | * 175 | * @param key 176 | * @return boolean 177 | * @throws Exception 178 | */ 179 | public boolean incr(String key) throws Exception { 180 | boolean blag = false; 181 | try { 182 | long ret = memCachedClient.incr(key, 1, 1); 183 | if (ret > 0) { 184 | blag = true; 185 | } 186 | } catch (MemcachedException e) { 187 | throw new Exception("MemcachedClient operation fail"); 188 | } catch (TimeoutException e) { 189 | throw new Exception("MemcachedClient operation timeout"); 190 | } catch (InterruptedException e) { 191 | // ignore 192 | } 193 | return blag; 194 | } 195 | 196 | /** 197 | * @param key 198 | * @param value 199 | * @return boolean 200 | * @throws Exception 201 | */ 202 | public boolean delete(String key) throws Exception{ 203 | boolean blag = false; 204 | try { 205 | blag = memCachedClient.delete(key); 206 | } catch (MemcachedException e) { 207 | throw new Exception("MemcachedClient operation fail"); 208 | } catch (TimeoutException e) { 209 | throw new Exception("MemcachedClient operation timeout"); 210 | } catch (InterruptedException e) { 211 | // ignore 212 | } 213 | return blag; 214 | } 215 | 216 | } -------------------------------------------------------------------------------- /src/src/com/sync/common/ReadProperties.java: -------------------------------------------------------------------------------- 1 | package com.sync.common; 2 | 3 | import java.io.BufferedInputStream; 4 | import java.io.FileInputStream; 5 | import java.io.FileNotFoundException; 6 | import java.io.IOException; 7 | import java.io.InputStream; 8 | import java.util.Properties; 9 | 10 | /** 11 | * config file read 12 | * 13 | * @author sasou web:http://www.php-gene.com/ 14 | * @version 1.0.0 15 | */ 16 | public class ReadProperties { 17 | static final String propertiesFilename = "SysConfig.properties"; 18 | private static Properties props = new Properties(); 19 | 20 | public static Properties readProperties() { 21 | try { 22 | InputStream inputStream = new BufferedInputStream(new FileInputStream(System.getProperty("user.dir") + "/" + propertiesFilename)); 23 | try { 24 | props.load(inputStream); 25 | } catch (IOException e) { 26 | 27 | } 28 | } catch (FileNotFoundException e) { 29 | 30 | } 31 | return props; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/src/com/sync/common/RedisApi.java: -------------------------------------------------------------------------------- 1 | package com.sync.common; 2 | 3 | import java.util.List; 4 | import java.util.Set; 5 | 6 | import redis.clients.jedis.Jedis; 7 | import redis.clients.jedis.JedisPool; 8 | import redis.clients.jedis.JedisPoolConfig; 9 | import redis.clients.jedis.exceptions.JedisConnectionException; 10 | 11 | /** 12 | * RedisApi 13 | * 14 | * @author sasou web:http://www.php-gene.com/ 15 | * @version 1.0.0 16 | */ 17 | 18 | public class RedisApi { 19 | private String canal_destination = null; 20 | private static JedisPool pool = null; 21 | 22 | public RedisApi(String name) { 23 | canal_destination = name; 24 | if (pool == null) { 25 | JedisPoolConfig config = new JedisPoolConfig(); 26 | config.setBlockWhenExhausted(true); 27 | config.setEvictionPolicyClassName("org.apache.commons.pool2.impl.DefaultEvictionPolicy"); 28 | config.setJmxEnabled(true); 29 | config.setMaxTotal(10); 30 | config.setMaxIdle(5); 31 | config.setMaxWaitMillis(1000 * 100); 32 | config.setTestOnBorrow(true); 33 | pool = new JedisPool(config, GetProperties.target.get(canal_destination).ip, GetProperties.target.get(canal_destination).port, 1000 * 10); 34 | } 35 | } 36 | 37 | /** 38 | * return Resource to pool 39 | * 40 | * @param pool 41 | * @param redis 42 | */ 43 | public void returnResource(Jedis redis) { 44 | if (redis != null) { 45 | redis.close(); 46 | } 47 | } 48 | 49 | /** 50 | * get data 51 | * 52 | * @param key 53 | * @return 54 | */ 55 | public String get(String key) throws Exception { 56 | String value = null; 57 | Jedis jedis = null; 58 | try { 59 | jedis = pool.getResource(); 60 | value = jedis.get(key); 61 | } catch (JedisConnectionException e) { 62 | throw new Exception(" redis link fail", e); 63 | } 64 | returnResource(jedis); 65 | return value; 66 | } 67 | 68 | /** 69 | * zrange data 70 | * 71 | * @param key 72 | * @return 73 | */ 74 | @SuppressWarnings("rawtypes") 75 | public Set zrange(String key) throws Exception { 76 | Set value = null; 77 | Jedis jedis = null; 78 | try { 79 | jedis = pool.getResource(); 80 | value = jedis.zrange(key, 0, -1); 81 | } catch (JedisConnectionException e) { 82 | throw new Exception(" redis link fail", e); 83 | } 84 | returnResource(jedis); 85 | return value; 86 | } 87 | 88 | /** 89 | * lrange data 90 | * 91 | * @param key 92 | * @return 93 | */ 94 | @SuppressWarnings("rawtypes") 95 | public List lrange(String key) throws Exception { 96 | List value = null; 97 | Jedis jedis = null; 98 | try { 99 | jedis = pool.getResource(); 100 | value = jedis.lrange(key, 0, -1); 101 | } catch (JedisConnectionException e) { 102 | throw new Exception(" redis link fail", e); 103 | } 104 | returnResource(jedis); 105 | return value; 106 | } 107 | 108 | /** 109 | * set string 110 | * 111 | * @param key 112 | * @return 113 | * @throws Exception 114 | */ 115 | public void set(String key, String value) throws Exception { 116 | Jedis jedis = null; 117 | try { 118 | jedis = pool.getResource(); 119 | jedis.set(key, value); 120 | } catch (JedisConnectionException e) { 121 | throw new Exception(" redis link fail", e); 122 | } 123 | returnResource(jedis); 124 | } 125 | 126 | /** 127 | * set data 128 | * 129 | * @param key 130 | * @return 131 | */ 132 | public void zadd(String key, String member) throws Exception { 133 | Jedis jedis = null; 134 | try { 135 | jedis = pool.getResource(); 136 | long score = (jedis.exists(key)) ? (jedis.zcard(key)) : 0; 137 | jedis.zadd(key, score, member); 138 | } catch (JedisConnectionException e) { 139 | throw new Exception(" redis link fail", e); 140 | } 141 | returnResource(jedis); 142 | } 143 | 144 | /** 145 | * push list in left 146 | * 147 | * @param key 148 | * @return 149 | */ 150 | public void lpush(String key, String member) throws Exception { 151 | Jedis jedis = null; 152 | try { 153 | jedis = pool.getResource(); 154 | jedis.lpush(key, member); 155 | } catch (JedisConnectionException e) { 156 | throw new Exception(" redis link fail", e); 157 | } 158 | returnResource(jedis); 159 | } 160 | 161 | /** 162 | * push list in right 163 | * 164 | * @param key 165 | * @return 166 | */ 167 | public void rpush(String key, String member) throws Exception { 168 | Jedis jedis = null; 169 | try { 170 | jedis = pool.getResource(); 171 | jedis.rpush(key, member); 172 | } catch (JedisConnectionException e) { 173 | throw new Exception(" redis link fail", e); 174 | } 175 | returnResource(jedis); 176 | } 177 | 178 | /** 179 | * exists 180 | * 181 | * @param key 182 | * @return 183 | */ 184 | public boolean exists(String key) throws Exception { 185 | 186 | Jedis jedis = null; 187 | boolean blag = false; 188 | try { 189 | jedis = pool.getResource(); 190 | blag = jedis.exists(key); 191 | } catch (JedisConnectionException e) { 192 | throw new Exception(" redis link fail", e); 193 | } 194 | returnResource(jedis); 195 | return blag; 196 | } 197 | 198 | /** 199 | * del 200 | * 201 | * @param key 202 | * @return 203 | */ 204 | public void del(String key) throws Exception { 205 | Jedis jedis = null; 206 | try { 207 | jedis = pool.getResource(); 208 | jedis.del(key); 209 | } catch (JedisConnectionException e) { 210 | throw new Exception(" redis link fail", e); 211 | } 212 | returnResource(jedis); 213 | } 214 | 215 | /** 216 | * lrem 217 | * 218 | * @param key 219 | * @return 220 | */ 221 | public void lrem(String key, String member) throws Exception { 222 | Jedis jedis = null; 223 | try { 224 | jedis = pool.getResource(); 225 | jedis.lrem(key, 1, member); 226 | } catch (JedisConnectionException e) { 227 | throw new Exception(" redis link fail", e); 228 | } 229 | returnResource(jedis); 230 | } 231 | 232 | /** 233 | * zrem 234 | * 235 | * @param key 236 | * @return 237 | */ 238 | public void zrem(String key, String member) throws Exception { 239 | Jedis jedis = null; 240 | try { 241 | jedis = pool.getResource(); 242 | jedis.zrem(key, member); 243 | } catch (JedisConnectionException e) { 244 | throw new Exception(" redis link fail", e); 245 | } 246 | returnResource(jedis); 247 | } 248 | 249 | /** 250 | * expire 251 | * 252 | * @param key 253 | * @param num 254 | */ 255 | public void expire(String key, int num) throws Exception { 256 | Jedis jedis = null; 257 | try { 258 | jedis = pool.getResource(); 259 | jedis.expire(key, num * 60); 260 | } catch (JedisConnectionException e) { 261 | throw new Exception(" redis link fail", e); 262 | } 263 | returnResource(jedis); 264 | } 265 | 266 | /** 267 | * incr 1 268 | * 269 | * @param key 270 | */ 271 | public void incr(String key) throws Exception { 272 | Jedis jedis = null; 273 | try { 274 | jedis = pool.getResource(); 275 | jedis.incr(key); 276 | } catch (JedisConnectionException e) { 277 | throw new Exception(" redis link fail", e); 278 | } 279 | returnResource(jedis); 280 | } 281 | 282 | /** 283 | * clear 284 | * 285 | * @param num 286 | */ 287 | public void clear() throws Exception { 288 | Jedis jedis = null; 289 | try { 290 | jedis = pool.getResource(); 291 | jedis.flushDB(); 292 | } catch (JedisConnectionException e) { 293 | throw new Exception(" redis link fail", e); 294 | } 295 | returnResource(jedis); 296 | } 297 | } 298 | -------------------------------------------------------------------------------- /src/src/com/sync/common/SsdbApi.java: -------------------------------------------------------------------------------- 1 | package com.sync.common; 2 | 3 | 4 | import com.udpwork.ssdb.*; 5 | 6 | 7 | /** 8 | * SsdbApi 9 | * 10 | * @author sasou web:http://www.php-gene.com/ 11 | * @version 1.0.0 12 | */ 13 | 14 | public class SsdbApi { 15 | private String canal_destination = null; 16 | private static SSDB ssdb = null; 17 | 18 | public SsdbApi(String name) { 19 | canal_destination = name; 20 | } 21 | 22 | public SSDB instance() throws Exception { 23 | if (ssdb == null) { 24 | ssdb = new SSDB(GetProperties.target.get(canal_destination).ip ,GetProperties.target.get(canal_destination).port, 1000 * 10); 25 | } 26 | return ssdb; 27 | } 28 | 29 | 30 | /** 31 | * get data 32 | * 33 | * @param key 34 | * @return 35 | * @throws Exception 36 | */ 37 | public String get(String key) throws Exception { 38 | byte[] resp = null; 39 | try { 40 | instance(); 41 | if (ssdb != null) { 42 | resp = ssdb.get(key); 43 | } 44 | } catch (Exception e) { 45 | if (ssdb != null) { 46 | ssdb.close(); 47 | ssdb = null; 48 | } 49 | throw new Exception(" ssdb link fail", e); 50 | } 51 | return resp.toString(); 52 | } 53 | 54 | 55 | /** 56 | * push list in right 57 | * 58 | * @param key 59 | * @return 60 | * @return 61 | */ 62 | public void rpush(String key, String member) throws Exception { 63 | try { 64 | instance(); 65 | if (ssdb != null) { 66 | ssdb.request("qpush", key, member); 67 | } 68 | } catch (Exception e) { 69 | if (ssdb != null) { 70 | ssdb.close(); 71 | ssdb = null; 72 | } 73 | throw new Exception(" ssdb link fail", e); 74 | } 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /src/src/com/sync/common/TargetData.java: -------------------------------------------------------------------------------- 1 | package com.sync.common; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | /** 7 | * TargetData 8 | * 9 | * @author sasou web:http://www.php-gene.com/ 10 | * @version 1.0.0 11 | */ 12 | public final class TargetData { 13 | public String type = ""; 14 | public String ip = ""; 15 | public int port = 0; 16 | public int deep = 1; 17 | public String plugin = ""; 18 | public String filter = ""; 19 | public String sign = ""; 20 | 21 | @SuppressWarnings("rawtypes") 22 | public Map filterMap = new HashMap(); 23 | 24 | /** 25 | * @return the type 26 | */ 27 | public String getType() { 28 | return type; 29 | } 30 | 31 | /** 32 | * @param type 33 | * the type to set 34 | */ 35 | public void setType(String type) { 36 | this.type = type; 37 | } 38 | 39 | /** 40 | * @return the ip 41 | */ 42 | public String getIp() { 43 | return ip; 44 | } 45 | 46 | /** 47 | * @param ip 48 | * the ip to set 49 | */ 50 | public void setIp(String ip) { 51 | this.ip = ip; 52 | } 53 | 54 | /** 55 | * @return the port 56 | */ 57 | public int getPort() { 58 | return port; 59 | } 60 | 61 | /** 62 | * @param port 63 | * the port to set 64 | */ 65 | public void setPort(int port) { 66 | this.port = port; 67 | } 68 | 69 | /** 70 | * @return the deep 71 | */ 72 | public int getDeep() { 73 | return deep; 74 | } 75 | 76 | /** 77 | * @param deep 78 | * the deep to set 79 | */ 80 | public void setDeep(int deep) { 81 | this.deep = deep; 82 | } 83 | 84 | /** 85 | * @return the plugin 86 | */ 87 | public String getPlugin() { 88 | return plugin; 89 | } 90 | 91 | /** 92 | * @param plugin 93 | * the plugin to set 94 | */ 95 | public void setPlugin(String plugin) { 96 | this.plugin = plugin; 97 | } 98 | 99 | /** 100 | * @return the filter 101 | */ 102 | public String getFilter() { 103 | return filter; 104 | } 105 | 106 | /** 107 | * @param filter 108 | * the filter to set 109 | */ 110 | public void setFilter(String filter) { 111 | this.filter = filter; 112 | } 113 | 114 | /** 115 | * @return the sign 116 | */ 117 | public String getSign() { 118 | return sign; 119 | } 120 | 121 | /** 122 | * @param sign 123 | * the sign to set 124 | */ 125 | public void setSign(String sign) { 126 | this.sign = sign; 127 | } 128 | 129 | /** 130 | * @return the filterMap 131 | */ 132 | @SuppressWarnings("rawtypes") 133 | public Map getFilterMap() { 134 | return filterMap; 135 | } 136 | 137 | /** 138 | * @param filterMap 139 | * the filterMap to set 140 | */ 141 | @SuppressWarnings("rawtypes") 142 | public void setFilterMap(Map filterMap) { 143 | this.filterMap = filterMap; 144 | } 145 | 146 | } 147 | -------------------------------------------------------------------------------- /src/src/com/sync/common/Tool.java: -------------------------------------------------------------------------------- 1 | package com.sync.common; 2 | 3 | import java.security.MessageDigest; 4 | 5 | public class Tool { 6 | 7 | public static boolean checkFilter(String destination, String db, String table) { 8 | if ("".equals(db) || "".equals(table)) { 9 | return false; 10 | } 11 | if (!"".equals(GetProperties.target.get(destination).filter)) { 12 | String key = db + "." + table; 13 | if (GetProperties.target.get(destination).filterMap.containsKey(key)) { 14 | return true; 15 | } 16 | return false; 17 | } 18 | return true; 19 | } 20 | 21 | public static String makeTargetName(String canal_destination, String db, String table) { 22 | int type = GetProperties.target.get(canal_destination).deep; 23 | String ret = null; 24 | switch(type) { 25 | case 1: 26 | ret = "sync_" + canal_destination + "_" + db + "_" + table; 27 | break; 28 | case 2: 29 | ret = "sync_" + canal_destination + "_" + db; 30 | break; 31 | case 3: 32 | ret = "sync_" + canal_destination; 33 | break; 34 | case 4: 35 | ret = "sync_" + "_" + db + "_" + table; 36 | break; 37 | default: 38 | ret = "sync_" + canal_destination + "_" + db + "_" + table; 39 | break; 40 | } 41 | return ret; 42 | } 43 | 44 | public static String md5(String s) { 45 | char hexDigits[]={'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; 46 | try { 47 | byte[] btInput = s.getBytes("UTF-8"); 48 | MessageDigest mdInst = MessageDigest.getInstance("MD5"); 49 | mdInst.update(btInput); 50 | byte[] md = mdInst.digest(); 51 | int j = md.length; 52 | char str[] = new char[j * 2]; 53 | int k = 0; 54 | for (int i = 0; i < j; i++) { 55 | byte byte0 = md[i]; 56 | str[k++] = hexDigits[byte0 >>> 4 & 0xf]; 57 | str[k++] = hexDigits[byte0 & 0xf]; 58 | } 59 | String rec = new String(str); 60 | return rec.toLowerCase(); 61 | } catch (Exception e) { 62 | e.printStackTrace(); 63 | return null; 64 | } 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/src/com/sync/common/WriteLog.java: -------------------------------------------------------------------------------- 1 | package com.sync.common; 2 | 3 | /** 4 | * WriteLog 5 | * 6 | * @author sasou web:http://www.php-gene.com/ 7 | * @version 1.0.0 8 | */ 9 | import java.io.File; 10 | import java.io.FileOutputStream; 11 | import java.io.IOException; 12 | import java.io.PrintWriter; 13 | import java.io.StringWriter; 14 | import java.util.Calendar; 15 | 16 | /** 17 | * write logString 18 | * 19 | * @param logString 20 | */ 21 | public class WriteLog { 22 | public static String base = null; 23 | 24 | public static void write(String type, String logString) { 25 | if (base == null) { 26 | base = System.getProperty("user.dir"); 27 | } 28 | 29 | String current = base + "/logs/"; 30 | try { 31 | String logFilePathName = null; 32 | Calendar cd = Calendar.getInstance(); 33 | int year = cd.get(Calendar.YEAR); 34 | String month = addZero(cd.get(Calendar.MONTH) + 1); 35 | String day = addZero(cd.get(Calendar.DAY_OF_MONTH)); 36 | String hour = addZero(cd.get(Calendar.HOUR_OF_DAY)); 37 | String min = addZero(cd.get(Calendar.MINUTE)); 38 | String sec = addZero(cd.get(Calendar.SECOND)); 39 | current += year + "-" + month + "-" + day + "/"; 40 | 41 | File fileParentDir = new File(current); 42 | if (!fileParentDir.exists()) { 43 | fileParentDir.mkdirs(); 44 | } 45 | 46 | logFilePathName = current + type + ".log"; 47 | 48 | FileOutputStream fos = new FileOutputStream(logFilePathName, true); 49 | String time = "[" + year + "-" + month + "-" + day + " " + hour + ":" + min + ":" + sec + "] "; 50 | String content = time + logString + "\r\n"; 51 | fos.write(content.getBytes()); 52 | fos.flush(); 53 | fos.close(); 54 | if (GetProperties.system_debug > 0) { 55 | System.out.println(logString); 56 | } 57 | } catch (Exception e) { 58 | e.printStackTrace(); 59 | } 60 | } 61 | 62 | /** 63 | * add 0 64 | * 65 | * @param i 66 | * @return 67 | */ 68 | public static String addZero(int i) { 69 | if (i < 10) { 70 | String tmpString = "0" + i; 71 | return tmpString; 72 | } else { 73 | return String.valueOf(i); 74 | } 75 | } 76 | 77 | 78 | /** 79 | * eString 80 | * 81 | * @param object e 82 | * @return string 83 | */ 84 | public static String eString(Exception e) { 85 | StringWriter sw = null; 86 | PrintWriter pw = null; 87 | try { 88 | sw = new StringWriter(); 89 | pw = new PrintWriter(sw); 90 | e.printStackTrace(pw); 91 | pw.flush(); 92 | sw.flush(); 93 | } finally { 94 | if (sw != null) { 95 | try { 96 | sw.close(); 97 | } catch (IOException e1) { 98 | e1.printStackTrace(); 99 | } 100 | } 101 | if (pw != null) { 102 | pw.close(); 103 | } 104 | } 105 | return sw.toString(); 106 | } 107 | 108 | } -------------------------------------------------------------------------------- /src/src/com/sync/process/Cache.java: -------------------------------------------------------------------------------- 1 | package com.sync.process; 2 | 3 | import java.net.InetSocketAddress; 4 | import java.util.HashSet; 5 | import java.util.Iterator; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import com.alibaba.otter.canal.client.CanalConnector; 10 | import com.alibaba.otter.canal.client.CanalConnectors; 11 | import com.alibaba.otter.canal.protocol.Message; 12 | import com.alibaba.otter.canal.protocol.CanalEntry.Column; 13 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 14 | import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; 15 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType; 16 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; 17 | import com.alibaba.otter.canal.protocol.CanalEntry.RowData; 18 | import com.sync.common.GetProperties; 19 | import com.sync.common.MemApi; 20 | import com.sync.common.RedisApi; 21 | import com.sync.common.Tool; 22 | import com.sync.common.WriteLog; 23 | import com.alibaba.fastjson.JSON; 24 | 25 | /** 26 | * Cache Producer 27 | * 28 | * @author sasou web:http://www.php-gene.com/ 29 | * @version 1.0.0 30 | */ 31 | public class Cache implements Runnable { 32 | private RedisApi RedisPool = null; 33 | private MemApi MemPool = null; 34 | private CanalConnector connector = null; 35 | private String thread_name = null; 36 | private String canal_destination = null; 37 | private String sign = null; 38 | 39 | public Cache(String name) { 40 | thread_name = "canal[" + name + "]:"; 41 | canal_destination = name; 42 | sign = GetProperties.target.get(canal_destination).sign; 43 | } 44 | 45 | public void process() { 46 | int batchSize = 1000; 47 | connector = CanalConnectors.newSingleConnector( 48 | new InetSocketAddress(GetProperties.canal.ip, GetProperties.canal.port), canal_destination, 49 | GetProperties.canal.username, GetProperties.canal.password); 50 | 51 | connector.connect(); 52 | connector.subscribe(); 53 | 54 | try { 55 | if("redis".equals(GetProperties.target.get(canal_destination).plugin)) { 56 | RedisPool = new RedisApi(canal_destination); 57 | } 58 | if("memcached".equals(GetProperties.target.get(canal_destination).plugin)) { 59 | MemPool = new MemApi(canal_destination); 60 | } 61 | 62 | WriteLog.write(canal_destination, thread_name + "Start-up success!"); 63 | while (true) { 64 | Message message = connector.getWithoutAck(batchSize); // get batch num 65 | long batchId = message.getId(); 66 | int size = message.getEntries().size(); 67 | if (!(batchId == -1 || size == 0)) { 68 | if (syncEntry(message.getEntries())) { 69 | connector.ack(batchId); // commit 70 | } else { 71 | connector.rollback(batchId); // rollback 72 | } 73 | } 74 | } 75 | } catch (Exception e) { 76 | WriteLog.write(canal_destination, thread_name + WriteLog.eString(e)); 77 | } 78 | } 79 | 80 | public void run() { 81 | while (true) { 82 | try { 83 | process(); 84 | } catch (Exception e) { 85 | WriteLog.write(canal_destination, thread_name + "canal link failure!"); 86 | } finally { 87 | if (connector != null) { 88 | connector.disconnect(); 89 | connector = null; 90 | } 91 | } 92 | } 93 | } 94 | 95 | private boolean syncEntry(List entrys) { 96 | String table = ""; 97 | boolean ret = true; 98 | for (Entry entry : entrys) { 99 | EntryType type = entry.getEntryType(); 100 | if (type == EntryType.TRANSACTIONBEGIN || type== EntryType.TRANSACTIONEND) { 101 | continue; 102 | } 103 | table = entry.getHeader().getSchemaName() + "." + entry.getHeader().getTableName(); 104 | if (".".equals(table)) { 105 | continue; 106 | } 107 | RowChange rowChage = null; 108 | try { 109 | rowChage = RowChange.parseFrom(entry.getStoreValue()); 110 | } catch (Exception e) { 111 | throw new RuntimeException( 112 | thread_name + "parser of eromanga-event has an error , data:" + entry.toString(), e); 113 | } 114 | 115 | EventType eventType = rowChage.getEventType(); 116 | HashSet versionField = new HashSet(); 117 | versionField.add(table); 118 | for (RowData rowData : rowChage.getRowDatasList()) { 119 | if (eventType == EventType.DELETE) { 120 | updateColumn(versionField, rowData.getBeforeColumnsList(), table); 121 | } else if (eventType == EventType.INSERT) { 122 | updateColumn(versionField, rowData.getAfterColumnsList(), table); 123 | } else { 124 | updateColumn(versionField, rowData.getBeforeColumnsList(), table); 125 | updateColumn(versionField, rowData.getAfterColumnsList(), table); 126 | } 127 | } 128 | 129 | try { 130 | Iterator iterator = versionField.iterator(); 131 | while (iterator.hasNext()) { 132 | if("redis".equals(GetProperties.target.get(canal_destination).plugin)) { 133 | RedisPool.incr(sign + Tool.md5(iterator.next())); 134 | } 135 | if("memcached".equals(GetProperties.target.get(canal_destination).plugin)) { 136 | MemPool.incr(sign + Tool.md5(iterator.next())); 137 | } 138 | } 139 | if (GetProperties.system_debug > 0) { 140 | String text = JSON.toJSONString(versionField); 141 | WriteLog.write(canal_destination + ".access", thread_name + "data(" + text + ")"); 142 | } 143 | } catch (Exception e) { 144 | WriteLog.write(canal_destination + ".error", thread_name + "cache link failure!" + WriteLog.eString(e)); 145 | ret = false; 146 | } 147 | versionField.clear(); 148 | versionField = null; 149 | } 150 | return ret; 151 | } 152 | 153 | @SuppressWarnings("rawtypes") 154 | private void updateColumn(HashSet versionField, List columns, String table) { 155 | Map field = (Map) GetProperties.target.get(canal_destination).filterMap.get(table); 156 | for (Column column : columns) { 157 | String key = table + "." + column.getName(); 158 | if (column.getIsKey()) { 159 | versionField.add(key + "." + column.getValue()); 160 | } else { 161 | if (field != null && field.containsKey(column.getName())) { 162 | versionField.add(key + "." + column.getValue()); 163 | } 164 | } 165 | } 166 | } 167 | 168 | protected void finalize() throws Throwable { 169 | if (connector != null) { 170 | connector.disconnect(); 171 | connector = null; 172 | } 173 | } 174 | 175 | } -------------------------------------------------------------------------------- /src/src/com/sync/process/ElasticSearch.java: -------------------------------------------------------------------------------- 1 | package com.sync.process; 2 | 3 | import java.net.InetSocketAddress; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Date; 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | import org.apache.commons.lang.time.DateFormatUtils; 11 | 12 | import com.alibaba.otter.canal.client.CanalConnector; 13 | import com.alibaba.otter.canal.client.CanalConnectors; 14 | import com.alibaba.otter.canal.protocol.Message; 15 | import com.alibaba.otter.canal.protocol.CanalEntry.Column; 16 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 17 | import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; 18 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType; 19 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; 20 | import com.alibaba.otter.canal.protocol.CanalEntry.RowData; 21 | import com.sync.common.EsApi; 22 | import com.sync.common.GetProperties; 23 | import com.sync.common.Tool; 24 | import com.sync.common.WriteLog; 25 | import com.alibaba.fastjson.JSON; 26 | 27 | /** 28 | * ElasticSearch Producer 29 | * 30 | * @author sasou web:http://www.php-gene.com/ 31 | * @version 1.0.0 32 | */ 33 | public class ElasticSearch implements Runnable { 34 | private EsApi es = null; 35 | private CanalConnector connector = null; 36 | private String thread_name = null; 37 | private String canal_destination = null; 38 | 39 | public ElasticSearch(String name) { 40 | thread_name = "canal[" + name + "]:"; 41 | canal_destination = name; 42 | } 43 | 44 | public void process() { 45 | int batchSize = 1000; 46 | connector = CanalConnectors.newSingleConnector( 47 | new InetSocketAddress(GetProperties.canal.ip, GetProperties.canal.port), canal_destination, 48 | GetProperties.canal.username, GetProperties.canal.password); 49 | 50 | connector.connect(); 51 | connector.subscribe(); 52 | 53 | try { 54 | es = new EsApi(canal_destination); 55 | WriteLog.write(canal_destination, thread_name + "Start-up success!"); 56 | while (true) { 57 | Message message = connector.getWithoutAck(batchSize); // get batch num 58 | long batchId = message.getId(); 59 | int size = message.getEntries().size(); 60 | if (!(batchId == -1 || size == 0)) { 61 | if (syncEntry(message.getEntries())) { 62 | connector.ack(batchId); // commit 63 | } else { 64 | connector.rollback(batchId); // rollback 65 | } 66 | } 67 | } 68 | } catch (Exception e) { 69 | WriteLog.write(canal_destination, thread_name + WriteLog.eString(e)); 70 | } 71 | } 72 | 73 | public void run() { 74 | while (true) { 75 | try { 76 | process(); 77 | } catch (Exception e) { 78 | WriteLog.write(canal_destination, thread_name + "canal link failure!"); 79 | } finally { 80 | if (connector != null) { 81 | connector.disconnect(); 82 | connector = null; 83 | } 84 | } 85 | } 86 | } 87 | 88 | private boolean syncEntry(List entrys) { 89 | String topic = ""; 90 | int no = 0; 91 | boolean ret = true; 92 | for (Entry entry : entrys) { 93 | EntryType type = entry.getEntryType(); 94 | if (type == EntryType.TRANSACTIONBEGIN || type== EntryType.TRANSACTIONEND) { 95 | continue; 96 | } 97 | String db = entry.getHeader().getSchemaName(); 98 | String table = entry.getHeader().getTableName(); 99 | if (!Tool.checkFilter(canal_destination, db, table)) { 100 | continue; 101 | } 102 | RowChange rowChage = null; 103 | try { 104 | rowChage = RowChange.parseFrom(entry.getStoreValue()); 105 | } catch (Exception e) { 106 | throw new RuntimeException( 107 | thread_name + "parser of eromanga-event has an error , data:" + entry.toString(), e); 108 | } 109 | 110 | EventType eventType = rowChage.getEventType(); 111 | Map data = new HashMap(); 112 | Map head = new HashMap(); 113 | head.put("binlog_file", entry.getHeader().getLogfileName()); 114 | head.put("binlog_pos", entry.getHeader().getLogfileOffset()); 115 | head.put("db", db); 116 | head.put("table", table); 117 | head.put("type", eventType); 118 | data.put("head", head); 119 | topic = Tool.makeTargetName(canal_destination, db, table); 120 | no = (int) entry.getHeader().getLogfileOffset(); 121 | for (RowData rowData : rowChage.getRowDatasList()) { 122 | if (eventType == EventType.DELETE) { 123 | head.put("id", getIndex(rowData.getBeforeColumnsList())); 124 | data.put("before", makeColumn(rowData.getBeforeColumnsList())); 125 | } else if (eventType == EventType.INSERT) { 126 | head.put("id", getIndex(rowData.getAfterColumnsList())); 127 | data.put("after", makeColumn(rowData.getAfterColumnsList())); 128 | } else { 129 | head.put("id", getIndex(rowData.getAfterColumnsList())); 130 | data.put("before", makeColumn(rowData.getBeforeColumnsList())); 131 | data.put("after", makeColumn(rowData.getAfterColumnsList())); 132 | } 133 | String text = JSON.toJSONString(data); 134 | try { 135 | ret = es.sync(topic, text); 136 | if (GetProperties.system_debug > 0) { 137 | WriteLog.write(canal_destination + ".access", thread_name + "data(" + topic + "," + no + ", " + text + ")"); 138 | } 139 | } catch (Exception e) { 140 | WriteLog.write(canal_destination + ".error", thread_name + "es link failure!"+ WriteLog.eString(e)); 141 | ret = false; 142 | } 143 | } 144 | data.clear(); 145 | data = null; 146 | } 147 | return ret; 148 | } 149 | 150 | private String getIndex(List columns) { 151 | String ret = ""; 152 | for (Column column : columns) { 153 | if (column.getIsKey()) { 154 | ret = (String) column.getValue().toString(); 155 | break; 156 | } 157 | } 158 | return ret; 159 | } 160 | 161 | private Map makeColumn(List columns) { 162 | Map one = new HashMap(); 163 | String pattern = "yyyy-MM-dd'T'HH:mm:ss.SSS"; 164 | for (Column column : columns) { 165 | String mt = column.getMysqlType(); 166 | if (mt.contains("timestamp") || mt.contains("datetime")) { 167 | if (!"".equals(column.getValue())) { 168 | Date date = stringToDate(column.getValue()); 169 | one.put(column.getName(), DateFormatUtils.format(date, pattern) + "+0800"); 170 | } 171 | } else { 172 | one.put(column.getName(), column.getValue()); 173 | } 174 | } 175 | one.put("@timestamp", DateFormatUtils.format(new Date(), pattern) + "+0800"); 176 | return one; 177 | } 178 | 179 | public static Date stringToDate(String source) { 180 | SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 181 | Date date = null; 182 | try { 183 | date = simpleDateFormat.parse(source); 184 | } catch (Exception e) { 185 | } 186 | return date; 187 | } 188 | 189 | protected void finalize() throws Throwable { 190 | if (connector != null) { 191 | connector.disconnect(); 192 | connector = null; 193 | } 194 | } 195 | 196 | } -------------------------------------------------------------------------------- /src/src/com/sync/process/Httpmq.java: -------------------------------------------------------------------------------- 1 | package com.sync.process; 2 | 3 | import java.net.InetSocketAddress; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | import com.alibaba.otter.canal.client.CanalConnector; 8 | import com.alibaba.otter.canal.client.CanalConnectors; 9 | import com.alibaba.otter.canal.protocol.Message; 10 | import com.alibaba.otter.canal.protocol.CanalEntry.Column; 11 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 12 | import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; 13 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType; 14 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; 15 | import com.alibaba.otter.canal.protocol.CanalEntry.RowData; 16 | import com.sync.common.GetProperties; 17 | import com.sync.common.HttpmqApi; 18 | import com.sync.common.Tool; 19 | import com.sync.common.WriteLog; 20 | import com.alibaba.fastjson.JSON; 21 | 22 | /** 23 | * Httpmq Producer 24 | * 25 | * @author sasou web:http://www.php-gene.com/ 26 | * @version 1.0.0 27 | */ 28 | public class Httpmq implements Runnable { 29 | private HttpmqApi httpmqApi = null; 30 | private CanalConnector connector = null; 31 | private String thread_name = null; 32 | private String canal_destination = null; 33 | 34 | public Httpmq(String name) { 35 | thread_name = "canal[" + name + "]:"; 36 | canal_destination = name; 37 | } 38 | 39 | public void process() { 40 | int batchSize = 1000; 41 | connector = CanalConnectors.newSingleConnector( 42 | new InetSocketAddress(GetProperties.canal.ip, GetProperties.canal.port), canal_destination, 43 | GetProperties.canal.username, GetProperties.canal.password); 44 | 45 | connector.connect(); 46 | connector.subscribe(); 47 | 48 | try { 49 | httpmqApi = new HttpmqApi(canal_destination); 50 | WriteLog.write(canal_destination, thread_name + "Start-up success!"); 51 | while (true) { 52 | Message message = connector.getWithoutAck(batchSize); // get batch num 53 | long batchId = message.getId(); 54 | int size = message.getEntries().size(); 55 | if (!(batchId == -1 || size == 0)) { 56 | if (syncEntry(message.getEntries())) { 57 | connector.ack(batchId); // commit 58 | } else { 59 | connector.rollback(batchId); // rollback 60 | } 61 | } 62 | } 63 | } catch (Exception e) { 64 | WriteLog.write(canal_destination, thread_name + WriteLog.eString(e)); 65 | } 66 | } 67 | 68 | public void run() { 69 | while (true) { 70 | try { 71 | process(); 72 | } catch (Exception e) { 73 | WriteLog.write(canal_destination, thread_name + "canal link failure!"); 74 | } finally { 75 | if (connector != null) { 76 | connector.disconnect(); 77 | connector = null; 78 | } 79 | } 80 | } 81 | } 82 | 83 | private boolean syncEntry(List entrys) { 84 | int no = 0; 85 | boolean ret = true; 86 | for (Entry entry : entrys) { 87 | EntryType type = entry.getEntryType(); 88 | if (type == EntryType.TRANSACTIONBEGIN || type== EntryType.TRANSACTIONEND) { 89 | continue; 90 | } 91 | String db = entry.getHeader().getSchemaName(); 92 | String table = entry.getHeader().getTableName(); 93 | if (!Tool.checkFilter(canal_destination, db, table)) { 94 | continue; 95 | } 96 | RowChange rowChage = null; 97 | try { 98 | rowChage = RowChange.parseFrom(entry.getStoreValue()); 99 | } catch (Exception e) { 100 | throw new RuntimeException( 101 | thread_name + "parser of eromanga-event has an error , data:" + entry.toString(), e); 102 | } 103 | 104 | EventType eventType = rowChage.getEventType(); 105 | Map data = new HashMap(); 106 | Map head = new HashMap(); 107 | head.put("binlog_file", entry.getHeader().getLogfileName()); 108 | head.put("binlog_pos", entry.getHeader().getLogfileOffset()); 109 | head.put("db", db); 110 | head.put("table", table); 111 | head.put("type", eventType); 112 | data.put("head", head); 113 | String topic = Tool.makeTargetName(canal_destination, db, table); 114 | no = (int) entry.getHeader().getLogfileOffset(); 115 | for (RowData rowData : rowChage.getRowDatasList()) { 116 | if (eventType == EventType.DELETE) { 117 | data.put("before", makeColumn(rowData.getBeforeColumnsList())); 118 | } else if (eventType == EventType.INSERT) { 119 | data.put("after", makeColumn(rowData.getAfterColumnsList())); 120 | } else { 121 | data.put("before", makeColumn(rowData.getBeforeColumnsList())); 122 | data.put("after", makeColumn(rowData.getAfterColumnsList())); 123 | } 124 | String text = JSON.toJSONString(data); 125 | try { 126 | if (!httpmqApi.put(topic, text)) { 127 | ret = false; 128 | } 129 | if (GetProperties.system_debug > 0) { 130 | WriteLog.write(canal_destination + ".access", thread_name + "data(" + topic + "," + no + ", " + text + ")"); 131 | } 132 | } catch (Exception e) { 133 | WriteLog.write(canal_destination + ".error", thread_name + "httpmq link failure!" + WriteLog.eString(e)); 134 | ret = false; 135 | } 136 | } 137 | data.clear(); 138 | data = null; 139 | } 140 | return ret; 141 | } 142 | 143 | private Map makeColumn(List columns) { 144 | Map one = new HashMap(); 145 | for (Column column : columns) { 146 | one.put(column.getName(), column.getValue()); 147 | } 148 | return one; 149 | } 150 | 151 | protected void finalize() throws Throwable { 152 | if (connector != null) { 153 | connector.disconnect(); 154 | connector = null; 155 | } 156 | } 157 | 158 | } -------------------------------------------------------------------------------- /src/src/com/sync/process/Kafka.java: -------------------------------------------------------------------------------- 1 | package com.sync.process; 2 | 3 | import java.net.InetSocketAddress; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.Properties; 8 | import java.util.concurrent.ExecutionException; 9 | 10 | import org.apache.kafka.clients.producer.KafkaProducer; 11 | import org.apache.kafka.clients.producer.ProducerRecord; 12 | import org.apache.kafka.clients.producer.RecordMetadata; 13 | import com.alibaba.otter.canal.client.CanalConnector; 14 | import com.alibaba.otter.canal.client.CanalConnectors; 15 | import com.alibaba.otter.canal.protocol.Message; 16 | import com.alibaba.otter.canal.protocol.CanalEntry.Column; 17 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 18 | import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; 19 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType; 20 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; 21 | import com.alibaba.otter.canal.protocol.CanalEntry.RowData; 22 | import com.sync.common.GetProperties; 23 | import com.sync.common.Tool; 24 | import com.sync.common.WriteLog; 25 | import com.alibaba.fastjson.JSON; 26 | 27 | /** 28 | * kafka Producer 29 | * 30 | * @author sasou web:http://www.php-gene.com/ 31 | * @version 1.0.0 32 | */ 33 | public class Kafka implements Runnable { 34 | private KafkaProducer producer; 35 | private CanalConnector connector = null; 36 | private String thread_name = null; 37 | private String canal_destination = null; 38 | 39 | public Kafka(String name) { 40 | thread_name = "canal[" + name + "]:"; 41 | canal_destination = name; 42 | } 43 | 44 | public void process() { 45 | Properties props = new Properties(); 46 | props.put("bootstrap.servers", GetProperties.target.get(canal_destination).ip + ":" + GetProperties.target.get(canal_destination).port); 47 | props.put("client.id", canal_destination + "_Producer"); 48 | props.put("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer"); 49 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 50 | 51 | int batchSize = 1000; 52 | connector = CanalConnectors.newSingleConnector( 53 | new InetSocketAddress(GetProperties.canal.ip, GetProperties.canal.port), canal_destination, 54 | GetProperties.canal.username, GetProperties.canal.password); 55 | 56 | connector.connect(); 57 | connector.subscribe(); 58 | 59 | try { 60 | producer = new KafkaProducer<>(props); 61 | WriteLog.write(canal_destination, thread_name + "Start-up success!"); 62 | while (true) { 63 | Message message = connector.getWithoutAck(batchSize); // get batch num 64 | long batchId = message.getId(); 65 | int size = message.getEntries().size(); 66 | if (!(batchId == -1 || size == 0)) { 67 | if (syncEntry(message.getEntries())) { 68 | connector.ack(batchId); // commit 69 | } else { 70 | connector.rollback(batchId); // rollback 71 | } 72 | } 73 | } 74 | } catch (Exception e) { 75 | WriteLog.write(canal_destination, thread_name + WriteLog.eString(e)); 76 | } 77 | } 78 | 79 | public void run() { 80 | while (true) { 81 | try { 82 | process(); 83 | } catch (Exception e) { 84 | WriteLog.write(canal_destination, thread_name + "canal link failure!"); 85 | } finally { 86 | if (connector != null) { 87 | connector.disconnect(); 88 | connector = null; 89 | } 90 | } 91 | } 92 | } 93 | 94 | private boolean syncEntry(List entrys) { 95 | String topic = ""; 96 | int no = 0; 97 | RecordMetadata metadata = null; 98 | boolean ret = true; 99 | for (Entry entry : entrys) { 100 | EntryType type = entry.getEntryType(); 101 | if (type == EntryType.TRANSACTIONBEGIN || type== EntryType.TRANSACTIONEND) { 102 | continue; 103 | } 104 | String db = entry.getHeader().getSchemaName(); 105 | String table = entry.getHeader().getTableName(); 106 | if (!Tool.checkFilter(canal_destination, db, table)) { 107 | continue; 108 | } 109 | RowChange rowChage = null; 110 | try { 111 | rowChage = RowChange.parseFrom(entry.getStoreValue()); 112 | } catch (Exception e) { 113 | throw new RuntimeException( 114 | thread_name + "parser of eromanga-event has an error , data:" + entry.toString(), e); 115 | } 116 | 117 | EventType eventType = rowChage.getEventType(); 118 | Map data = new HashMap(); 119 | Map head = new HashMap(); 120 | head.put("binlog_file", entry.getHeader().getLogfileName()); 121 | head.put("binlog_pos", entry.getHeader().getLogfileOffset()); 122 | head.put("db", db); 123 | head.put("table", table); 124 | head.put("type", eventType); 125 | data.put("head", head); 126 | topic = Tool.makeTargetName(canal_destination, db, table); 127 | no = (int) entry.getHeader().getLogfileOffset(); 128 | for (RowData rowData : rowChage.getRowDatasList()) { 129 | if (eventType == EventType.DELETE) { 130 | data.put("before", makeColumn(rowData.getBeforeColumnsList())); 131 | } else if (eventType == EventType.INSERT) { 132 | data.put("after", makeColumn(rowData.getAfterColumnsList())); 133 | } else { 134 | data.put("before", makeColumn(rowData.getBeforeColumnsList())); 135 | data.put("after", makeColumn(rowData.getAfterColumnsList())); 136 | } 137 | String text = JSON.toJSONString(data); 138 | try { 139 | metadata = producer.send(new ProducerRecord<>(topic, no, text)).get(); 140 | if (metadata == null) { 141 | ret = false; 142 | } 143 | if (GetProperties.system_debug > 0) { 144 | WriteLog.write(canal_destination + ".access", thread_name + "data(" + topic + "," + no + ", " + text + ")"); 145 | } 146 | } catch (InterruptedException | ExecutionException e) { 147 | WriteLog.write(canal_destination + ".error", thread_name + "kafka link failure!"+ WriteLog.eString(e)); 148 | ret = false; 149 | } 150 | } 151 | data.clear(); 152 | data = null; 153 | } 154 | return ret; 155 | } 156 | 157 | private Map makeColumn(List columns) { 158 | Map one = new HashMap(); 159 | for (Column column : columns) { 160 | one.put(column.getName(), column.getValue()); 161 | } 162 | return one; 163 | } 164 | 165 | protected void finalize() throws Throwable { 166 | if (connector != null) { 167 | connector.disconnect(); 168 | connector = null; 169 | } 170 | } 171 | 172 | } -------------------------------------------------------------------------------- /src/src/com/sync/process/Prop.java: -------------------------------------------------------------------------------- 1 | package com.sync.process; 2 | 3 | import com.sync.common.GetProperties; 4 | 5 | public class Prop implements Runnable { 6 | 7 | public void run() { 8 | while (true) { 9 | GetProperties.update(); 10 | try { 11 | Thread.sleep(1000 * 10); 12 | } catch (InterruptedException e) { 13 | 14 | } 15 | } 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/src/com/sync/process/Redis.java: -------------------------------------------------------------------------------- 1 | package com.sync.process; 2 | 3 | import java.net.InetSocketAddress; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | import com.alibaba.otter.canal.client.CanalConnector; 8 | import com.alibaba.otter.canal.client.CanalConnectors; 9 | import com.alibaba.otter.canal.protocol.Message; 10 | import com.alibaba.otter.canal.protocol.CanalEntry.Column; 11 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 12 | import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; 13 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType; 14 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; 15 | import com.alibaba.otter.canal.protocol.CanalEntry.RowData; 16 | import com.sync.common.GetProperties; 17 | import com.sync.common.RedisApi; 18 | import com.sync.common.Tool; 19 | import com.sync.common.WriteLog; 20 | import com.alibaba.fastjson.JSON; 21 | 22 | /** 23 | * Redis Producer 24 | * 25 | * @author sasou web:http://www.php-gene.com/ 26 | * @version 1.0.0 27 | */ 28 | public class Redis implements Runnable { 29 | private RedisApi RedisPool = null; 30 | private CanalConnector connector = null; 31 | private String thread_name = null; 32 | private String canal_destination = null; 33 | 34 | public Redis(String name) { 35 | thread_name = "canal[" + name + "]:"; 36 | canal_destination = name; 37 | } 38 | 39 | public void process() { 40 | int batchSize = 1000; 41 | connector = CanalConnectors.newSingleConnector( 42 | new InetSocketAddress(GetProperties.canal.ip, GetProperties.canal.port), canal_destination, 43 | GetProperties.canal.username, GetProperties.canal.password); 44 | 45 | connector.connect(); 46 | connector.subscribe(); 47 | 48 | try { 49 | RedisPool = new RedisApi(canal_destination); 50 | WriteLog.write(canal_destination, thread_name + "Start-up success!"); 51 | while (true) { 52 | Message message = connector.getWithoutAck(batchSize); // get batch num 53 | long batchId = message.getId(); 54 | int size = message.getEntries().size(); 55 | if (!(batchId == -1 || size == 0)) { 56 | if (syncEntry(message.getEntries())) { 57 | connector.ack(batchId); // commit 58 | } else { 59 | connector.rollback(batchId); // rollback 60 | } 61 | } 62 | } 63 | } catch (Exception e) { 64 | WriteLog.write(canal_destination, thread_name + WriteLog.eString(e)); 65 | } 66 | } 67 | 68 | public void run() { 69 | while (true) { 70 | try { 71 | process(); 72 | } catch (Exception e) { 73 | WriteLog.write(canal_destination, thread_name + "canal link failure!"); 74 | } finally { 75 | if (connector != null) { 76 | connector.disconnect(); 77 | connector = null; 78 | } 79 | } 80 | } 81 | } 82 | 83 | private boolean syncEntry(List entrys) { 84 | String topic = ""; 85 | int no = 0; 86 | boolean ret = true; 87 | for (Entry entry : entrys) { 88 | EntryType type = entry.getEntryType(); 89 | if (type == EntryType.TRANSACTIONBEGIN || type== EntryType.TRANSACTIONEND) { 90 | continue; 91 | } 92 | String db = entry.getHeader().getSchemaName(); 93 | String table = entry.getHeader().getTableName(); 94 | if (!Tool.checkFilter(canal_destination, db, table)) { 95 | continue; 96 | } 97 | RowChange rowChage = null; 98 | try { 99 | rowChage = RowChange.parseFrom(entry.getStoreValue()); 100 | } catch (Exception e) { 101 | throw new RuntimeException( 102 | thread_name + "parser of eromanga-event has an error , data:" + entry.toString(), e); 103 | } 104 | 105 | EventType eventType = rowChage.getEventType(); 106 | Map data = new HashMap(); 107 | Map head = new HashMap(); 108 | head.put("binlog_file", entry.getHeader().getLogfileName()); 109 | head.put("binlog_pos", entry.getHeader().getLogfileOffset()); 110 | head.put("db", db); 111 | head.put("table", table); 112 | head.put("type", eventType); 113 | data.put("head", head); 114 | topic = Tool.makeTargetName(canal_destination, db, table); 115 | no = (int) entry.getHeader().getLogfileOffset(); 116 | for (RowData rowData : rowChage.getRowDatasList()) { 117 | if (eventType == EventType.DELETE) { 118 | data.put("before", makeColumn(rowData.getBeforeColumnsList())); 119 | } else if (eventType == EventType.INSERT) { 120 | data.put("after", makeColumn(rowData.getAfterColumnsList())); 121 | } else { 122 | data.put("before", makeColumn(rowData.getBeforeColumnsList())); 123 | data.put("after", makeColumn(rowData.getAfterColumnsList())); 124 | } 125 | String text = JSON.toJSONString(data); 126 | try { 127 | RedisPool.rpush(topic, text); 128 | if (GetProperties.system_debug > 0) { 129 | WriteLog.write(canal_destination + ".access", thread_name + "data(" + topic + "," + no + ", " + text + ")"); 130 | } 131 | } catch (Exception e) { 132 | WriteLog.write(canal_destination + ".error", thread_name + "redis link failure!" + WriteLog.eString(e)); 133 | ret = false; 134 | } 135 | } 136 | data.clear(); 137 | data = null; 138 | } 139 | return ret; 140 | } 141 | 142 | private Map makeColumn(List columns) { 143 | Map one = new HashMap(); 144 | for (Column column : columns) { 145 | one.put(column.getName(), column.getValue()); 146 | } 147 | return one; 148 | } 149 | 150 | protected void finalize() throws Throwable { 151 | if (connector != null) { 152 | connector.disconnect(); 153 | connector = null; 154 | } 155 | } 156 | 157 | } -------------------------------------------------------------------------------- /src/src/com/sync/process/Ssdb.java: -------------------------------------------------------------------------------- 1 | package com.sync.process; 2 | 3 | import java.net.InetSocketAddress; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | import com.alibaba.otter.canal.client.CanalConnector; 8 | import com.alibaba.otter.canal.client.CanalConnectors; 9 | import com.alibaba.otter.canal.protocol.Message; 10 | import com.alibaba.otter.canal.protocol.CanalEntry.Column; 11 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 12 | import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; 13 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType; 14 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; 15 | import com.alibaba.otter.canal.protocol.CanalEntry.RowData; 16 | import com.sync.common.GetProperties; 17 | import com.sync.common.SsdbApi; 18 | import com.sync.common.Tool; 19 | import com.sync.common.WriteLog; 20 | import com.alibaba.fastjson.JSON; 21 | 22 | /** 23 | * Ssdb Producer 24 | * 25 | * @author sasou web:http://www.php-gene.com/ 26 | * @version 1.0.0 27 | */ 28 | public class Ssdb implements Runnable { 29 | private SsdbApi ssdb = null; 30 | private CanalConnector connector = null; 31 | private String thread_name = null; 32 | private String canal_destination = null; 33 | 34 | public Ssdb(String name) { 35 | thread_name = "canal[" + name + "]:"; 36 | canal_destination = name; 37 | } 38 | 39 | public void process() { 40 | int batchSize = 1000; 41 | connector = CanalConnectors.newSingleConnector( 42 | new InetSocketAddress(GetProperties.canal.ip, GetProperties.canal.port), canal_destination, 43 | GetProperties.canal.username, GetProperties.canal.password); 44 | 45 | connector.connect(); 46 | connector.subscribe(); 47 | 48 | try { 49 | ssdb = new SsdbApi(canal_destination); 50 | WriteLog.write(canal_destination, thread_name + "Start-up success!"); 51 | while (true) { 52 | Message message = connector.getWithoutAck(batchSize); // get batch num 53 | long batchId = message.getId(); 54 | int size = message.getEntries().size(); 55 | if (!(batchId == -1 || size == 0)) { 56 | if (syncEntry(message.getEntries())) { 57 | connector.ack(batchId); // commit 58 | } else { 59 | connector.rollback(batchId); // rollback 60 | } 61 | } 62 | } 63 | } catch (Exception e) { 64 | WriteLog.write(canal_destination, thread_name + WriteLog.eString(e)); 65 | } 66 | } 67 | 68 | public void run() { 69 | while (true) { 70 | try { 71 | process(); 72 | } catch (Exception e) { 73 | WriteLog.write(canal_destination, thread_name + "canal link failure!"); 74 | } finally { 75 | if (connector != null) { 76 | connector.disconnect(); 77 | connector = null; 78 | } 79 | } 80 | } 81 | } 82 | 83 | private boolean syncEntry(List entrys) { 84 | String topic = ""; 85 | int no = 0; 86 | boolean ret = true; 87 | for (Entry entry : entrys) { 88 | EntryType type = entry.getEntryType(); 89 | if (type == EntryType.TRANSACTIONBEGIN || type== EntryType.TRANSACTIONEND) { 90 | continue; 91 | } 92 | String db = entry.getHeader().getSchemaName(); 93 | String table = entry.getHeader().getTableName(); 94 | String path = db + "." + table; 95 | if (".".equals(path)) { 96 | continue; 97 | } 98 | if (!Tool.checkFilter(canal_destination, db, table)) { 99 | continue; 100 | } 101 | RowChange rowChage = null; 102 | try { 103 | rowChage = RowChange.parseFrom(entry.getStoreValue()); 104 | } catch (Exception e) { 105 | throw new RuntimeException( 106 | thread_name + "parser of eromanga-event has an error , data:" + entry.toString(), e); 107 | } 108 | 109 | EventType eventType = rowChage.getEventType(); 110 | Map data = new HashMap(); 111 | Map head = new HashMap(); 112 | head.put("binlog_file", entry.getHeader().getLogfileName()); 113 | head.put("binlog_pos", entry.getHeader().getLogfileOffset()); 114 | head.put("db", db); 115 | head.put("table", table); 116 | head.put("type", eventType); 117 | data.put("head", head); 118 | topic = Tool.makeTargetName(canal_destination, db, table); 119 | no = (int) entry.getHeader().getLogfileOffset(); 120 | for (RowData rowData : rowChage.getRowDatasList()) { 121 | if (eventType == EventType.DELETE) { 122 | data.put("before", makeColumn(rowData.getBeforeColumnsList(), path)); 123 | } else if (eventType == EventType.INSERT) { 124 | data.put("after", makeColumn(rowData.getAfterColumnsList(), path)); 125 | } else { 126 | data.put("before", makeColumn(rowData.getBeforeColumnsList(), path)); 127 | data.put("after", makeColumn(rowData.getAfterColumnsList(), path)); 128 | } 129 | String text = JSON.toJSONString(data); 130 | try { 131 | ssdb.rpush(topic, text); 132 | if (GetProperties.system_debug > 0) { 133 | WriteLog.write(canal_destination + ".access", thread_name + "data(" + topic + "," + no + ", " + text + ")"); 134 | } 135 | } catch (Exception e) { 136 | WriteLog.write(canal_destination + ".error", thread_name + "ssdb link failure!" + WriteLog.eString(e)); 137 | ret = false; 138 | } 139 | } 140 | data.clear(); 141 | data = null; 142 | } 143 | return ret; 144 | } 145 | 146 | private Map makeColumn(List columns, String table) { 147 | Map one = new HashMap(); 148 | @SuppressWarnings("rawtypes") 149 | Map field = (Map) GetProperties.target.get(canal_destination).filterMap.get(table); 150 | for (Column column : columns) { 151 | if (column.getIsKey()) { 152 | one.put(column.getName(), column.getValue()); 153 | } else { 154 | if (field != null && field.containsKey(column.getName())) { 155 | one.put(column.getName(), column.getValue()); 156 | } 157 | } 158 | } 159 | return one; 160 | } 161 | 162 | protected void finalize() throws Throwable { 163 | if (connector != null) { 164 | connector.disconnect(); 165 | connector = null; 166 | } 167 | } 168 | 169 | } -------------------------------------------------------------------------------- /src/src/com/sync/process/task.java: -------------------------------------------------------------------------------- 1 | package com.sync.process; 2 | 3 | import com.sync.common.GetProperties; 4 | 5 | /** 6 | * config file read 7 | * 8 | * @author sasou web:http://www.php-gene.com/ 9 | * @version 1.0.0 10 | */ 11 | public final class task { 12 | 13 | public static void main(String[] args) { 14 | // init 15 | GetProperties.update(); 16 | if (GetProperties.canal.destination == null) { 17 | System.out.println("error:canal destination is null!"); 18 | return; 19 | } 20 | int num = GetProperties.canal.destination.length; 21 | if (num > 0) { 22 | for (int i = 0; i < num; i++) { 23 | if (!"".equals(GetProperties.canal.destination[i])) { 24 | String type = GetProperties.target.get(GetProperties.canal.destination[i]).type; 25 | switch (type) { 26 | case "kafka": 27 | new Thread(new Kafka(GetProperties.canal.destination[i])).start(); 28 | break; 29 | case "redis": 30 | new Thread(new Redis(GetProperties.canal.destination[i])).start(); 31 | break; 32 | case "elasticsearch": 33 | new Thread(new ElasticSearch(GetProperties.canal.destination[i])).start(); 34 | break; 35 | case "httpmq": 36 | new Thread(new Httpmq(GetProperties.canal.destination[i])).start(); 37 | break; 38 | case "cache": 39 | new Thread(new Cache(GetProperties.canal.destination[i])).start(); 40 | break; 41 | case "ssdb": 42 | new Thread(new Ssdb(GetProperties.canal.destination[i])).start(); 43 | break; 44 | default: 45 | System.out.println("error:not support type!"); 46 | break; 47 | } 48 | } 49 | } 50 | } 51 | new Thread(new Prop()).start(); 52 | } 53 | 54 | } 55 | --------------------------------------------------------------------------------