├── .DS_Store ├── README.md ├── canal-client ├── .DS_Store ├── .classpath ├── .project ├── .settings │ └── org.eclipse.jdt.core.prefs ├── README.md ├── bin │ ├── .DS_Store │ └── canal │ │ └── client │ │ ├── CanalClientTest.class │ │ ├── rabbitmq.class │ │ └── redis.class ├── canal_client.jar ├── conf │ ├── .DS_Store │ └── canal.properties ├── data │ ├── .DS_Store │ └── binlog_2016.log ├── lib │ ├── .DS_Store │ ├── canal.client-1.0.22.jar │ ├── canal.common-1.0.22.jar │ ├── canal.example-1.0.22.jar │ ├── canal.protocol-1.0.22.jar │ ├── commons-io-2.4.jar │ ├── commons-lang-2.6.jar │ ├── commons-logging-1.1.1.jar │ ├── fastjson-1.1.35.jar │ ├── guava-18.0.jar │ ├── jcl-over-slf4j-1.7.12.jar │ ├── jedis-2.9.0.jar │ ├── log4j-1.2.14.jar │ ├── logback-classic-1.1.3.jar │ ├── logback-core-1.1.3.jar │ ├── netty-3.2.5.Final.jar │ ├── protobuf-java-2.4.1.jar │ ├── rabbitmq-client.jar │ ├── slf4j-api-1.7.12.jar │ ├── spring-2.5.6.jar │ ├── zkclient-0.1.jar │ └── zookeeper-3.4.5.jar ├── src │ ├── .DS_Store │ └── canal │ │ ├── .DS_Store │ │ └── client │ │ ├── .DS_Store │ │ ├── CanalClientTest.java │ │ ├── rabbitmq.java │ │ └── redis.java └── start_canal_client.sh ├── img ├── .DS_Store ├── canal-mysql-nosql.png ├── redis-hash.png └── system-image.png └── python_sync_nosql ├── .DS_Store ├── config.py ├── config.pyc ├── get_file.py ├── get_rabbitmq.py ├── meta.log ├── startup.py └── sync_redis.py /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/.DS_Store -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 下图是最基本的web服务器的结构图。 5 | ![image](https://github.com/liukelin/canal_mysql_nosql_sync/raw/master/img/system-image.png) 6 | 7 | 基于 Canal 的 MySql RabbitMQ Redis/memcached/mongodb 的nosql同步 （多读、nosql延时不严格 需求） 8 | 9 | 1.mysql主从配置 10 | 11 | 2.对mysql binlog(row) parser 这一步交给canal 12 | 13 | 3.MQ对解析后binlog增量数据的推送 14 | 15 | 4.对MQ数据的消费（接收+数据解析，考虑消费速度，MQ队列的阻塞） 16 | 17 | 5.数据写入/修改到nosql （redis的主从/hash分片） 18 | 19 | 6.保证对应关系的简单性：一个mysql表对应一个 redis实例（redis单线程，多实例保证分流不阻塞），关联关系数据交给接口业务 20 | 21 | 数据：mysql->binlog->MQ->redis(不过期、关闭RDB、AOF保证读写性能) （nosql数据仅用crontab脚本维护） 22 | 23 | 请求：http->webserver->redis(有数据)->返回数据 （完全避免用户直接读取mysql） 24 | 25 | ->redis(无数据)->返回空 26 | 27 | 28 | 29 | 传统 Mysql Redis/memcached nosql的缓存 （业务同步） 30 | 从cache读取数据-> 31 | 32 | 1.对数据在mysql的hash算法分布(db/table/分区)，每个hash为节点（nosql数据全部失效时候，可保证mysql各节点可支持直接读取的性能） 33 | 34 | 2.mysql主从 35 | 36 | 3.nosql数据的hash算法分布(多实例、DB)，每个hash为节点 37 | 38 | 4.nosql数据震荡处理 （当某节点挂了寻找替代节点算法（多层hash替代节点）。。。） 39 | 40 | 5.恢复节点数据 41 | 42 | 6.请求：http->webserver->【对key计算一致性hash节点】->connect对应的redis实例 43 | 44 | ->1.redis(有数据)-> 返回数据 45 | 46 | ->2.redis(无数据)-> mysql (并写入数据redis) -> 返回数据 47 | 48 | ->3.redis节点挂掉-> 业务寻址hash替代节点 49 | -> 3.1 redis(有数据) -> 返回数据 50 | 51 | -> 3.2 redis(无数据) -> mysql(并写入数据redis) -> 返回数据 52 | 53 | 54 | ![image](https://github.com/liukelin/canal_mysql_nosql_sync/raw/master/img/canal-mysql-nosql.png) 55 | 56 | 57 | 为什么要使用消息队列（MQ）进行binlog传输: 58 | 59 | 1.增加缓冲，binlog生产端（canal client）只负责生产而不需要考虑消费端的消费能力, 不等待阻塞。 60 | 61 | 2.binlog 消费端: 可实时根据MQ消息的堆积情况，动态 增加/减少 消费端的数量，达到合理的资源利用和消费 62 | 63 | 64 | 部署: 65 | 66 | 阿里canal纯java开发，所以要先安装java环境 67 | 68 | 安装jdk(推荐jdk1.8): 69 | 安装过程参考网上资料，（注意环境变量配置） 70 | 71 | mysql配置： 72 | 1.编辑mysql配置文件 73 | $ sudo vim /etc/my.cnf 74 | 75 | [mysqld] 76 | log-bin=mysql-bin # 77 | binlog-format=ROW #选择row模式 78 | server_id=1 #实例唯一ID，不能和canal的slaveId重复 79 | 80 | 保存并退出，并重启mysql 81 | $ sudo service mysql restart 82 | 83 | 2.创建 mysql账号密码（账号密码自定 权限自定） 84 | 85 | CREATE USER canal IDENTIFIED BY 'canal'; 86 | GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%'; 87 | -- GRANT ALL PRIVILEGES ON *.* TO 'canal'@'%' ; 88 | FLUSH PRIVILEGES; 89 | 90 | canal server 配置启动： 91 | 92 | canal server 模拟mysql从库并向mysql发送dump命令获取mysql binlog数据。 93 | 94 | 1.下载解压项目，这里提供了1.0.22版本:canal.deployer-1.0.22.tar.gz(https://github.com/liukelin/canal_mysql_nosql_sync/files/426724/canal.deployer-1.0.22.tar.gz) 95 | 可从阿里项目下载最新版本 deployer ：https://github.com/alibaba/canal/releases 96 | 97 | 2.配置项目： 98 | # 公共配置 99 | $ sudo vim conf/canal.properties 100 | 101 | canal.port= 11111 # canal server 运行端口，保证该端口为占用状态，或者使用其他未占用端口 102 | 103 | 保存退出。 104 | 105 | # 实例配置 106 | $ sudo vim conf/example/instance.properties 107 | 108 | # position info 109 | canal.instance.master.address = 127.0.0.1:3306 # mysql连接 110 | 111 | canal.instance.dbUsername = canal # mysql账号 112 | canal.instance.dbPassword = canal # 密码 113 | canal.instance.defaultDatabaseName = test # 需要同步的库名 114 | canal.instance.connectionCharset = UTF-8 # mysql编码 115 | 116 | 保存退出。 117 | 118 | 更多配置查看：http://agapple.iteye.com/blog/1831873 119 | 120 | 3.启动： 121 | $ sh bin/startup.sh 122 | 123 | 日志文件：$ less logs/canal/canal.log # canal server端运行日志 124 | $ less logs/example/example.log # canal client端连接日志 125 | $ logs/example/meta.log # 实例binlog 读取记录文件（记录变更位置，默认为新增变更(tail)） 126 | 127 | canal client 配置启动： 128 | 129 | canal client将从canal server获取的binlog数据最终以json行格式保存到指定文件(也可省略这步，直接发送到MQ)。 130 | 131 | binlog生产端和消费端的之间，增加MQ作为缓冲，增加容错度和动态扩展性 132 | 133 | 1.下载解压项目，这里自己写了个基于1.0.22版本的项目:canal_client1.0.22.zip(https://github.com/liukelin/canal_mysql_nosql_sync/releases ), 源码查看：canal-client 134 | 135 | 2.基本配置 136 | 137 | $vim conf/canal.properties 138 | 139 | # cancal server host， 上面 canal server的IP 140 | canal.server.host = 127.0.0.1 141 | 142 | # cancal server port，上面 canal server的启动端口 143 | canal.server.port = 11111 144 | 145 | # 数据保存路径 ，自行指定 146 | canal.binlog.dir = db_data 147 | 148 | # 可选rabbitmq/redis/kafka 作为队列（这里使用 rabbitmq 作为队列传输） 149 | canal.mq = rabbitmq 150 | 151 | ###### rabbitmq 基本配置 ##### 152 | rabbitmq.host = 127.0.0.1 153 | rabbitmq.port = 5672 154 | rabbitmq.user = test 155 | rabbitmq.pass = 123456 156 | 157 | 158 | 保存退出。 159 | 160 | 3.启动canal client： 161 | 162 | $ sh start_canal_client.sh 163 | 164 | 165 | 修改mysql数据触发。 166 | 167 | 最终结果： 168 | eventType ：操作类型（UPDATE/INSERTDELETE） 169 | db： 涉及库 170 | table: 涉及表 171 | before:变更前数据 172 | after: 变更后数据 173 | time: 操作时间 174 | 175 | 176 | $less db_data/binlog_xxxx.log 177 | 178 | {"binlog":"mysql-bin.000009:1235","db":"test","table":"users","eventType":"UPDATE","before":{"uid":"8","username":"duobao153713223"},"after":{"uid":"8","username":"duobao153713223"},"time":"2016-08-22 17:47:25"} 179 | 180 | {"binlog":"mysql-bin.000009:1533","db":"test","table":"users","eventType":"DELETE","before":"","after":{"uid":"8","username":"duobao153713223"},"time":"2016-08-22 17:48:09"} 181 | 182 | {"binlog":"mysql-bin.000009:1790","db":"test","table":"users","eventType":"INSERT","before":"","after":{"uid":"9","username":"test2"},"time":"2016-08-22 17:48:45"} 183 | 184 | 消费数据：（这里使用python3/rabbitmq/redis 作为案例，实际可根据业务需求） 185 | 186 | 流程 ：file数据-> MQ -> nosql 187 | 188 | MQ: rabbitMQ 189 | 190 | 语言：python3 191 | 192 | NoSql: redis 193 | 194 | 195 | 多项目订阅需求，如：client1和client2 需要消费这些数据， 他们得到的数据一样 196 | 开始考虑直接用队列： 197 | 队列数据： [A, B, C, D] 198 | client1 ： 199 | 消费进程1：获取AB 200 | 消费进程2：获取CD 201 | 202 | client2 ： 203 | 消费进程1：获取AB 204 | 消费进程2：获取CD 205 | 206 | 这样的话，如果使用rabbitMQ 就必须给每个 client 提供独立的队列。并独立消费 207 | 1、使用kafka，利用他的分组group,每个client 为一个组，这样就可保证，数据给每个组一致。 208 | 2、对每个项目需求开独立的 canal server instance 和 canal client实例 209 | 210 | 211 | 配置： 212 | 语言：python3 213 | pip：pika redis 214 | 215 | 项目代码： python_sync_nosql 216 | 修改配置文件config.py 217 | # 最终存储数据redis 218 | redis_host = '127.0.0.1' 219 | redis_port = 6379 220 | 221 | ###### rabbitmq 基本配置 ##### 222 | rabbitmq_host = '127.0.0.1' 223 | rabbitmq_port = 5672 224 | rabbitmq_user = 'test' 225 | rabbitmq_pass = '123456' 226 | 227 | # 设置对每个table存储使用的key字段 228 | redis_cache_map = { 229 | # db 230 | 'test':{ 231 | # table ： kid 232 | 'users':'uid', 233 | } 234 | } 235 | 236 | 运行脚本： 237 | $ python3 startup.py 238 | 239 | 240 | 数据最终存储为Redis 的 hash结构，key为 db_table_id 241 | ![image](https://github.com/liukelin/canal_mysql_nosql_sync/raw/master/img/redis-hash.png) 242 | 243 | 244 |

资源下载

245 | 246 | canal server 服务端deployer： https://github.com/alibaba/canal/releases/tag/canal-1.0.22 247 | 248 | canal client 客户端： https://github.com/liukelin/canal_mysql_nosql_sync/releases/tag/1.0.22.2 249 | 250 | 数据消费写入nosql例子: python_sync_nosql 这里是消费rabbitmq数据最终同步到redis 251 | -------------------------------------------------------------------------------- /canal-client/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/.DS_Store -------------------------------------------------------------------------------- /canal-client/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /canal-client/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | canal-client 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | 15 | org.eclipse.jdt.core.javanature 16 | 17 | 18 | -------------------------------------------------------------------------------- /canal-client/.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled 3 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 4 | org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve 5 | org.eclipse.jdt.core.compiler.compliance=1.7 6 | org.eclipse.jdt.core.compiler.debug.lineNumber=generate 7 | org.eclipse.jdt.core.compiler.debug.localVariable=generate 8 | org.eclipse.jdt.core.compiler.debug.sourceFile=generate 9 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error 10 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error 11 | org.eclipse.jdt.core.compiler.source=1.7 12 | -------------------------------------------------------------------------------- /canal-client/README.md: -------------------------------------------------------------------------------- 1 | canal client端封装 2 | 3 | 获取canal server 的binlog 4 | 5 | ↓ 6 | 7 | 将binlog 数据组装成json格式 8 | 9 | ↓ 10 | 11 | 使用MQ传输 （rabbitmq、redis、kafka） 12 | 13 | -------------------------------------------------------------------------------- /canal-client/bin/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/bin/.DS_Store -------------------------------------------------------------------------------- /canal-client/bin/canal/client/CanalClientTest.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/bin/canal/client/CanalClientTest.class -------------------------------------------------------------------------------- /canal-client/bin/canal/client/rabbitmq.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/bin/canal/client/rabbitmq.class -------------------------------------------------------------------------------- /canal-client/bin/canal/client/redis.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/bin/canal/client/redis.class -------------------------------------------------------------------------------- /canal-client/canal_client.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/canal_client.jar -------------------------------------------------------------------------------- /canal-client/conf/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/conf/.DS_Store -------------------------------------------------------------------------------- /canal-client/conf/canal.properties: -------------------------------------------------------------------------------- 1 | ########### cancal client conf ############# 2 | 3 | ########### liukelin ############# 4 | 5 | # cancal server host 6 | canal.server.host = 192.168.179.184 7 | 8 | # cancal server port 9 | canal.server.port = 11111 10 | 11 | # 实例 默认 example/instance.properties 12 | canal.server.instance = example 13 | 14 | # 每次获取binlog数据 行数 15 | canal.batchsize = 1000 16 | 17 | # 每次获取等待时间单位/ms 18 | canal.sleep = 1000 19 | 20 | 21 | ############ 写入到文件 ############### 22 | # 数据保存路径 23 | canal.binlog.dir = /Users/liukelin/Desktop/canal-otter-mycat-cobar/canal_object/data 24 | 25 | # 数据保存格式 y=>YY m=>YYMM d=>YYMMdd h每小时 i每分钟 26 | canal.binlog.filename = y 27 | 28 | ########## 启用队列 redis/rabbitmq/kafka ############ 29 | canal.mq = redis 30 | 31 | ######写入rabbitmq ##### 32 | rabbitmq.host = 192.168.179.184 33 | rabbitmq.port = 5672 34 | rabbitmq.user = test 35 | rabbitmq.pass = 123456 36 | # 队列名称 37 | rabbitmq.queuename = canal_binlog_data 38 | # 队列持久化 39 | rabbitmq.durable = true 40 | # 是否ack 41 | rabbitmq.ack = false 42 | 43 | 44 | ######写入redis ###### 45 | #持久化自行设置 RDB/AOF 46 | redis.host = 192.168.179.184 47 | redis.port = 6379 48 | redis.user = 49 | redis.pass = 50 | redis.queuename = canal_binlog_data 51 | 52 | 53 | 54 | # print 是否打印数据 55 | canal.print = true 56 | -------------------------------------------------------------------------------- /canal-client/data/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/data/.DS_Store -------------------------------------------------------------------------------- /canal-client/data/binlog_2016.log: -------------------------------------------------------------------------------- 1 | {"binlog":"mysql-bin.000052:256","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"6","img":"ee","sex":"32","uid":"5"},"after":{"id":"6","img":"eeasd 多岁的","sex":"32","uid":"5"},"time":"2016-08-23 15:43:55"} 2 | {"binlog":"mysql-bin.000052:506","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"1","img":"444","sex":"333","uid":"2"},"after":{"id":"1","img":"444设定的","sex":"333","uid":"2"},"time":"2016-08-23 15:44:01"} 3 | {"binlog":"mysql-bin.000052:754","db":"kelin_test","table":"user","eventType":"INSERT","before":"","after":{"id":"9","img":"盛大的","sex":"0","uid":"18"},"time":"2016-08-23 15:44:10"} 4 | {"binlog":"mysql-bin.000052:980","db":"kelin_test","table":"user","eventType":"INSERT","before":"","after":{"id":"10","img":"盛大的","sex":"0","uid":"19"},"time":"2016-08-23 15:44:24"} 5 | {"binlog":"mysql-bin.000052:1206","db":"kelin_test","table":"user","eventType":"DELETE","before":"","after":{"id":"6","img":"eeasd 多岁的","sex":"32","uid":"5"},"time":"2016-08-23 15:44:36"} 6 | {"binlog":"mysql-bin.000058:256","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"9","img":"盛大的","sex":"0","uid":"18"},"after":{"id":"9","img":"盛大的2","sex":"0","uid":"18"},"time":"2016-08-26 17:09:26"} 7 | {"binlog":"mysql-bin.000058:508","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"9","img":"盛大的2","sex":"0","uid":"18"},"after":{"id":"9","img":"盛大的23","sex":"0","uid":"18"},"time":"2016-08-26 17:15:01"} 8 | {"binlog":"mysql-bin.000058:762","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"10","img":"盛大的","sex":"0","uid":"19"},"after":{"id":"10","img":"盛大的4","sex":"0","uid":"19"},"time":"2016-08-26 17:17:05"} 9 | {"binlog":"mysql-bin.000058:1014","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"9","img":"盛大的23","sex":"0","uid":"18"},"after":{"id":"9","img":"2323","sex":"0","uid":"18"},"time":"2016-08-26 17:18:06"} 10 | {"binlog":"mysql-bin.000058:1014","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"10","img":"盛大的4","sex":"0","uid":"19"},"after":{"id":"10","img":"2323","sex":"0","uid":"19"},"time":"2016-08-26 17:18:06"} 11 | {"binlog":"mysql-bin.000058:1306","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"9","img":"2323","sex":"0","uid":"18"},"after":{"id":"9","img":"232we3","sex":"0","uid":"18"},"time":"2016-08-26 17:26:32"} 12 | {"binlog":"mysql-bin.000058:1306","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"10","img":"2323","sex":"0","uid":"19"},"after":{"id":"10","img":"232we3","sex":"0","uid":"19"},"time":"2016-08-26 17:26:32"} 13 | {"binlog":"mysql-bin.000060:256","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"9","img":"232we3","sex":"0","uid":"18"},"after":{"id":"9","img":"dd232we3","sex":"0","uid":"18"},"time":"2016-08-27 16:58:39"} 14 | {"binlog":"mysql-bin.000060:503","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"10","img":"232we3","sex":"0","uid":"19"},"after":{"id":"10","img":"232we3f","sex":"0","uid":"19"},"time":"2016-08-27 16:59:43"} 15 | {"binlog":"mysql-bin.000060:749","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"5","img":"{\"\\\\:xx\"}","sex":"2","uid":"4"},"after":{"id":"5","img":"{\"\\\\:xx\"}是","sex":"2","uid":"4"},"time":"2016-08-27 17:09:50"} 16 | {"binlog":"mysql-bin.000060:749","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"5","img":"{\"\\\\:xx\"}","sex":"2","uid":"4"},"after":{"id":"5","img":"{\"\\\\:xx\"}是","sex":"2","uid":"4"},"time":"2016-08-27 17:13:21"} 17 | {"binlog":"mysql-bin.000060:749","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"5","img":"{\"\\\\:xx\"}","sex":"2","uid":"4"},"after":{"id":"5","img":"{\"\\\\:xx\"}是","sex":"2","uid":"4"},"time":"2016-08-27 17:13:21"} 18 | {"binlog":"mysql-bin.000060:1003","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"9","img":"dd232we3","sex":"0","uid":"18"},"after":{"id":"9","img":"dd232dwe3","sex":"0","uid":"18"},"time":"2016-08-27 17:15:09"} 19 | {"binlog":"mysql-bin.000060:1253","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"9","img":"dd232dwe3","sex":"0","uid":"18"},"after":{"id":"9","img":"dd232dwe3dd","sex":"0","uid":"18"},"time":"2016-08-27 17:15:56"} 20 | {"binlog":"mysql-bin.000060:1506","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"9","img":"dd232dwe3dd","sex":"0","uid":"18"},"after":{"id":"9","img":"dd232dw","sex":"0","uid":"18"},"time":"2016-08-27 17:16:52"} 21 | {"binlog":"mysql-bin.000060:1757","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"10","img":"232we3f","sex":"0","uid":"19"},"after":{"id":"10","img":"232wed3f","sex":"0","uid":"19"},"time":"2016-08-27 17:17:25"} 22 | {"binlog":"mysql-bin.000060:2005","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"5","img":"{\"\\\\:xx\"}是","sex":"2","uid":"4"},"after":{"id":"5","img":"{\"\\\\:xx\"}d是","sex":"2","uid":"4"},"time":"2016-08-27 17:20:10"} 23 | {"binlog":"mysql-bin.000060:2263","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"9","img":"dd232dw","sex":"0","uid":"18"},"after":{"id":"9","img":"dd","sex":"0","uid":"18"},"time":"2016-08-27 17:21:52"} 24 | {"binlog":"mysql-bin.000060:2505","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"1","img":"444设定的","sex":"333","uid":"2"},"after":{"id":"1","img":"444设定的w","sex":"333","uid":"2"},"time":"2016-08-27 17:23:59"} 25 | {"binlog":"mysql-bin.000060:2763","db":"kelin_test","table":"user","eventType":"DELETE","before":"","after":{"id":"9","img":"dd","sex":"0","uid":"18"},"time":"2016-08-27 17:26:19"} 26 | {"binlog":"mysql-bin.000060:2982","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"10","img":"232wed3f","sex":"0","uid":"19"},"after":{"id":"10","img":"232wfed3f","sex":"0","uid":"19"},"time":"2016-08-27 17:27:02"} 27 | {"binlog":"mysql-bin.000060:3232","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"10","img":"232wfed3f","sex":"0","uid":"19"},"after":{"id":"10","img":"的","sex":"0","uid":"19"},"time":"2016-08-27 17:27:29"} 28 | {"binlog":"mysql-bin.000060:3477","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"10","img":"的","sex":"0","uid":"19"},"after":{"id":"10","img":"的都懂得","sex":"0","uid":"19"},"time":"2016-08-27 17:28:12"} 29 | {"binlog":"mysql-bin.000062:256","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"10","img":"的都懂得","sex":"0","uid":"19"},"after":{"id":"10","img":"的都懂得d","sex":"0","uid":"19"},"time":"2016-08-29 17:59:07"} 30 | {"binlog":"mysql-bin.000062:514","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"8","img":"\"马克吐温\"的","sex":"123","uid":"6"},"after":{"id":"8","img":"\"马克吐温\"的ee","sex":"123","uid":"6"},"time":"2016-08-29 18:00:24"} 31 | {"binlog":"mysql-bin.000062:783","db":"kelin_test","table":"user","eventType":"UPDATE","before":{"id":"10","img":"的都懂得d","sex":"0","uid":"19"},"after":{"id":"10","img":"的都懂得ddd","sex":"0","uid":"19"},"time":"2016-08-29 18:01:30"} 32 | -------------------------------------------------------------------------------- /canal-client/lib/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/.DS_Store -------------------------------------------------------------------------------- /canal-client/lib/canal.client-1.0.22.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/canal.client-1.0.22.jar -------------------------------------------------------------------------------- /canal-client/lib/canal.common-1.0.22.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/canal.common-1.0.22.jar -------------------------------------------------------------------------------- /canal-client/lib/canal.example-1.0.22.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/canal.example-1.0.22.jar -------------------------------------------------------------------------------- /canal-client/lib/canal.protocol-1.0.22.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/canal.protocol-1.0.22.jar -------------------------------------------------------------------------------- /canal-client/lib/commons-io-2.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/commons-io-2.4.jar -------------------------------------------------------------------------------- /canal-client/lib/commons-lang-2.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/commons-lang-2.6.jar -------------------------------------------------------------------------------- /canal-client/lib/commons-logging-1.1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/commons-logging-1.1.1.jar -------------------------------------------------------------------------------- /canal-client/lib/fastjson-1.1.35.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/fastjson-1.1.35.jar -------------------------------------------------------------------------------- /canal-client/lib/guava-18.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/guava-18.0.jar -------------------------------------------------------------------------------- /canal-client/lib/jcl-over-slf4j-1.7.12.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/jcl-over-slf4j-1.7.12.jar -------------------------------------------------------------------------------- /canal-client/lib/jedis-2.9.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/jedis-2.9.0.jar -------------------------------------------------------------------------------- /canal-client/lib/log4j-1.2.14.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/log4j-1.2.14.jar -------------------------------------------------------------------------------- /canal-client/lib/logback-classic-1.1.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/logback-classic-1.1.3.jar -------------------------------------------------------------------------------- /canal-client/lib/logback-core-1.1.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/logback-core-1.1.3.jar -------------------------------------------------------------------------------- /canal-client/lib/netty-3.2.5.Final.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/netty-3.2.5.Final.jar -------------------------------------------------------------------------------- /canal-client/lib/protobuf-java-2.4.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/protobuf-java-2.4.1.jar -------------------------------------------------------------------------------- /canal-client/lib/rabbitmq-client.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/rabbitmq-client.jar -------------------------------------------------------------------------------- /canal-client/lib/slf4j-api-1.7.12.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/slf4j-api-1.7.12.jar -------------------------------------------------------------------------------- /canal-client/lib/spring-2.5.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/spring-2.5.6.jar -------------------------------------------------------------------------------- /canal-client/lib/zkclient-0.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/zkclient-0.1.jar -------------------------------------------------------------------------------- /canal-client/lib/zookeeper-3.4.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/lib/zookeeper-3.4.5.jar -------------------------------------------------------------------------------- /canal-client/src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/src/.DS_Store -------------------------------------------------------------------------------- /canal-client/src/canal/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/src/canal/.DS_Store -------------------------------------------------------------------------------- /canal-client/src/canal/client/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sansom/canal_mysql_nosql_sync/965ec92a85a48f3e593e6fadd27f2e91acd30cad/canal-client/src/canal/client/.DS_Store -------------------------------------------------------------------------------- /canal-client/src/canal/client/CanalClientTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * canal client 3 | * 从canal server 获取 binlog，并写入文件 4 | * @date 2016-08-13 5 | * @author liukelin 6 | * @email 314566990@qq.com 7 | */ 8 | package canal.client; 9 | 10 | import java.net.InetSocketAddress; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.HashMap; 15 | 16 | import com.alibaba.otter.canal.client.CanalConnector; 17 | import com.alibaba.otter.canal.client.CanalConnectors; 18 | import com.alibaba.otter.canal.protocol.CanalEntry.Column; 19 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 20 | import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; 21 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType; 22 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; 23 | import com.alibaba.otter.canal.protocol.CanalEntry.RowData; 24 | import com.alibaba.otter.canal.protocol.Message; 25 | import com.alibaba.fastjson.JSON; //alibaba的FastJson(高性能JSON开发包) 26 | 27 | //读取 Properties 28 | import java.util.Properties; 29 | import java.io.InputStream; 30 | import java.io.IOException; 31 | import java.io.FileInputStream; 32 | //import java.io.UnsupportedEncodingException; 33 | 34 | //时间 35 | import java.util.Date; 36 | import java.util.concurrent.TimeoutException; 37 | import java.text.SimpleDateFormat; 38 | 39 | //写入文件 40 | import java.io.FileWriter; 41 | //url encode 42 | //import java.net.URLEncoder; 43 | 44 | //rabbitmq(rabbitmq-client.jar) 45 | //import com.rabbitmq.client.ConnectionFactory; 46 | //import com.rabbitmq.client.Connection; 47 | //import com.rabbitmq.client.Channel; 48 | //import com.rabbitmq.client.MessageProperties; 49 | 50 | public class CanalClientTest { 51 | 52 | private static String path = CanalClientTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); 53 | public static String canal_print = "0"; 54 | 55 | //canal server 56 | public static String host = "127.0.0.1"; 57 | public static int port = 11111; 58 | public static String instance = "example"; 59 | public static int batchSize = 1000; //每次获取数据数量 60 | public static int sleep = 1000; //无数据时等待时间 61 | 62 | //file 63 | public static String canal_binlog_filename = "h"; //保存文件名 64 | public static String data_dir = "data"; //数据保存路径 65 | 66 | //mq 67 | public static String canal_mq; // redis/rabbitmq/kafka 68 | 69 | // rabbitmq 70 | public static String rabbitmq_host = "127.0.0.1"; 71 | public static String rabbitmq_port = "5672"; 72 | public static String rabbitmq_user = ""; 73 | public static String rabbitmq_pass = ""; 74 | public static String rabbitmq_queuename = "canal_binlog_data"; //队列名称 75 | public static String rabbitmq_ack = "false"; //ack 76 | public static String rabbitmq_durable = "false"; //队列持久 77 | public static Map rabbitmq_conf; 78 | 79 | // redis 80 | public static String redis_host = "127.0.0.1"; 81 | public static String redis_port = "5672"; 82 | public static String redis_user = ""; 83 | public static String redis_pass = ""; 84 | public static String redis_queuename = "canal_binlog_data"; //队列名称 85 | public static Map redis_conf; 86 | 87 | public static void main(String args[]) { 88 | String conf_path = path.substring(0, path.lastIndexOf("/")) + "/conf/canal.properties"; 89 | //String host = AddressUtils.getHostIp() 90 | 91 | System.out.println("#=====canal client====================\r\n#=====2016====================\r\n" + 92 | "#=====liukelin====================\r\n" + 93 | "#=====conf:"+conf_path); 94 | 95 | //读取配置 96 | try { 97 | Properties prop = new Properties(); 98 | InputStream in = new FileInputStream(conf_path); 99 | // InputStream in = new FileInputStream("/Users/liukelin/Desktop/canal-otter-mycat-cobar/canal_object/conf/canal.properties"); 100 | 101 | prop.load(in); 102 | String conf_host = prop.getProperty("canal.server.host"); 103 | String conf_port = prop.getProperty("canal.server.port"); 104 | String conf_instance = prop.getProperty("canal.server.instance"); 105 | 106 | String conf_batchsize = prop.getProperty("canal.batchsize"); 107 | String conf_sleep = prop.getProperty("canal.sleep"); 108 | String conf_dir = prop.getProperty("canal.binlog.dir"); 109 | String conf_filename = prop.getProperty("canal.binlog.filename"); 110 | String conf_print = prop.getProperty("canal.print"); 111 | 112 | String conf_mq = prop.getProperty("canal.mq"); 113 | 114 | String conf_rabbitmq_host = prop.getProperty("rabbitmq.host"); 115 | String conf_rabbitmq_port = prop.getProperty("rabbitmq.port"); 116 | String conf_rabbitmq_user = prop.getProperty("rabbitmq.user"); 117 | String conf_rabbitmq_pass = prop.getProperty("rabbitmq.pass"); 118 | String conf_rabbitmq_queuename = prop.getProperty("rabbitmq.queuename"); 119 | String conf_rabbitmq_ack = prop.getProperty("rabbitmq.ack"); 120 | String conf_rabbitmq_durable = prop.getProperty("rabbitmq.durable"); 121 | 122 | String conf_redis_host = prop.getProperty("redis.host"); 123 | String conf_redis_port = prop.getProperty("redis.port"); 124 | String conf_redis_user = prop.getProperty("redis.user"); 125 | String conf_redis_pass = prop.getProperty("redis.pass"); 126 | String conf_redis_queuename = prop.getProperty("redis.queuename"); 127 | 128 | if ( conf_host!= null && conf_host!=""){ 129 | host = conf_host.trim(); 130 | } 131 | if ( conf_port!= null && conf_port!=""){ 132 | port = Integer.parseInt(conf_port.trim()); 133 | } 134 | 135 | if ( conf_instance!= null && conf_instance!=""){ 136 | instance = conf_instance.trim(); 137 | } 138 | if ( conf_batchsize!= null && conf_batchsize!=""){ 139 | batchSize = Integer.parseInt(conf_batchsize.trim()); 140 | } 141 | if (conf_sleep!= null && conf_sleep!=""){ 142 | sleep = Integer.parseInt(conf_sleep.trim()); 143 | } 144 | if (conf_dir!= null && conf_dir!=""){ 145 | data_dir = conf_dir.trim(); 146 | } 147 | if (conf_filename!= null && conf_filename!=""){ 148 | canal_binlog_filename = conf_filename.trim(); 149 | } 150 | if (conf_print!= null && conf_print!=""){ 151 | canal_print = conf_print.trim(); 152 | } 153 | if (conf_mq!= null && conf_mq!=""){ 154 | canal_mq = conf_mq.trim(); 155 | } 156 | 157 | if (conf_rabbitmq_host!= null && conf_rabbitmq_host!=""){ 158 | rabbitmq_host = conf_rabbitmq_host.trim(); 159 | } 160 | if (conf_rabbitmq_port!= null && conf_rabbitmq_port!=""){ 161 | rabbitmq_port = conf_rabbitmq_port.trim(); 162 | } 163 | if (conf_rabbitmq_user!= null && conf_rabbitmq_user!=""){ 164 | rabbitmq_user = conf_rabbitmq_user.trim(); 165 | } 166 | if (conf_rabbitmq_pass!= null && conf_rabbitmq_pass!=""){ 167 | rabbitmq_pass = conf_rabbitmq_pass.trim(); 168 | } 169 | if (conf_rabbitmq_queuename!= null && conf_rabbitmq_queuename!=""){ 170 | rabbitmq_queuename = conf_rabbitmq_queuename.trim(); 171 | } 172 | if (conf_rabbitmq_ack!= null && conf_rabbitmq_ack!=""){ 173 | rabbitmq_ack = conf_rabbitmq_ack.trim(); 174 | } 175 | if (conf_rabbitmq_durable!= null && conf_rabbitmq_durable!=""){ 176 | rabbitmq_durable = conf_rabbitmq_durable.trim(); 177 | } 178 | 179 | if (conf_redis_port!= null && conf_redis_port!=""){ 180 | redis_host = conf_redis_host.trim(); 181 | } 182 | if (conf_redis_port!= null && conf_redis_port!=""){ 183 | redis_port = conf_redis_port.trim(); 184 | } 185 | if (conf_redis_user!= null && conf_redis_user!=""){ 186 | redis_user = conf_redis_user.trim(); 187 | } 188 | if (conf_redis_pass!= null && conf_redis_pass!=""){ 189 | redis_pass = conf_redis_pass.trim(); 190 | } 191 | if (conf_redis_queuename!= null && conf_redis_queuename!=""){ 192 | redis_queuename = conf_redis_queuename.trim(); 193 | } 194 | 195 | rabbitmq_conf = new HashMap(); 196 | rabbitmq_conf.put("rabbitmq_host", rabbitmq_host); 197 | rabbitmq_conf.put("rabbitmq_port", rabbitmq_port); 198 | rabbitmq_conf.put("rabbitmq_user", rabbitmq_user); 199 | rabbitmq_conf.put("rabbitmq_pass", rabbitmq_pass); 200 | rabbitmq_conf.put("rabbitmq_queuename", rabbitmq_queuename); 201 | rabbitmq_conf.put("rabbitmq_ack", rabbitmq_ack); 202 | rabbitmq_conf.put("rabbitmq_durable", rabbitmq_durable); 203 | 204 | redis_conf = new HashMap(); 205 | redis_conf.put("host", redis_host); 206 | redis_conf.put("port", redis_port); 207 | redis_conf.put("user", redis_user); 208 | redis_conf.put("pass", redis_pass); 209 | redis_conf.put("queuename", redis_queuename); 210 | 211 | 212 | System.out.println("#=====host:"+host+":"+port+ "\r\n#=====instance:"+instance+"\r\n"); 213 | 214 | } catch (IOException e) { 215 | e.printStackTrace(); 216 | System.out.println("#=====load conf/canal.properties error!"); 217 | } 218 | 219 | // 创建链接 (example 为server conf/example配置) 220 | CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(host, port), instance, "", ""); 221 | //int emptyCount = 0; 222 | try { 223 | connector.connect(); 224 | connector.subscribe(".*\\..*"); 225 | connector.rollback(); 226 | 227 | System.out.println("connect success!\r\n startup..."); 228 | 229 | while (true){ 230 | Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据 231 | long batchId = message.getId(); 232 | int size = message.getEntries().size(); 233 | if (batchId == -1 || size == 0) { 234 | //System.out.println("empty count : " + emptyCount); 235 | try { 236 | Thread.sleep(sleep); // 等待时间 237 | } catch (InterruptedException e) { 238 | 239 | } 240 | } else { 241 | //System.out.printf("message[batchId=%s,size=%s] \n", batchId, size); 242 | printEntry(message.getEntries()); 243 | } 244 | 245 | connector.ack(batchId); // 提交确认 246 | // connector.rollback(batchId); // 处理失败, 回滚数据 247 | } 248 | //System.out.println("empty too many times, exit"); 249 | } finally { 250 | System.out.println("connect error!"); 251 | connector.disconnect(); 252 | } 253 | } 254 | 255 | private static void printEntry(List entrys) { 256 | SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 257 | String timeStr = df.format(new Date()); 258 | 259 | ArrayList dataArray = new ArrayList (); 260 | 261 | //循环每行binlog 262 | for (Entry entry : entrys) { 263 | if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) { 264 | continue; 265 | } 266 | 267 | RowChange rowChage = null; 268 | try { 269 | rowChage = RowChange.parseFrom(entry.getStoreValue()); 270 | } catch (Exception e) { 271 | throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),e); 272 | } 273 | 274 | //单条 binlog sql 275 | EventType eventType = rowChage.getEventType(); 276 | /** 277 | System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", 278 | entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), 279 | entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), 280 | eventType)); **/ 281 | 282 | String header_str = "{\"binlog\":\"" + entry.getHeader().getLogfileName()+ ":" + entry.getHeader().getLogfileOffset() + "\"," + 283 | "\"db\":\"" + entry.getHeader().getSchemaName() + "\"," + 284 | "\"table\":\"" + entry.getHeader().getTableName() + "\","; 285 | //受影响 数据行 286 | for (RowData rowData : rowChage.getRowDatasList()) { 287 | String row_str = "\"eventType\":\"" + eventType +"\","; 288 | String before = "\"\""; 289 | String after = "\"\""; 290 | 291 | //获取字段值 292 | if (eventType == EventType.DELETE) { 293 | after = printColumn(rowData.getBeforeColumnsList()); 294 | } else if (eventType == EventType.INSERT) { 295 | after = printColumn(rowData.getAfterColumnsList()); 296 | } else { //update 297 | //System.out.println("-------> before"); 298 | before = printColumn(rowData.getBeforeColumnsList()); 299 | //System.out.println("-------> after"); 300 | after = printColumn(rowData.getAfterColumnsList()); 301 | } 302 | 303 | String row_data = header_str + row_str + "\"before\":" +before + ",\"after\":" + after + ",\"time\":\"" + timeStr +"\"}"; 304 | dataArray.add(row_data); 305 | save_data_logs(row_data); 306 | //System.out.println(row_data); 307 | } 308 | } 309 | 310 | // ArrayList TO String[] 311 | String[] strArr = dataArray.toArray(new String[]{}); 312 | try { 313 | if(canal_mq.equals("rabbitmq")){ 314 | rabbitmq r = new rabbitmq(); 315 | r.push_rabbitmq(rabbitmq_conf,strArr); 316 | //push_rabbitmq(strArr); 317 | }else if(canal_mq.equals("redis")){ 318 | redis r = new redis(); 319 | r.push_redis(redis_conf, strArr); 320 | }else if(canal_mq.equals("kafka")){ 321 | 322 | } 323 | } catch (IOException e) { 324 | // TODO Auto-generated catch block 325 | e.printStackTrace(); 326 | System.out.println("push "+ canal_mq +" error!"); 327 | } 328 | dataArray = null; 329 | } 330 | 331 | // 获取字段 变更 (1、使用map转换为json。 2、使用urlencode。 避免拼接json错误) 332 | private static String printColumn(List columns) { 333 | //String column_str = ""; 334 | Map column_map = new HashMap(); 335 | for (Column column : columns) { 336 | String column_name = column.getName(); 337 | String column_value = column.getValue(); 338 | 339 | /** 340 | String a = ""; 341 | String b = ""; 342 | String c = ""; 343 | try { 344 | column_value = new String(column_value.getBytes(),"UTF-8"); 345 | a=new String(column.getValue().getBytes("Shift-JIS"),"GBK"); 346 | b= new String(column.getValue().getBytes("Shift_JIS"), "GB2312"); 347 | c= new String(column.getValue().getBytes("ISO-8859-1"), "UTF-8"); 348 | 349 | } catch (UnsupportedEncodingException e) { 350 | // TODO Auto-generated catch block 351 | //e.printStackTrace(); 352 | //column_value = column.getValue(); 353 | } 354 | System.out.println("column_value:" + column_value + " : "+getEncoding(column_value)+ 355 | ", a:"+ a+" : " +getEncoding(a) + 356 | ", b:"+ b+" : " +getEncoding(b) + 357 | ", c:"+ c+" : " +getEncoding(c)); 358 | **/ 359 | column_map.put(column_name, column_value); 360 | //System.out.println(column.getName() + " : " + column.getValue() + " update=" + column.getUpdated()); 361 | } 362 | return JSON.toJSONString(column_map); 363 | } 364 | 365 | //save data file 366 | private static void save_data_logs(String row_data){ 367 | if (canal_print.equals("true")){ 368 | System.out.println(row_data); 369 | } 370 | 371 | String ts = "yyyyMMdd"; 372 | if (canal_binlog_filename.equals("y")){ 373 | ts = "yyyy"; 374 | }else if (canal_binlog_filename.equals("m")){ 375 | ts = "yyyyMM"; 376 | }else if (canal_binlog_filename.equals("d")){ 377 | ts = "yyyyMMdd"; 378 | }else if (canal_binlog_filename.equals("h")){ 379 | ts = "yyyyMMddHH"; 380 | }else if (canal_binlog_filename.equals("i")){ 381 | ts = "yyyyMMddHHmm"; 382 | }else{ 383 | 384 | } 385 | SimpleDateFormat df2 = new SimpleDateFormat(ts); 386 | String timeStr2 = df2.format(new Date()); 387 | String filename = data_dir + "/binlog_" + timeStr2 + ".log"; 388 | 389 | FileWriter writer; 390 | try { 391 | writer = new FileWriter(filename, true); 392 | writer.write(row_data + "\r\n"); 393 | writer.flush(); 394 | writer.close(); 395 | } catch (IOException e) { 396 | e.printStackTrace(); 397 | System.out.println("write file error!"); 398 | } 399 | } 400 | 401 | /** 402 | // 将信息push 到 rabbitmq 403 | private static void push_rabbitmq(String[] argv) throws java.io.IOException{ 404 | ConnectionFactory factory = new ConnectionFactory(); 405 | factory.setHost(rabbitmq_host); 406 | factory.setPort(Integer.parseInt(rabbitmq_port)); 407 | factory.setUsername(rabbitmq_user); 408 | factory.setPassword(rabbitmq_pass); 409 | 410 | Boolean durable = false; 411 | if(rabbitmq_durable.equals("true")) {durable=true;} 412 | 413 | Connection connection = null; 414 | try { 415 | connection = factory.newConnection(); 416 | } catch (TimeoutException e1) { 417 | // TODO Auto-generated catch block 418 | e1.printStackTrace(); 419 | System.out.println("connection rabbitmq error!"); 420 | } 421 | Channel channel = connection.createChannel(); 422 | channel.queueDeclare(rabbitmq_queuename, durable, false, false, null); 423 | 424 | //String message = getMessage(argv); 425 | for(int i=0;i conf, String[] argv) throws java.io.IOException{ 32 | String rabbitmq_host = conf.get("rabbitmq_host"); 33 | int rabbitmq_port = Integer.parseInt(conf.get("rabbitmq_port")); 34 | String rabbitmq_user = conf.get("rabbitmq_user"); 35 | String rabbitmq_pass = conf.get("rabbitmq_pass"); 36 | String rabbitmq_queuename = conf.get("rabbitmq_queuename"); 37 | String rabbitmq_ack = conf.get("rabbitmq_ack"); 38 | String rabbitmq_durable = conf.get("rabbitmq_durable"); 39 | Boolean durable = false; 40 | if(rabbitmq_durable.equals("true")) {durable=true;} 41 | 42 | ConnectionFactory factory = new ConnectionFactory(); 43 | factory.setHost(rabbitmq_host); 44 | factory.setPort(rabbitmq_port); 45 | factory.setUsername(rabbitmq_user); 46 | factory.setPassword(rabbitmq_pass); 47 | 48 | Connection connection = null; 49 | try { 50 | connection = factory.newConnection(); 51 | } catch (TimeoutException e1) { 52 | // TODO Auto-generated catch block 53 | e1.printStackTrace(); 54 | System.out.println("connection rabbitmq error!"); 55 | } 56 | Channel channel = connection.createChannel(); 57 | channel.queueDeclare(rabbitmq_queuename, durable, false, false, null); 58 | 59 | //String message = getMessage(argv); 60 | for(int i=0;i conf, String[] argv) throws java.io.IOException { 29 | String host = conf.get("host"); 30 | int port = Integer.parseInt(conf.get("port")); 31 | String user = conf.get("user"); 32 | String pass = conf.get("pass"); 33 | String queuename = conf.get("queuename"); 34 | 35 | Jedis jedis = new Jedis(host, port); 36 | for(int i=0;i