├── .gitignore
├── pom.xml
├── readme.md
└── src
    ├── main
        ├── java
        │   └── cn
        │   │   └── zhengyk
        │   │       └── sync
        │   │           ├── Application.java
        │   │           ├── config
        │   │               ├── CanalConfig.java
        │   │               ├── RedisConfig.java
        │   │               └── ThreadPoolConfig.java
        │   │           ├── handler
        │   │               ├── AbstractHandler.java
        │   │               ├── DeleteHandler.java
        │   │               ├── InsertHandler.java
        │   │               └── UpdateHandler.java
        │   │           ├── schedul
        │   │               └── CanalSchedul.java
        │   │           └── utils
        │   │               └── RedisUtil.java
        └── resources
        │   ├── application-dev.yml
        │   ├── application.yml
        │   ├── logback-spring.xml
        │   └── sql
        │       └── blog.sql
    └── test
        └── java
            └── cn
                └── zhengyk
                    └── sync
                        └── ApplicationTests.java


/.gitignore:
--------------------------------------------------------------------------------
 1 | /.idea
 2 | .idea/*
 3 | /.mvn
 4 | .mvn/*
 5 | mvnw
 6 | mvnw.*
 7 | /test
 8 | /target
 9 | /out
10 | /logs
11 | */target/*
12 | *.iml
13 | *.class
14 | .mymetadata
15 | .checkstyle
16 | .classpath
17 | .project
18 | .class
19 | .war
20 | .zip
21 | .rar
22 | *.jar
23 | *.pdb
24 | .pdb
25 | *.log
26 | .log.*
27 | rebel.xml
28 | .DS_Store
29 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 3 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 4 |     <modelVersion>4.0.0</modelVersion>
 5 |     <parent>
 6 |         <groupId>org.springframework.boot</groupId>
 7 |         <artifactId>spring-boot-starter-parent</artifactId>
 8 |         <version>2.0.3.RELEASE</version>
 9 |         <relativePath/> <!-- lookup parent from repository -->
10 |     </parent>
11 |     <groupId>cn.zhengyk</groupId>
12 |     <artifactId>cannal-sync-mysql-redis</artifactId>
13 |     <version>0.0.1-SNAPSHOT</version>
14 |     <name>cannal-sync-mysql-redis</name>
15 |     <description>利用 Canal 同步 mysql 和 redis</description>
16 | 
17 |     <properties>
18 |         <java.version>1.8</java.version>
19 |     </properties>
20 | 
21 |     <dependencies>
22 | 
23 |         <dependency>
24 |             <groupId>org.springframework.boot</groupId>
25 |             <artifactId>spring-boot-starter-test</artifactId>
26 |             <scope>test</scope>
27 |         </dependency>
28 | 
29 |         <!--<dependency>-->
30 |             <!--<groupId>org.springframework.boot</groupId>-->
31 |             <!--<artifactId>spring-boot-starter-web</artifactId>-->
32 |         <!--</dependency>-->
33 | 
34 |         <dependency>
35 |             <groupId>org.springframework.boot</groupId>
36 |             <artifactId>spring-boot-starter-data-redis</artifactId>
37 |         </dependency>
38 | 
39 |         <dependency>
40 |             <groupId>org.springframework.boot</groupId>
41 |             <artifactId>spring-boot-configuration-processor</artifactId>
42 |             <optional>true</optional>
43 |         </dependency>
44 | 
45 |         <!--<dependency>-->
46 |         <!--<groupId>org.mybatis.spring.boot</groupId>-->
47 |         <!--<artifactId>mybatis-spring-boot-starter</artifactId>-->
48 |         <!--<version>1.3.2</version>-->
49 |         <!--</dependency>-->
50 | 
51 |         <!--<dependency>-->
52 |             <!--<groupId>mysql</groupId>-->
53 |             <!--<artifactId>mysql-connector-java</artifactId>-->
54 |             <!--<scope>runtime</scope>-->
55 |         <!--</dependency>-->
56 |         <dependency>
57 |             <groupId>org.projectlombok</groupId>
58 |             <artifactId>lombok</artifactId>
59 |             <optional>true</optional>
60 |         </dependency>
61 | 
62 | 
63 |         <dependency>
64 |             <groupId>com.alibaba</groupId>
65 |             <artifactId>fastjson</artifactId>
66 |             <version>1.2.51</version>
67 |         </dependency>
68 | 
69 |         <!-- https://mvnrepository.com/artifact/com.alibaba.otter/canal.protocol -->
70 |         <dependency>
71 |             <groupId>com.alibaba.otter</groupId>
72 |             <artifactId>canal.protocol</artifactId>
73 |             <version>1.1.2</version>
74 |         </dependency>
75 |         <dependency>
76 |             <groupId>com.alibaba.otter</groupId>
77 |             <artifactId>canal.client</artifactId>
78 |             <version>1.1.2</version>
79 |         </dependency>
80 |     </dependencies>
81 | 
82 |     <build>
83 |         <plugins>
84 |             <plugin>
85 |                 <groupId>org.springframework.boot</groupId>
86 |                 <artifactId>spring-boot-maven-plugin</artifactId>
87 |             </plugin>
88 |         </plugins>
89 |     </build>
90 | 
91 | </project>
92 | 


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
 1 | 利用Canal中间件使MySQL与Redis保持数据同步
 2 | 采用责任链模式，压测过，线程安全。
 3 | 
 4 | 此项目不光可以同步Redis,还可同步 ES、MongoDB、Solr.....套路都是一样的。
 5 | 
 6 | ###安装 Canal Server
 7 | 
 8 | 下载地址：https://github.com/alibaba/canal/releases
 9 | 
10 | ```linux
11 | wget https://github.com/alibaba/canal/releases/download/canal-1.1.2/canal.deployer-1.1.2.tar.gz
12 | ```
13 | 
14 | 下载后解压缩
15 | 
16 | 编辑conf/example/instance.properties
17 | ```properties
18 | #Canal 伪装成子节点的 id，与 master id 不一样
19 | canal.instance.mysql.slaveId = 99
20 | #要订阅的数据库地址
21 | canal.instance.master.address = 127.0.0.1:3306 
22 | canal.instance.master.journal.name =
23 | canal.instance.master.position =
24 | canal.instance.master.timestamp =
25 | 
26 | #Canal 账户，数据库会专门开通一个账户给 Canal 连接
27 | canal.instance.dbUsername = canal
28 | canal.instance.dbPassword = canal
29 | 
30 | ```
31 | 
32 | #### 启动、停止 Canal 服务
33 | 
34 | ```linux
35 | sh bin/start.sh
36 | sh bin/restart.sh
37 | sh bin/stop.sh
38 | ```
39 | 
40 | ### 安装 Redis
41 | 
42 | [http://zhengyk.cn/2017/09/01/redis/Redis01/](http://zhengyk.cn/2017/09/01/redis/Redis01/)
43 | 
44 | ### 项目配置信息
45 | 
46 | ```yml
47 | spring: 
48 |   #Redis配置
49 |   redis:
50 |     host: localhost
51 |     port: 7000
52 |     password: Yakai2018@
53 |     database: 0
54 |     jedis:
55 |       pool:
56 |         #最大连接数（负值表示没有限制）
57 |         max-active: 100
58 |         #最大空闲链接
59 |         max-idle: 10
60 |         #最小空闲链接
61 |         min-idle: 5
62 |         #最大阻塞时间 负值表示不限制
63 |         max-wait: -1ms
64 |         
65 | # canal相关配置
66 | canal:
67 |   host: localhost
68 |   port: 11111
69 |   destination: example
70 |   username:
71 |   password:
72 |   subscribe: test.blog
73 |   batchSize: 1000
74 | # subscribe 过滤规则
75 | # 1）  所有：.*   or  .*\\..*
76 | # 2）  "test"库下所有表： test\\..*
77 | # 3）  "test"下的以"sys"打头的表：test\\.sys.*
78 | # 4）  "test"下的具体一张表：test.blog   blog表
79 | # 5）  多个规则组合使用：test\\..*,test.sys_user,test.sys_role (逗号分隔)        
80 | ```
81 | 
82 | 运行启动项目即可，当我们对数据库进行增、删、改的操作时，Redis 也会相应变化。
83 | 
84 | 


--------------------------------------------------------------------------------
/src/main/java/cn/zhengyk/sync/Application.java:
--------------------------------------------------------------------------------
 1 | package cn.zhengyk.sync;
 2 | 
 3 | //import org.mybatis.spring.annotation.MapperScan;
 4 | import org.springframework.boot.SpringApplication;
 5 | import org.springframework.boot.autoconfigure.SpringBootApplication;
 6 | import org.springframework.scheduling.annotation.EnableScheduling;
 7 | import org.springframework.transaction.annotation.EnableTransactionManagement;
 8 | 
 9 | 
10 | @EnableScheduling     //开启定时任务支持
11 | @SpringBootApplication
12 | @EnableTransactionManagement //开启事务支持
13 | //@MapperScan("cn.zhengyk.sync.dao")
14 | public class Application {
15 | 
16 | 	public static void main(String[] args) {
17 | 		SpringApplication.run(Application.class, args);
18 | 	}
19 | 
20 | }
21 | 
22 | 


--------------------------------------------------------------------------------
/src/main/java/cn/zhengyk/sync/config/CanalConfig.java:
--------------------------------------------------------------------------------
 1 | package cn.zhengyk.sync.config;
 2 | 
 3 | import com.alibaba.otter.canal.client.CanalConnector;
 4 | import com.alibaba.otter.canal.client.CanalConnectors;
 5 | import com.google.common.collect.Lists;
 6 | import lombok.Setter;
 7 | import lombok.extern.slf4j.Slf4j;
 8 | import org.springframework.boot.context.properties.ConfigurationProperties;
 9 | import org.springframework.context.annotation.Bean;
10 | import org.springframework.stereotype.Component;
11 | 
12 | import java.net.InetSocketAddress;
13 | 
14 | /**
15 |  * @author: Yakai Zheng（zhengyk@cloud-young.com）
16 |  * @date: Created on 2018/12/16
17 |  * @description:
18 |  * @version: 1.0
19 |  */
20 | @Slf4j
21 | @Setter
22 | @Component
23 | @ConfigurationProperties("canal")
24 | public class CanalConfig {
25 | 
26 |     private String host;
27 |     private Integer port;
28 |     private String destination;
29 |     private String username;
30 |     private String password;
31 |     private String subscribe;
32 | 
33 |     @Bean
34 |     public CanalConnector getCanalConnector() {
35 |         CanalConnector canalConnector = CanalConnectors.newClusterConnector(Lists.newArrayList(new InetSocketAddress(host, port)), destination, username, password);
36 |         canalConnector.connect();
37 |         // 指定要订阅的数据库和表
38 |         canalConnector.subscribe(subscribe);
39 |         // 回滚到上次中断的位置
40 |         canalConnector.rollback();
41 |         log.info("canal客户端启动......");
42 |         return canalConnector;
43 |     }
44 | }
45 | 


--------------------------------------------------------------------------------
/src/main/java/cn/zhengyk/sync/config/RedisConfig.java:
--------------------------------------------------------------------------------
 1 | package cn.zhengyk.sync.config;
 2 | 
 3 | import org.springframework.beans.factory.annotation.Autowired;
 4 | import org.springframework.context.annotation.Bean;
 5 | import org.springframework.context.annotation.Configuration;
 6 | import org.springframework.data.redis.connection.RedisConnectionFactory;
 7 | import org.springframework.data.redis.core.RedisTemplate;
 8 | import org.springframework.data.redis.serializer.StringRedisSerializer;
 9 | /**
10 |  * @author: Yakai Zheng（zhengyk@cloud-young.com）
11 |  * @date: Created on 2018/12/16
12 |  * @description:
13 |  * @version: 1.0
14 |  */
15 | @Configuration
16 | public class RedisConfig {
17 |     @Autowired
18 |     private RedisConnectionFactory factory;
19 | 
20 |     @Bean
21 |     public RedisTemplate<String, Object> redisTemplate() {
22 |         RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>();
23 |         redisTemplate.setKeySerializer(new StringRedisSerializer());
24 |         redisTemplate.setHashKeySerializer(new StringRedisSerializer());
25 |         redisTemplate.setHashValueSerializer(new StringRedisSerializer());
26 |         redisTemplate.setValueSerializer(new StringRedisSerializer());
27 |         redisTemplate.setConnectionFactory(factory);
28 |         return redisTemplate;
29 |     }
30 | 
31 | }
32 | 


--------------------------------------------------------------------------------
/src/main/java/cn/zhengyk/sync/config/ThreadPoolConfig.java:
--------------------------------------------------------------------------------
 1 | package cn.zhengyk.sync.config;
 2 | 
 3 | import org.springframework.context.annotation.Bean;
 4 | import org.springframework.context.annotation.Configuration;
 5 | import org.springframework.scheduling.annotation.EnableAsync;
 6 | import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 7 | 
 8 | import java.util.concurrent.Executor;
 9 | import java.util.concurrent.ThreadPoolExecutor;
10 | 
11 | /**
12 |  * @author: Yakai Zheng（zhengyk@cloud-young.com）
13 |  * @date: Created on 2018/12/18
14 |  * @description:
15 |  * @version: 1.0
16 |  */
17 | @EnableAsync
18 | @Configuration
19 | public class ThreadPoolConfig {
20 | 
21 |     @Bean("canal")
22 |     public Executor taskExecutor() {
23 |         ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
24 |         executor.setCorePoolSize(10);
25 |         executor.setMaxPoolSize(20);
26 |         executor.setQueueCapacity(200);
27 |         executor.setKeepAliveSeconds(60);
28 |         executor.setThreadNamePrefix("canal-");
29 |         // CallerRunsPolicy 重试添加当前的任务，他会自动重复调用execute()方法
30 |         executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
31 |         return executor;
32 |     }
33 | }
34 | 


--------------------------------------------------------------------------------
/src/main/java/cn/zhengyk/sync/handler/AbstractHandler.java:
--------------------------------------------------------------------------------
 1 | package cn.zhengyk.sync.handler;
 2 | 
 3 | import cn.zhengyk.sync.utils.RedisUtil;
 4 | import com.alibaba.otter.canal.protocol.CanalEntry.Column;
 5 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
 6 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
 7 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
 8 | import com.google.protobuf.InvalidProtocolBufferException;
 9 | import lombok.extern.slf4j.Slf4j;
10 | import org.springframework.beans.factory.annotation.Autowired;
11 | 
12 | import java.util.List;
13 | import java.util.Map;
14 | import java.util.Optional;
15 | import java.util.stream.Collectors;
16 | 
17 | /**
18 |  * @author: Yakai Zheng（zhengyk@cloud-young.com）
19 |  * @date: Created on 2018/12/18
20 |  * @description:   使用责任链模式  链顺序：增->删->改
21 |  * @version: 1.0
22 |  */
23 | @Slf4j
24 | public abstract class AbstractHandler {
25 | 
26 |     /**
27 |      * 下一个执行者
28 |      */
29 |     protected AbstractHandler nextHandler;
30 | 
31 |     /**
32 |      * 事件类型 UPDATE、DELETE、INSERT
33 |      */
34 |     protected EventType eventType;
35 | 
36 |     @Autowired
37 |     protected RedisUtil redisUtil;
38 | 
39 | 
40 |     /**
41 |      * 传递处理的事件
42 |      */
43 |     public void handleMessage(Entry entry){
44 |         if(this.eventType == entry.getHeader().getEventType()){
45 |             //发生写入操作的库名
46 |             String database = entry.getHeader().getSchemaName();
47 |             //发生写入操作的表名
48 |             String table = entry.getHeader().getTableName();
49 |             log.info("监听到数据库：{}，表：{} 的 {} 事件",database,table, eventType.toString());
50 |             //如果 rowChange 不为空,则执行 handleRowChange()
51 |             Optional.ofNullable(this.getRowChange(entry))
52 |                     .ifPresent(this::handleRowChange);
53 |         }else{
54 |             if(nextHandler != null){
55 |                 nextHandler.handleMessage(entry);
56 |             }
57 |         }
58 |     }
59 | 
60 |     /**
61 |      * 处理数据库 UPDATE、DELETE、INSERT 的数据
62 |      */
63 |     public abstract void handleRowChange(RowChange rowChange);
64 | 
65 |     /**
66 |      * 获得数据库 UPDATE、DELETE、INSERT 的数据
67 |      */
68 |     private RowChange getRowChange(Entry entry){
69 |         RowChange rowChange = null;
70 |         try {
71 |             rowChange = RowChange.parseFrom(entry.getStoreValue());
72 |         } catch (InvalidProtocolBufferException e) {
73 |             log.error("根据CanalEntry获取RowChange异常:", e);
74 |         }
75 |         return rowChange;
76 |     }
77 | 
78 |     /**
79 |      * columns 转 map
80 |      */
81 |     protected Map<String,String> columnsToMap(List<Column> columns) {
82 |         return columns.stream().collect(Collectors.toMap(Column::getName, Column::getValue));
83 |     }
84 | 
85 | 
86 | }
87 | 


--------------------------------------------------------------------------------
/src/main/java/cn/zhengyk/sync/handler/DeleteHandler.java:
--------------------------------------------------------------------------------
 1 | package cn.zhengyk.sync.handler;
 2 | 
 3 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
 4 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
 5 | import lombok.extern.slf4j.Slf4j;
 6 | import org.springframework.beans.factory.annotation.Autowired;
 7 | import org.springframework.stereotype.Component;
 8 | 
 9 | /**
10 |  * @author: Yakai Zheng（zhengyk@cloud-young.com）
11 |  * @date: Created on 2018/12/18
12 |  * @description:  监听到数据库删除操作后执行的逻辑
13 |  * @version: 1.0
14 |  */
15 | @Slf4j
16 | @Component
17 | public class DeleteHandler extends AbstractHandler {
18 | 
19 |     public DeleteHandler(){
20 |         eventType = EventType.DELETE;
21 |     }
22 | 
23 |     @Autowired
24 |     public void setNextHandler(UpdateHandler updateHandler) {
25 |         this.nextHandler = updateHandler;
26 |     }
27 | 
28 |     @Override
29 |     public void handleRowChange(RowChange rowChange) {
30 |         rowChange.getRowDatasList().forEach(rowData -> {
31 |             rowData.getBeforeColumnsList().forEach(column -> {
32 |                 if("id".equals(column.getName())){
33 |                     //清除 redis 缓存
34 |                     log.info("清除 Redis 缓存 key={} 成功!\r\n","blog:"+column.getValue());
35 |                     redisUtil.del("blog:"+column.getValue());
36 |                 }
37 |             });
38 |         });
39 |     }
40 | }
41 | 


--------------------------------------------------------------------------------
/src/main/java/cn/zhengyk/sync/handler/InsertHandler.java:
--------------------------------------------------------------------------------
 1 | package cn.zhengyk.sync.handler;
 2 | 
 3 | import com.alibaba.fastjson.JSONObject;
 4 | import com.alibaba.otter.canal.protocol.CanalEntry.Column;
 5 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
 6 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
 7 | import lombok.extern.slf4j.Slf4j;
 8 | import org.springframework.beans.factory.annotation.Autowired;
 9 | import org.springframework.stereotype.Component;
10 | 
11 | import java.util.List;
12 | import java.util.Map;
13 | 
14 | /**
15 |  * @author: Yakai Zheng（zhengyk@cloud-young.com）
16 |  * @date: Created on 2018/12/18
17 |  * @description: 监听到数据库插入操作后执行的逻辑
18 |  * @version: 1.0
19 |  */
20 | @Slf4j
21 | @Component
22 | public class InsertHandler extends AbstractHandler{
23 | 
24 |     public InsertHandler(){
25 |         this.eventType = EventType.INSERT;
26 |     }
27 | 
28 |     @Autowired
29 |     public void setNextHandler(DeleteHandler deleteHandler) {
30 |         this.nextHandler = deleteHandler;
31 |     }
32 | 
33 |     @Override
34 |     public void handleRowChange(RowChange rowChange) {
35 |         rowChange.getRowDatasList().forEach(rowData -> {
36 |             //每一行的每列数据  字段名->值
37 |             List<Column> afterColumnsList = rowData.getAfterColumnsList();
38 |             Map<String, String> map = super.columnsToMap(afterColumnsList);
39 |             String id = map.get("id");
40 |             String jsonStr = JSONObject.toJSONString(map);
41 |             log.info("新增的数据：{}\r\n",jsonStr);
42 |             redisUtil.setDefault("blog:"+id, jsonStr);
43 |         });
44 |     }
45 | 
46 | 
47 | }
48 | 


--------------------------------------------------------------------------------
/src/main/java/cn/zhengyk/sync/handler/UpdateHandler.java:
--------------------------------------------------------------------------------
 1 | package cn.zhengyk.sync.handler;
 2 | 
 3 | import com.alibaba.fastjson.JSONObject;
 4 | import com.alibaba.otter.canal.protocol.CanalEntry.Column;
 5 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
 6 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
 7 | import lombok.extern.slf4j.Slf4j;
 8 | import org.springframework.stereotype.Component;
 9 | 
10 | import java.util.List;
11 | import java.util.Map;
12 | 
13 | /**
14 |  * @author: Yakai Zheng（zhengyk@cloud-young.com）
15 |  * @date: Created on 2018/12/18
16 |  * @description:  监听到数据库更新操作后执行的逻辑
17 |  * @version: 1.0
18 |  */
19 | @Slf4j
20 | @Component
21 | public class UpdateHandler extends AbstractHandler {
22 | 
23 |     public UpdateHandler(){
24 |         this.eventType = EventType.UPDATE;
25 |     }
26 | 
27 | 
28 |     @Override
29 |     public void handleRowChange(RowChange rowChange) {
30 |         rowChange.getRowDatasList().forEach(rowData -> {
31 |             //每一行的每列数据  字段名->值
32 |             List<Column> beforeColumnsList = rowData.getBeforeColumnsList();
33 |             Map<String, String> beforeMap = super.columnsToMap(beforeColumnsList);
34 |             log.info("更新前数据：{}",JSONObject.toJSONString(beforeMap));
35 | 
36 |             List<Column> afterColumnsList = rowData.getAfterColumnsList();
37 |             Map<String, String> afterMap = super.columnsToMap(afterColumnsList);
38 |             String id = afterMap.get("id");
39 |             String afterJsonStr = JSONObject.toJSONString(afterMap);
40 |             log.info("更新后数据：{}\r\n",afterJsonStr);
41 |             /**
42 |              *  高并发下，为保证数据一致性，当数据库更新后，不建议去更新缓存，
43 |              *  而是建议直接删除缓存，由查询时再设置到缓存。
44 |              *  这里为了演示，对缓存作更新操作，具体看业务需求。
45 |              */
46 |             redisUtil.setDefault("blog:"+id,afterJsonStr);
47 | 
48 |         });
49 |     }
50 | }
51 | 


--------------------------------------------------------------------------------
/src/main/java/cn/zhengyk/sync/schedul/CanalSchedul.java:
--------------------------------------------------------------------------------
 1 | package cn.zhengyk.sync.schedul;
 2 | 
 3 | import cn.zhengyk.sync.handler.InsertHandler;
 4 | import com.alibaba.otter.canal.client.CanalConnector;
 5 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
 6 | import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
 7 | import com.alibaba.otter.canal.protocol.Message;
 8 | import lombok.extern.slf4j.Slf4j;
 9 | import org.springframework.beans.factory.annotation.Autowired;
10 | import org.springframework.beans.factory.annotation.Value;
11 | import org.springframework.scheduling.annotation.Async;
12 | import org.springframework.scheduling.annotation.Scheduled;
13 | import org.springframework.stereotype.Component;
14 | 
15 | import java.util.List;
16 | 
17 | /**
18 |  * @author: Yakai Zheng（zhengyk@cloud-young.com）
19 |  * @date: Created on 2018/12/16
20 |  * @description:
21 |  * @version: 1.0
22 |  */
23 | @Slf4j
24 | @Component
25 | public class CanalSchedul{
26 | 
27 |     @Autowired
28 |     private CanalConnector canalConnector;
29 | 
30 |     @Autowired
31 |     private InsertHandler insertHandler;
32 | 
33 |     @Value("${canal.batchSize}")
34 |     private int batchSize;
35 | 
36 |     @Async("canal")
37 |     @Scheduled(fixedDelay = 200)  //每200毫秒拉取一次数据
38 |     public void fetch() {
39 |         try {
40 |             Message message = canalConnector.getWithoutAck(batchSize);
41 |             long batchId = message.getId();
42 |             log.debug("batchId={}" , batchId);
43 |             try {
44 |                 List<Entry> entries = message.getEntries();
45 |                 if (batchId != -1 && entries.size() > 0) {
46 |                     entries.forEach(entry -> {
47 |                         if (entry.getEntryType() == EntryType.ROWDATA) {
48 |                             insertHandler.handleMessage(entry);
49 |                         }
50 |                     });
51 |                 }
52 |                 canalConnector.ack(batchId);
53 |             } catch (Exception e) {
54 |                 log.error("批量获取 mysql 同步信息失败，batchId回滚,batchId=" + batchId, e);
55 |                 canalConnector.rollback(batchId);
56 |             }
57 |         } catch (Exception e) {
58 |             log.error("Canal定时任务异常！", e);
59 |         }
60 |     }
61 | }
62 | 


--------------------------------------------------------------------------------
/src/main/java/cn/zhengyk/sync/utils/RedisUtil.java:
--------------------------------------------------------------------------------
  1 | package cn.zhengyk.sync.utils;
  2 | 
  3 | import org.springframework.beans.factory.annotation.Autowired;
  4 | import org.springframework.data.redis.core.RedisTemplate;
  5 | import org.springframework.stereotype.Component;
  6 | import org.springframework.util.CollectionUtils;
  7 | 
  8 | import java.util.concurrent.TimeUnit;
  9 | 
 10 | /**
 11 |  * Redis 工具类
 12 |  * @author Yakai Zheng
 13 |  */
 14 | @Component
 15 | public final class RedisUtil {
 16 | 
 17 | 	/**
 18 | 	 * 默认过期时长，单位：秒
 19 | 	 */
 20 | 	public final static long DEFAULT_EXPIRE = 1800;
 21 | 
 22 | 	@Autowired
 23 | 	private RedisTemplate<String, Object> redisTemplate;
 24 | 
 25 | 	// =============================common============================
 26 | 
 27 | 	/**
 28 | 	 * 指定 key 失效时间
 29 | 	 * @param key 键
 30 | 	 * @param seconds 时间(秒)
 31 | 	 * @return
 32 | 	 */
 33 | 	public void expire(String key, long seconds) {
 34 | 		if (seconds > 0) {
 35 | 			redisTemplate.expire(key, seconds, TimeUnit.SECONDS);
 36 | 		}
 37 | 	}
 38 | 
 39 | 	/**
 40 | 	 * 指定 key 默认失效时间
 41 | 	 * @param key 键
 42 | 	 * @return
 43 | 	 */
 44 | 	public void expireDefault(String key) {
 45 | 		this.expire(key, DEFAULT_EXPIRE);
 46 | 	}
 47 | 
 48 | 	/**
 49 | 	 * 根据key 获取过期时间
 50 | 	 * @param key 键 不能为null
 51 | 	 * @return 时间(秒) 返回0代表为永久有效
 52 | 	 */
 53 | 	public long getExpire(String key) {
 54 | 		return redisTemplate.getExpire(key, TimeUnit.SECONDS);
 55 | 	}
 56 | 	/**
 57 | 	 * 判断key是否存在
 58 | 	 * @param key 键
 59 | 	 * @return true 存在 false不存在
 60 | 	 */
 61 | 	public boolean hasKey(String key) {
 62 | 		return redisTemplate.hasKey(key);
 63 | 	}
 64 | 	/**
 65 | 	 * 删除key
 66 | 	 * @param key 可以传一个值 或多个
 67 | 	 */
 68 | 	public void del(String... key) {
 69 | 		if (key != null && key.length > 0) {
 70 | 			if (key.length == 1) {
 71 | 				redisTemplate.delete(key[0]);
 72 | 			} else {
 73 | 				redisTemplate.delete(CollectionUtils.arrayToList(key));
 74 | 			}
 75 | 		}
 76 | 	}
 77 | 	// ============================String=============================
 78 | 	/**
 79 | 	 * 获取 key 的 value
 80 | 	 * @param key 键
 81 | 	 * @return 值
 82 | 	 */
 83 | 	public Object get(String key) {
 84 | 		return key == null ? null : redisTemplate.opsForValue().get(key);
 85 | 	}
 86 | 	/**
 87 | 	 * 永久 set
 88 | 	 * @param key 键
 89 | 	 * @param value 值
 90 | 	 * @return true成功 false失败
 91 | 	 */
 92 | 	public void set(String key, Object value) {
 93 | 		redisTemplate.opsForValue().set(key, value);
 94 | 	}
 95 | 
 96 | 	/**
 97 | 	 * 默认set 默认实效时间 DEFAULT_EXPIRE
 98 | 	 * @param key 键
 99 | 	 * @param value 值
100 | 	 * @return true成功 false失败
101 | 	 */
102 | 	public void setDefault(String key, Object value) {
103 | 		this.set(key, value, DEFAULT_EXPIRE);
104 | 	}
105 | 
106 | 	/**
107 | 	 * 普通缓存放入并设置时间
108 | 	 * @param key 键
109 | 	 * @param value 值
110 | 	 * @param seconds 时间(秒) time要大于0 如果time小于等于0 将设置无限期
111 | 	 * @return true成功 false 失败
112 | 	 */
113 | 	public void set(String key, Object value, long seconds) {
114 | 		if (seconds > 0) {
115 | 			redisTemplate.opsForValue().set(key, value, seconds, TimeUnit.SECONDS);
116 | 		} else {
117 | 			set(key, value);
118 | 		}
119 | 	}
120 | 	/**
121 | 	 * 递增
122 | 	 * @param key 键
123 | 	 * @param delta 要增加几(大于0)
124 | 	 * @return
125 | 	 */
126 | 	public long incr(String key, long delta) {
127 | 		if (delta <= 0) {
128 | 			throw new RuntimeException("递增因子必须大于0");
129 | 		}
130 | 		return redisTemplate.opsForValue().increment(key, delta);
131 | 	}
132 | 	/**
133 | 	 * 递减
134 | 	 * @param key 键
135 | 	 * @param delta 要减少几(小于0)
136 | 	 * @return
137 | 	 */
138 | 	public long decr(String key, long delta) {
139 | 		if (delta <= 0) {
140 | 			throw new RuntimeException("递减因子必须大于0");
141 | 		}
142 | 		return redisTemplate.opsForValue().increment(key, -delta);
143 | 	}
144 | }


--------------------------------------------------------------------------------
/src/main/resources/application-dev.yml:
--------------------------------------------------------------------------------
 1 | spring:
 2 | #  datasource:
 3 | #    #使用springboot2.0后内置的HikariCP
 4 | #    driver-class-name: com.mysql.jdbc.Driver
 5 | #    url: jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=UTF-8
 6 | #    username: yakai
 7 | #    password: Yakai2018@
 8 | #    type: com.zaxxer.hikari.HikariDataSource
 9 | #    hikari:
10 | #      minimum-idle: 5
11 | #      maximum-pool-size: 15
12 | #      auto-commit: true
13 | #      idle-timeout: 30000
14 | #      pool-name: HikariCP
15 | #      max-lifetime: 1800000
16 | #      connection-timeout: 30000
17 | #      connection-test-query: SELECT 1
18 |   #Redis配置
19 |   redis:
20 |     host: localhost
21 |     port: 7000
22 |     password: Yakai2018@
23 |     database: 0
24 |     jedis:
25 |       pool:
26 |         #最大连接数（负值表示没有限制）
27 |         max-active: 100
28 |         #最大空闲链接
29 |         max-idle: 10
30 |         #最小空闲链接
31 |         min-idle: 5
32 |         #最大阻塞时间 负值表示不限制
33 |         max-wait: -1ms
34 | 
35 | 
36 | 
37 | #mybatis:
38 | #  mapper-locations: classpath:sqlmap/*.xml
39 | #  type-aliases-package: cn.zhengyk.sync.model
40 | 
41 | # canal相关配置
42 | canal:
43 |   host: localhost
44 |   port: 11111
45 |   destination: example
46 |   username:
47 |   password:
48 |   subscribe: test.blog
49 |   batchSize: 1000
50 | # subscribe 过滤规则
51 | # 1）  所有：.*   or  .*\\..*
52 | # 2）  "test"库下所有表： test\\..*
53 | # 3）  "test"下的以"sys"打头的表：test\\.sys.*
54 | # 4）  "test"下的具体一张表：test.blog   blog表
55 | # 5）  多个规则组合使用：test\\..*,test.sys_user,test.sys_role (逗号分隔)
56 | 
57 | 


--------------------------------------------------------------------------------
/src/main/resources/application.yml:
--------------------------------------------------------------------------------
 1 | spring:
 2 |   application:
 3 |     name: cannal-sync-redis
 4 |   profiles:
 5 |     active: dev
 6 | server:
 7 |   port: 22222
 8 | 
 9 | 
10 | 


--------------------------------------------------------------------------------
/src/main/resources/logback-spring.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <configuration>
 3 | 
 4 |     <!--定义日志文件的存储地址-->
 5 |     <property name="LOG_HOME" value="logs/canal" />
 6 |     <!-- 控制台输出 -->
 7 |     <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
 8 |         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
 9 |             <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符-->
10 |             <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{50}) %L - %msg %n</pattern>
11 |             <charset>UTF-8</charset>
12 |         </encoder>
13 |     </appender>
14 |     <!-- 按照每天生成日志文件 -->
15 |     <appender name="FILE"  class="ch.qos.logback.core.rolling.RollingFileAppender">
16 |         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
17 |             <!--日志文件输出的文件名-->
18 |             <FileNamePattern>${LOG_HOME}/%d{yyyy-MM-dd}/%d{yyyy-MM-dd}.log</FileNamePattern>
19 |             <!--日志文件保留天数-->
20 |             <MaxHistory>5</MaxHistory>
21 |         </rollingPolicy>
22 |         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
23 |             <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} %L - %msg%n</pattern>
24 |             <charset>UTF-8</charset>
25 |         </encoder>
26 |         <!--日志文件最大的大小-->
27 |         <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
28 |             <MaxFileSize>1GB</MaxFileSize>
29 |         </triggeringPolicy>
30 |     </appender>
31 |     <springProfile name="dev,test,prod">
32 |         <root level="INFO">
33 |             <appender-ref ref="CONSOLE" />
34 |             <appender-ref ref="FILE" />
35 |         </root>
36 |         <logger name="cn.zhengyk.sync.dao" level="DEBUG" />
37 |     </springProfile>
38 | 
39 |     <!--<springProfile name="prod">-->
40 |         <!--<root level="INFO">-->
41 |             <!--<appender-ref ref="CONSOLE" />-->
42 |             <!--<appender-ref ref="FILE" />-->
43 |         <!--</root>-->
44 |         <!--<logger name="cn.zhengyk.sync.dao" level="DEBUG" />-->
45 |     <!--</springProfile>-->
46 | 
47 | </configuration>


--------------------------------------------------------------------------------
/src/main/resources/sql/blog.sql:
--------------------------------------------------------------------------------
 1 | 
 2 | SET NAMES utf8mb4;
 3 | SET FOREIGN_KEY_CHECKS = 0;
 4 | DROP TABLE IF EXISTS `blog`;
 5 | CREATE TABLE `blog` (
 6 |   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
 7 |   `author` varchar(20) NOT NULL DEFAULT '',
 8 |   `title` varchar(100) NOT NULL,
 9 |   `subtitle` varchar(100) NOT NULL,
10 |   `content` longtext NOT NULL,
11 |   `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
12 |   `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
13 |   PRIMARY KEY (`id`)
14 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
15 | 
16 | SET FOREIGN_KEY_CHECKS = 1;
17 | 


--------------------------------------------------------------------------------
/src/test/java/cn/zhengyk/sync/ApplicationTests.java:
--------------------------------------------------------------------------------
 1 | package cn.zhengyk.sync;
 2 | 
 3 | import org.junit.Test;
 4 | import org.junit.runner.RunWith;
 5 | import org.springframework.boot.test.context.SpringBootTest;
 6 | import org.springframework.test.context.junit4.SpringRunner;
 7 | 
 8 | @RunWith(SpringRunner.class)
 9 | @SpringBootTest
10 | public class ApplicationTests {
11 | 
12 |     @Test
13 |     public void contextLoads() {
14 |     }
15 | 
16 | }
17 | 
18 | 


--------------------------------------------------------------------------------