├── .gitattributes ├── .gitignore ├── src ├── main │ ├── resources │ │ ├── mapping.properties │ │ ├── logback.xml │ │ ├── application.properties │ │ ├── mybatis │ │ │ ├── mybatis-config.xml │ │ │ └── mapper │ │ │ │ └── BaseMapper.xml │ │ └── static │ │ │ └── index.html │ ├── java │ │ └── com │ │ │ └── star │ │ │ └── sync │ │ │ └── elasticsearch │ │ │ ├── service │ │ │ ├── SyncService.java │ │ │ ├── TransactionalService.java │ │ │ ├── ElasticsearchService.java │ │ │ ├── MappingService.java │ │ │ └── impl │ │ │ │ ├── TransactionalServiceImpl.java │ │ │ │ ├── ElasticsearchServiceImpl.java │ │ │ │ ├── SyncServiceImpl.java │ │ │ │ └── MappingServiceImpl.java │ │ │ ├── event │ │ │ ├── DeleteAbstractCanalEvent.java │ │ │ ├── InsertAbstractCanalEvent.java │ │ │ ├── UpdateAbstractCanalEvent.java │ │ │ └── AbstractCanalEvent.java │ │ │ ├── CanalMysqlElasticsearchSyncApplication.java │ │ │ ├── controller │ │ │ ├── handler │ │ │ │ └── ControllerHandler.java │ │ │ └── SyncController.java │ │ │ ├── model │ │ │ ├── request │ │ │ │ └── SyncByTableRequest.java │ │ │ ├── response │ │ │ │ └── Response.java │ │ │ ├── IndexTypeModel.java │ │ │ └── DatabaseTableModel.java │ │ │ ├── dao │ │ │ └── BaseDao.java │ │ │ ├── client │ │ │ ├── ElasticsearchClient.java │ │ │ └── CanalClient.java │ │ │ ├── listener │ │ │ ├── DeleteCanalListener.java │ │ │ ├── UpdateCanalListener.java │ │ │ ├── InsertCanalListener.java │ │ │ └── AbstractCanalListener.java │ │ │ ├── scheduling │ │ │ └── CanalScheduling.java │ │ │ └── util │ │ │ └── JsonUtil.java │ ├── bin │ │ ├── stop.sh │ │ └── startup.sh │ └── assembly │ │ ├── dev.xml │ │ └── release.xml └── test │ └── java │ └── com │ └── star │ └── sync │ └── elasticsearch │ ├── CanalMysqlElasticsearchSyncApplicationTests.java │ ├── service │ └── impl │ │ └── ElasticsearchServiceImplTest.java │ └── CommonTest.java ├── README.md ├── pom.xml └── LICENSE /.gitattributes: -------------------------------------------------------------------------------- 1 | *.html linguist-language=Java -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | !.mvn/wrapper/maven-wrapper.jar 3 | 4 | ### STS ### 5 | .apt_generated 6 | .classpath 7 | .factorypath 8 | .project 9 | .settings 10 | .springBeans 11 | 12 | ### IntelliJ IDEA ### 13 | .idea 14 | *.iws 15 | *.iml 16 | *.ipr 17 | 18 | ### NetBeans ### 19 | nbproject/private/ 20 | build/ 21 | nbbuild/ 22 | dist/ 23 | nbdist/ 24 | .nb-gradle/ -------------------------------------------------------------------------------- /src/main/resources/mapping.properties: -------------------------------------------------------------------------------- 1 | # 配置数据库和Elasticsearch所对应的映射关系 2 | # 格式：dbEsMapping[${database}.${table}]=${index}.${type} 3 | dbEsMapping[test_info.user]=test_info.user 4 | dbEsMapping[test_info.admin]=test_info2.admin 5 | 6 | # 配置数据库主键列名，默认为id 7 | # 格式：tablePrimaryKeyMap[${database}.${table}]=${primaryKey} 8 | tablePrimaryKeyMap[test_info.user]=id 9 | tablePrimaryKeyMap[test_info.admin]=pk -------------------------------------------------------------------------------- /src/test/java/com/star/sync/elasticsearch/CanalMysqlElasticsearchSyncApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.test.context.junit4.SpringRunner; 7 | 8 | @RunWith(SpringRunner.class) 9 | @SpringBootTest 10 | public class CanalMysqlElasticsearchSyncApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/service/SyncService.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.service; 2 | 3 | import com.star.sync.elasticsearch.model.request.SyncByTableRequest; 4 | 5 | /** 6 | * @author wangchao 7 | * @version 1.0 8 | * @since 2017-08-31 17:48:00 9 | */ 10 | public interface SyncService { 11 | /** 12 | * 通过database和table同步数据库 13 | * 14 | * @param request 请求参数 15 | * @return 后台同步进程执行成功与否 16 | */ 17 | boolean syncByTable(SyncByTableRequest request); 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/service/TransactionalService.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.service; 2 | 3 | import com.star.sync.elasticsearch.model.IndexTypeModel; 4 | import com.star.sync.elasticsearch.model.request.SyncByTableRequest; 5 | 6 | /** 7 | * @author wangchao 8 | * @version 1.0.0 9 | * @since 2018-05-21 23:22:00 10 | */ 11 | public interface TransactionalService { 12 | 13 | /** 14 | * 开启事务的读取mysql并插入到Elasticsearch中（读锁） 15 | */ 16 | void batchInsertElasticsearch(SyncByTableRequest request, String primaryKey, long from, long to, IndexTypeModel indexTypeModel); 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/event/DeleteAbstractCanalEvent.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.event; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 4 | 5 | /** 6 | * @author wangchao 7 | * @version 1.0 8 | * @since 2017-08-26 22:31:00 9 | */ 10 | public class DeleteAbstractCanalEvent extends AbstractCanalEvent { 11 | /** 12 | * Create a new ApplicationEvent. 13 | * 14 | * @param source the object on which the event initially occurred (never {@code null}) 15 | */ 16 | public DeleteAbstractCanalEvent(Entry source) { 17 | super(source); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/event/InsertAbstractCanalEvent.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.event; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 4 | 5 | /** 6 | * @author wangchao 7 | * @version 1.0 8 | * @since 2017-08-26 22:30:00 9 | */ 10 | public class InsertAbstractCanalEvent extends AbstractCanalEvent { 11 | /** 12 | * Create a new ApplicationEvent. 13 | * 14 | * @param source the object on which the event initially occurred (never {@code null}) 15 | */ 16 | public InsertAbstractCanalEvent(Entry source) { 17 | super(source); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/event/UpdateAbstractCanalEvent.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.event; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 4 | 5 | /** 6 | * @author wangchao 7 | * @version 1.0 8 | * @since 2017-08-26 22:30:00 9 | */ 10 | public class UpdateAbstractCanalEvent extends AbstractCanalEvent { 11 | /** 12 | * Create a new ApplicationEvent. 13 | * 14 | * @param source the object on which the event initially occurred (never {@code null}) 15 | */ 16 | public UpdateAbstractCanalEvent(Entry source) { 17 | super(source); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/service/ElasticsearchService.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.service; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * @author wangchao 7 | * @version 1.0 8 | * @since 2017-08-26 16:19:00 9 | */ 10 | public interface ElasticsearchService { 11 | void insertById(String index, String type, String id, Map dataMap); 12 | 13 | void batchInsertById(String index, String type, Map> idDataMap); 14 | 15 | void update(String index, String type, String id, Map dataMap); 16 | 17 | void deleteById(String index, String type, String id); 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/event/AbstractCanalEvent.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.event; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 4 | import org.springframework.context.ApplicationEvent; 5 | 6 | /** 7 | * @author wangchao 8 | * @version 1.0 9 | * @since 2017-08-26 22:22:00 10 | */ 11 | public abstract class AbstractCanalEvent extends ApplicationEvent { 12 | 13 | /** 14 | * Create a new ApplicationEvent. 15 | * 16 | * @param source the object on which the event initially occurred (never {@code null}) 17 | */ 18 | public AbstractCanalEvent(Entry source) { 19 | super(source); 20 | } 21 | 22 | public Entry getEntry() { 23 | return (Entry) source; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/CanalMysqlElasticsearchSyncApplication.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch; 2 | 3 | import org.mybatis.spring.annotation.MapperScan; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.scheduling.annotation.EnableScheduling; 7 | import org.springframework.transaction.annotation.EnableTransactionManagement; 8 | 9 | /** 10 | * @author wangchao 11 | * @version 1.0 12 | * @since 2017-08-25 17:26:00 13 | */ 14 | @SpringBootApplication 15 | @EnableScheduling 16 | @EnableTransactionManagement 17 | @MapperScan("com.star.sync.elasticsearch.dao") 18 | public class CanalMysqlElasticsearchSyncApplication { 19 | public static void main(String[] args) { 20 | SpringApplication.run(CanalMysqlElasticsearchSyncApplication.class, args); 21 | } 22 | } -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | [%d{yyyy-MM-dd HH:mm:ss.SSS}] [%level] [%thread] [%logger{36}] --- %msg%n 6 | 7 | 8 | 9 | ../logs/info.log 10 | 11 | 12 | ../logs/info.log.%d{yyyy-MM-dd} 13 | 300 14 | 15 | 16 | [%d{yyyy-MM-dd HH:mm:ss.SSS}] [%level] [%thread] [%logger{36}] --- %msg%n 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # 服务名 2 | spring.application.name=canal_mysql_elasticsearch_sync 3 | # 服务端口号 4 | server.port=8828 5 | 6 | spring.datasource.driver-class-name=com.mysql.jdbc.Driver 7 | # 数据库配置，不用写database名 8 | spring.datasource.url=jdbc:mysql://127.0.0.1:3306?characterSet=utf8mb4&useSSL=false 9 | # 账号 10 | spring.datasource.username=root 11 | # 密码 12 | spring.datasource.password=123456 13 | spring.datasource.dbcp2.max-idle=10 14 | spring.datasource.dbcp2.min-idle=5 15 | spring.datasource.dbcp2.initial-size=2 16 | spring.datasource.dbcp2.validation-query=SELECT 1 17 | spring.datasource.dbcp2.test-while-idle=true 18 | 19 | mybatis.config-location=classpath:mybatis/mybatis-config.xml 20 | mybatis.mapper-locations=classpath:mybatis/mapper/*.xml 21 | 22 | # canal配置 23 | canal.host=127.0.0.1 24 | canal.port=11111 25 | canal.destination=example 26 | canal.username= 27 | canal.password= 28 | 29 | # elasticsearch配置 30 | elasticsearch.cluster.name=my-elasticsearch 31 | elasticsearch.host=127.0.0.1 32 | elasticsearch.port=9300 -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/controller/handler/ControllerHandler.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.controller.handler; 2 | 3 | import com.star.sync.elasticsearch.model.response.Response; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.web.bind.annotation.ControllerAdvice; 7 | import org.springframework.web.bind.annotation.ExceptionHandler; 8 | import org.springframework.web.bind.annotation.ResponseBody; 9 | 10 | import javax.servlet.http.HttpServletResponse; 11 | 12 | /** 13 | * @author wangchao 14 | * @version 1.0 15 | * @since 2017-08-31 20:55:00 16 | */ 17 | @ControllerAdvice 18 | public class ControllerHandler { 19 | private static final Logger logger = LoggerFactory.getLogger(ControllerHandler.class); 20 | 21 | @ExceptionHandler 22 | @ResponseBody 23 | public Object exceptionHandler(Exception e, HttpServletResponse response) { 24 | logger.error("unknown_error", e); 25 | return new Response<>(2, e.getMessage(), null).toString(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/resources/mybatis/mybatis-config.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/test/java/com/star/sync/elasticsearch/service/impl/ElasticsearchServiceImplTest.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.service.impl; 2 | 3 | import com.google.common.collect.Maps; 4 | import org.elasticsearch.client.transport.TransportClient; 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.springframework.boot.test.context.SpringBootTest; 8 | import org.springframework.test.context.junit4.SpringRunner; 9 | 10 | import javax.annotation.Resource; 11 | 12 | import java.time.LocalDateTime; 13 | import java.util.Date; 14 | import java.util.Map; 15 | 16 | import static org.junit.Assert.*; 17 | 18 | /** 19 | * @author wangchao 20 | * @version 1.0 21 | * @since 2017-09-01 15:01:00 22 | */ 23 | @RunWith(SpringRunner.class) 24 | @SpringBootTest 25 | public class ElasticsearchServiceImplTest { 26 | 27 | @Resource 28 | private TransportClient transportClient; 29 | 30 | @Test 31 | public void insertById() throws Exception { 32 | Map map = Maps.newHashMap(); 33 | map.put("id", 111); 34 | map.put("name", "name"); 35 | map.put("date", LocalDateTime.now()); 36 | transportClient.prepareIndex("test", "test", "888").setSource(map).get(); 37 | } 38 | 39 | } -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/model/request/SyncByTableRequest.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.model.request; 2 | 3 | import org.hibernate.validator.constraints.NotBlank; 4 | 5 | /** 6 | * @author wangchao 7 | * @version 1.0 8 | * @since 2017-08-31 17:52:00 9 | */ 10 | public class SyncByTableRequest { 11 | @NotBlank 12 | private String database; 13 | @NotBlank 14 | private String table; 15 | private Integer stepSize = 500; 16 | private Long from; 17 | private Long to; 18 | 19 | public Long getFrom() { 20 | return from; 21 | } 22 | 23 | public void setFrom(Long from) { 24 | this.from = from; 25 | } 26 | 27 | public Long getTo() { 28 | return to; 29 | } 30 | 31 | public void setTo(Long to) { 32 | this.to = to; 33 | } 34 | 35 | public String getDatabase() { 36 | return database; 37 | } 38 | 39 | public void setDatabase(String database) { 40 | this.database = database; 41 | } 42 | 43 | public String getTable() { 44 | return table; 45 | } 46 | 47 | public void setTable(String table) { 48 | this.table = table; 49 | } 50 | 51 | public Integer getStepSize() { 52 | return stepSize; 53 | } 54 | 55 | public void setStepSize(Integer stepSize) { 56 | this.stepSize = stepSize; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/model/response/Response.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.model.response; 2 | 3 | import com.star.sync.elasticsearch.util.JsonUtil; 4 | 5 | /** 6 | * @author wangchao 7 | * @version 1.0 8 | * @since 2017-08-31 17:58:00 9 | */ 10 | public class Response { 11 | private int code = 0; 12 | private String message = "SUCCESS"; 13 | private T data; 14 | 15 | public Response(int code, String message, T data) { 16 | this.code = code; 17 | this.message = message; 18 | this.data = data; 19 | } 20 | 21 | public Response(T data) { 22 | this.data = data; 23 | } 24 | 25 | public static Response success(T result) { 26 | return new Response<>(result); 27 | } 28 | 29 | public int getCode() { 30 | return code; 31 | } 32 | 33 | public void setCode(int code) { 34 | this.code = code; 35 | } 36 | 37 | public String getMessage() { 38 | return message; 39 | } 40 | 41 | public void setMessage(String message) { 42 | this.message = message; 43 | } 44 | 45 | public T getData() { 46 | return data; 47 | } 48 | 49 | public void setData(T data) { 50 | this.data = data; 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | return JsonUtil.toJson(this); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/model/IndexTypeModel.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.model; 2 | 3 | import com.google.common.base.Objects; 4 | 5 | /** 6 | * @author wangchao 7 | * @version 1.0 8 | * @since 2017-08-26 22:57:00 9 | */ 10 | public class IndexTypeModel { 11 | private String index; 12 | private String type; 13 | 14 | public IndexTypeModel() { 15 | } 16 | 17 | public IndexTypeModel(String index, String type) { 18 | this.index = index; 19 | this.type = type; 20 | } 21 | 22 | public String getIndex() { 23 | return index; 24 | } 25 | 26 | public void setIndex(String index) { 27 | this.index = index; 28 | } 29 | 30 | public String getType() { 31 | return type; 32 | } 33 | 34 | public void setType(String type) { 35 | this.type = type; 36 | } 37 | 38 | @Override 39 | public boolean equals(Object o) { 40 | if (this == o) { 41 | return true; 42 | } 43 | if (o == null || getClass() != o.getClass()) { 44 | return false; 45 | } 46 | IndexTypeModel that = (IndexTypeModel) o; 47 | return Objects.equal(index, that.index) && 48 | Objects.equal(type, that.type); 49 | } 50 | 51 | @Override 52 | public int hashCode() { 53 | return Objects.hashCode(index, type); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/service/MappingService.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.service; 2 | 3 | import com.star.sync.elasticsearch.model.DatabaseTableModel; 4 | import com.star.sync.elasticsearch.model.IndexTypeModel; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * 配置文件映射服务 10 | * 11 | * @author wangchao 12 | * @version 1.0 13 | * @since 2017-08-26 22:54:00 14 | */ 15 | public interface MappingService { 16 | 17 | /** 18 | * 通过database和table，获取与之对应的index和type 19 | * 20 | * @param databaseTableModel mysql 21 | * @return Elasticsearch 22 | */ 23 | IndexTypeModel getIndexType(DatabaseTableModel databaseTableModel); 24 | 25 | /** 26 | * 通过index和type，获取与之对应的database和table 27 | * 28 | * @param indexTypeModel Elasticsearch 29 | * @return mysql 30 | */ 31 | DatabaseTableModel getDatabaseTableModel(IndexTypeModel indexTypeModel); 32 | 33 | /** 34 | * 获取数据库表的主键映射 35 | */ 36 | Map getTablePrimaryKeyMap(); 37 | 38 | /** 39 | * 设置数据库表的主键映射 40 | */ 41 | void setTablePrimaryKeyMap(Map tablePrimaryKeyMap); 42 | 43 | /** 44 | * 获取Elasticsearch的数据转换后类型 45 | * 46 | * @param mysqlType mysql数据类型 47 | * @param data 具体数据 48 | * @return Elasticsearch对应的数据类型 49 | */ 50 | Object getElasticsearchTypeObject(String mysqlType, String data); 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/model/DatabaseTableModel.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.model; 2 | 3 | import com.google.common.base.Objects; 4 | 5 | /** 6 | * @author wangchao 7 | * @version 1.0 8 | * @since 2017-08-26 22:55:00 9 | */ 10 | public class DatabaseTableModel { 11 | private String database; 12 | private String table; 13 | 14 | public DatabaseTableModel() { 15 | } 16 | 17 | public DatabaseTableModel(String database, String table) { 18 | this.database = database; 19 | this.table = table; 20 | } 21 | 22 | public String getDatabase() { 23 | return database; 24 | } 25 | 26 | public void setDatabase(String database) { 27 | this.database = database; 28 | } 29 | 30 | public String getTable() { 31 | return table; 32 | } 33 | 34 | public void setTable(String table) { 35 | this.table = table; 36 | } 37 | 38 | @Override 39 | public boolean equals(Object o) { 40 | if (this == o) { 41 | return true; 42 | } 43 | if (o == null || getClass() != o.getClass()) { 44 | return false; 45 | } 46 | DatabaseTableModel that = (DatabaseTableModel) o; 47 | return Objects.equal(database, that.database) && 48 | Objects.equal(table, that.table); 49 | } 50 | 51 | @Override 52 | public int hashCode() { 53 | return Objects.hashCode(database, table); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/controller/SyncController.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.controller; 2 | 3 | import com.star.sync.elasticsearch.model.request.SyncByTableRequest; 4 | import com.star.sync.elasticsearch.model.response.Response; 5 | import com.star.sync.elasticsearch.service.SyncService; 6 | import com.star.sync.elasticsearch.util.JsonUtil; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.stereotype.Controller; 10 | import org.springframework.validation.annotation.Validated; 11 | import org.springframework.web.bind.annotation.RequestMapping; 12 | import org.springframework.web.bind.annotation.ResponseBody; 13 | 14 | import javax.annotation.Resource; 15 | 16 | /** 17 | * @author wangchao 18 | * @version 1.0 19 | * @since 2017-08-29 19:32:00 20 | */ 21 | @Controller 22 | @RequestMapping("/sync") 23 | public class SyncController { 24 | private static final Logger logger = LoggerFactory.getLogger(SyncController.class); 25 | 26 | @Resource 27 | private SyncService syncService; 28 | 29 | /** 30 | * 通过库名和表名全量同步数据 31 | * 32 | * @param request 请求参数 33 | */ 34 | @RequestMapping("/byTable") 35 | @ResponseBody 36 | public String syncTable(@Validated SyncByTableRequest request) { 37 | logger.debug("request_info: " + JsonUtil.toJson(request)); 38 | String response = Response.success(syncService.syncByTable(request)).toString(); 39 | logger.debug("response_info: " + JsonUtil.toJson(request)); 40 | return response; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/bin/stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cygwin=false; 4 | linux=false; 5 | case "`uname`" in 6 | CYGWIN*) 7 | cygwin=true 8 | ;; 9 | Linux*) 10 | linux=true 11 | ;; 12 | esac 13 | 14 | get_pid() { 15 | STR=$1 16 | PID=$2 17 | if $cygwin; then 18 | JAVA_CMD="$JAVA_HOME\bin\java" 19 | JAVA_CMD=`cygpath --path --unix $JAVA_CMD` 20 | JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'` 21 | else 22 | if $linux; then 23 | if [ ! -z "$PID" ]; then 24 | JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` 25 | else 26 | JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` 27 | fi 28 | else 29 | if [ ! -z "$PID" ]; then 30 | JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` 31 | else 32 | JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'` 33 | fi 34 | fi 35 | fi 36 | echo $JAVA_PID; 37 | } 38 | 39 | base=`dirname $0`/.. 40 | pidfile=$base/bin/sync.pid 41 | if [ ! -f "$pidfile" ];then 42 | echo "canal_mysql_elasticsearch_sync is not running. exists" 43 | exit 44 | fi 45 | 46 | pid=`cat $pidfile` 47 | if [ "$pid" == "" ] ; then 48 | pid=`get_pid "appName=canal_mysql_elasticsearch_sync"` 49 | fi 50 | 51 | echo -e "`hostname`: stopping canal_mysql_elasticsearch_sync $pid ... " 52 | kill $pid 53 | 54 | LOOPS=0 55 | while (true); 56 | do 57 | gpid=`get_pid "appName=canal_mysql_elasticsearch_sync" "$pid"` 58 | if [ "$gpid" == "" ] ; then 59 | echo "Oook! cost:$LOOPS" 60 | `rm $pidfile` 61 | break; 62 | fi 63 | let LOOPS=LOOPS+1 64 | sleep 1 65 | done -------------------------------------------------------------------------------- /src/main/assembly/dev.xml: -------------------------------------------------------------------------------- 1 | 3 | dist 4 | 5 | dir 6 | 7 | false 8 | 9 | 10 | . 11 | / 12 | 13 | README* 14 | 15 | 16 | 17 | ./src/main/bin 18 | bin 19 | 20 | **/* 21 | 22 | 0755 23 | 24 | 25 | ./src/main/conf 26 | /conf 27 | 28 | **/* 29 | 30 | 31 | 32 | ./src/main/resources 33 | /conf 34 | 35 | **/*.properties 36 | logback.xml 37 | 38 | 39 | 40 | target 41 | logs 42 | 43 | **/* 44 | 45 | 46 | 47 | 48 | 49 | lib 50 | 51 | junit:junit 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /src/main/assembly/release.xml: -------------------------------------------------------------------------------- 1 | 3 | dist 4 | 5 | tar.gz 6 | 7 | false 8 | 9 | 10 | . 11 | / 12 | 13 | README* 14 | 15 | 16 | 17 | ./src/main/bin 18 | bin 19 | 20 | **/* 21 | 22 | 0755 23 | 24 | 25 | ./src/main/conf 26 | /conf 27 | 28 | **/* 29 | 30 | 31 | 32 | ./src/main/resources 33 | /conf 34 | 35 | **/*.properties 36 | logback.xml 37 | 38 | 39 | 40 | target 41 | logs 42 | 43 | **/* 44 | 45 | 46 | 47 | 48 | 49 | lib 50 | 51 | junit:junit 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/dao/BaseDao.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.dao; 2 | 3 | import org.apache.ibatis.annotations.Param; 4 | import org.springframework.stereotype.Repository; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | /** 10 | * @author wangchao 11 | * @version 1.0 12 | * @since 2017-08-29 10:17:00 13 | */ 14 | @Repository 15 | public interface BaseDao { 16 | 17 | Map selectByPK(@Param("key") String key, @Param("value") Object value, @Param("databaseName") String databaseName, @Param("tableName") String tableName); 18 | 19 | List> selectByPKs(@Param("key") String key, @Param("valueList") List valueList, @Param("databaseName") String databaseName, @Param("tableName") String tableName); 20 | 21 | List> selectByPKsLockInShareMode(@Param("key") String key, @Param("valueList") List valueList, @Param("databaseName") String databaseName, @Param("tableName") String tableName); 22 | 23 | Long count(@Param("databaseName") String databaseName, @Param("tableName") String tableName); 24 | 25 | Long selectMaxPK(@Param("key") String key, @Param("databaseName") String databaseName, @Param("tableName") String tableName); 26 | 27 | Long selectMinPK(@Param("key") String key, @Param("databaseName") String databaseName, @Param("tableName") String tableName); 28 | 29 | List> selectByPKInterval(@Param("key") String key, @Param("minPK") long minPK, @Param("maxPK") long maxPK, @Param("databaseName") String databaseName, @Param("tableName") String tableName); 30 | 31 | List> selectByPKIntervalLockInShareMode(@Param("key") String key, @Param("minPK") long minPK, @Param("maxPK") long maxPK, @Param("databaseName") String databaseName, @Param("tableName") String tableName); 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/client/ElasticsearchClient.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.client; 2 | 3 | import org.elasticsearch.client.transport.TransportClient; 4 | import org.elasticsearch.common.settings.Settings; 5 | import org.elasticsearch.common.transport.InetSocketTransportAddress; 6 | import org.elasticsearch.transport.client.PreBuiltTransportClient; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.beans.factory.DisposableBean; 10 | import org.springframework.beans.factory.annotation.Value; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.stereotype.Component; 13 | 14 | import java.net.InetAddress; 15 | 16 | /** 17 | * @author wangchao 18 | * @version 1.0 19 | * @since 2017-08-25 17:32:00 20 | */ 21 | @Component 22 | public class ElasticsearchClient implements DisposableBean { 23 | private static final Logger logger = LoggerFactory.getLogger(ElasticsearchClient.class); 24 | private TransportClient transportClient; 25 | 26 | @Value("${elasticsearch.cluster.name}") 27 | private String clusterName; 28 | @Value("${elasticsearch.host}") 29 | private String host; 30 | @Value("${elasticsearch.port}") 31 | private String port; 32 | 33 | @Bean 34 | public TransportClient getTransportClient() throws Exception { 35 | Settings settings = Settings.builder().put("cluster.name", clusterName) 36 | .put("client.transport.sniff", true) 37 | .build(); 38 | transportClient = new PreBuiltTransportClient(settings) 39 | .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), Integer.valueOf(port))); 40 | logger.info("elasticsearch transportClient 连接成功"); 41 | return transportClient; 42 | } 43 | 44 | @Override 45 | public void destroy() throws Exception { 46 | if (transportClient != null) { 47 | transportClient.close(); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/client/CanalClient.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.client; 2 | 3 | import com.alibaba.otter.canal.client.CanalConnector; 4 | import com.alibaba.otter.canal.client.CanalConnectors; 5 | import com.google.common.collect.Lists; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.beans.factory.DisposableBean; 9 | import org.springframework.beans.factory.annotation.Value; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.stereotype.Component; 12 | 13 | import java.net.InetSocketAddress; 14 | 15 | /** 16 | * @author wangchao 17 | * @version 1.0 18 | * @since 2017-08-25 17:26:00 19 | */ 20 | @Component 21 | public class CanalClient implements DisposableBean { 22 | private static final Logger logger = LoggerFactory.getLogger(CanalClient.class); 23 | private CanalConnector canalConnector; 24 | 25 | @Value("${canal.host}") 26 | private String canalHost; 27 | @Value("${canal.port}") 28 | private String canalPort; 29 | @Value("${canal.destination}") 30 | private String canalDestination; 31 | @Value("${canal.username}") 32 | private String canalUsername; 33 | @Value("${canal.password}") 34 | private String canalPassword; 35 | 36 | @Bean 37 | public CanalConnector getCanalConnector() { 38 | canalConnector = CanalConnectors.newClusterConnector(Lists.newArrayList(new InetSocketAddress(canalHost, Integer.valueOf(canalPort))), canalDestination, canalUsername, canalPassword); 39 | canalConnector.connect(); 40 | // 指定filter，格式 {database}.{table}，这里不做过滤，过滤操作留给用户 41 | canalConnector.subscribe(); 42 | // 回滚寻找上次中断的位置 43 | canalConnector.rollback(); 44 | logger.info("canal客户端启动成功"); 45 | return canalConnector; 46 | } 47 | 48 | @Override 49 | public void destroy() throws Exception { 50 | if (canalConnector != null) { 51 | canalConnector.disconnect(); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/listener/DeleteCanalListener.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.listener; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry.Column; 4 | import com.alibaba.otter.canal.protocol.CanalEntry.RowData; 5 | import com.star.sync.elasticsearch.event.DeleteAbstractCanalEvent; 6 | import com.star.sync.elasticsearch.service.ElasticsearchService; 7 | import com.star.sync.elasticsearch.service.MappingService; 8 | import org.apache.commons.lang.StringUtils; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.springframework.stereotype.Component; 12 | 13 | import javax.annotation.Resource; 14 | import java.util.List; 15 | import java.util.Optional; 16 | 17 | /** 18 | * @author wangchao 19 | * @version 1.0 20 | * @since 2017-08-26 22:33:00 21 | */ 22 | @Component 23 | public class DeleteCanalListener extends AbstractCanalListener { 24 | private static final Logger logger = LoggerFactory.getLogger(DeleteCanalListener.class); 25 | 26 | @Resource 27 | private MappingService mappingService; 28 | 29 | @Resource 30 | private ElasticsearchService elasticsearchService; 31 | 32 | @Override 33 | protected void doSync(String database, String table, String index, String type, RowData rowData) { 34 | List columns = rowData.getBeforeColumnsList(); 35 | String primaryKey = Optional.ofNullable(mappingService.getTablePrimaryKeyMap().get(database + "." + table)).orElse("id"); 36 | Column idColumn = columns.stream().filter(column -> column.getIsKey() && primaryKey.equals(column.getName())).findFirst().orElse(null); 37 | if (idColumn == null || StringUtils.isBlank(idColumn.getValue())) { 38 | logger.warn("insert_column_find_null_warn insert从column中找不到主键,database=" + database + ",table=" + table); 39 | return; 40 | } 41 | logger.debug("insert_column_id_info insert主键id,database=" + database + ",table=" + table + ",id=" + idColumn.getValue()); 42 | elasticsearchService.deleteById(index, type, idColumn.getValue()); 43 | logger.debug("insert_es_info 同步es插入操作成功！database=" + database + ",table=" + table + ",id=" + idColumn.getValue()); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/listener/UpdateCanalListener.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.listener; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry.Column; 4 | import com.alibaba.otter.canal.protocol.CanalEntry.RowData; 5 | import com.star.sync.elasticsearch.event.UpdateAbstractCanalEvent; 6 | import com.star.sync.elasticsearch.service.ElasticsearchService; 7 | import com.star.sync.elasticsearch.service.MappingService; 8 | import org.apache.commons.lang.StringUtils; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.springframework.stereotype.Component; 12 | 13 | import javax.annotation.Resource; 14 | import java.util.List; 15 | import java.util.Map; 16 | import java.util.Optional; 17 | 18 | /** 19 | * @author wangchao 20 | * @version 1.0 21 | * @since 2017-08-26 22:32:00 22 | */ 23 | @Component 24 | public class UpdateCanalListener extends AbstractCanalListener { 25 | private static final Logger logger = LoggerFactory.getLogger(UpdateCanalListener.class); 26 | 27 | @Resource 28 | private MappingService mappingService; 29 | 30 | @Resource 31 | private ElasticsearchService elasticsearchService; 32 | 33 | @Override 34 | protected void doSync(String database, String table, String index, String type, RowData rowData) { 35 | List columns = rowData.getAfterColumnsList(); 36 | String primaryKey = Optional.ofNullable(mappingService.getTablePrimaryKeyMap().get(database + "." + table)).orElse("id"); 37 | Column idColumn = columns.stream().filter(column -> column.getIsKey() && primaryKey.equals(column.getName())).findFirst().orElse(null); 38 | if (idColumn == null || StringUtils.isBlank(idColumn.getValue())) { 39 | logger.warn("update_column_find_null_warn update从column中找不到主键,database=" + database + ",table=" + table); 40 | return; 41 | } 42 | logger.debug("update_column_id_info update主键id,database=" + database + ",table=" + table + ",id=" + idColumn.getValue()); 43 | Map dataMap = parseColumnsToMap(columns); 44 | elasticsearchService.update(index, type, idColumn.getValue(), dataMap); 45 | logger.debug("update_es_info 同步es插入操作成功！database=" + database + ",table=" + table + ",data=" + dataMap); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/com/star/sync/elasticsearch/CommonTest.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch; 2 | 3 | import org.apache.commons.lang.RandomStringUtils; 4 | import org.joda.time.DateTime; 5 | import org.joda.time.format.DateTimeFormat; 6 | import org.junit.Test; 7 | 8 | import java.math.BigDecimal; 9 | import java.util.concurrent.ExecutorService; 10 | import java.util.concurrent.LinkedBlockingQueue; 11 | import java.util.concurrent.ThreadFactory; 12 | import java.util.concurrent.ThreadPoolExecutor; 13 | import java.util.concurrent.TimeUnit; 14 | 15 | /** 16 | * @author wangchao 17 | * @version 1.0 18 | * @since 2017-09-01 16:22:00 19 | */ 20 | public class CommonTest { 21 | 22 | @Test 23 | public void testCommon() throws Exception { 24 | System.out.println(DateTime.parse("2017-09-01 16:21:17", DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")).toDate()); 25 | // long i = 7989501; 26 | long maxPK = 9025632; 27 | 28 | for (long j = 0; j < 9025632; j++) { 29 | System.out.println(String.format("当前同步pk=%s，总共total=%s，进度=%s%%", j, maxPK, new BigDecimal(j * 100).divide(new BigDecimal(maxPK), 2, BigDecimal.ROUND_HALF_UP))); 30 | } 31 | } 32 | 33 | @Test 34 | public void testThread() throws Exception { 35 | ExecutorService executorService = new ThreadPoolExecutor(10, 10, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), new ThreadFactory() { 36 | @Override 37 | public Thread newThread(Runnable r) { 38 | Thread t = new Thread(r); 39 | t.setName("11"); 40 | return t; 41 | } 42 | }); 43 | for (int i = 0; i < 5; i++) { 44 | executorService.submit(new Runnable() { 45 | @Override 46 | public void run() { 47 | while (true) { 48 | System.out.println(Thread.currentThread().getName() + " is run"); 49 | try { 50 | Thread.sleep(1000); 51 | } catch (InterruptedException e) { 52 | e.printStackTrace(); 53 | } 54 | } 55 | } 56 | }); 57 | } 58 | Thread.sleep(10000); 59 | executorService.shutdown(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/service/impl/TransactionalServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.service.impl; 2 | 3 | import com.star.sync.elasticsearch.dao.BaseDao; 4 | import com.star.sync.elasticsearch.model.IndexTypeModel; 5 | import com.star.sync.elasticsearch.model.request.SyncByTableRequest; 6 | import com.star.sync.elasticsearch.service.ElasticsearchService; 7 | import com.star.sync.elasticsearch.service.TransactionalService; 8 | import org.springframework.stereotype.Service; 9 | import org.springframework.transaction.annotation.Transactional; 10 | 11 | import javax.annotation.Resource; 12 | import java.sql.Timestamp; 13 | import java.time.LocalDateTime; 14 | import java.time.ZoneId; 15 | import java.util.List; 16 | import java.util.Map; 17 | import java.util.stream.Collectors; 18 | 19 | /** 20 | * @author wangchao 21 | * @version 1.0.0 22 | * @since 2018-05-21 23:23:00 23 | */ 24 | @Service 25 | public class TransactionalServiceImpl implements TransactionalService { 26 | 27 | @Resource 28 | private BaseDao baseDao; 29 | 30 | @Resource 31 | private ElasticsearchService elasticsearchService; 32 | 33 | @Transactional(readOnly = true, rollbackFor = Exception.class) 34 | @Override 35 | public void batchInsertElasticsearch(SyncByTableRequest request, String primaryKey, long from, long to, IndexTypeModel indexTypeModel) { 36 | List> dataList = baseDao.selectByPKIntervalLockInShareMode(primaryKey, from, to, request.getDatabase(), request.getTable()); 37 | if (dataList == null || dataList.isEmpty()) { 38 | return; 39 | } 40 | dataList = convertDateType(dataList); 41 | Map> dataMap = dataList.parallelStream().collect(Collectors.toMap(strObjMap -> String.valueOf(strObjMap.get(primaryKey)), map -> map)); 42 | elasticsearchService.batchInsertById(indexTypeModel.getIndex(), indexTypeModel.getType(), dataMap); 43 | } 44 | 45 | private List> convertDateType(List> source) { 46 | source.parallelStream().forEach(map -> map.forEach((key, value) -> { 47 | if (value instanceof Timestamp) { 48 | map.put(key, LocalDateTime.ofInstant(((Timestamp) value).toInstant(), ZoneId.systemDefault())); 49 | } 50 | })); 51 | return source; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/listener/InsertCanalListener.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.listener; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry.Column; 4 | import com.alibaba.otter.canal.protocol.CanalEntry.RowData; 5 | import com.star.sync.elasticsearch.event.InsertAbstractCanalEvent; 6 | import com.star.sync.elasticsearch.service.ElasticsearchService; 7 | import com.star.sync.elasticsearch.service.MappingService; 8 | import com.star.sync.elasticsearch.util.JsonUtil; 9 | import org.apache.commons.lang.StringUtils; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.stereotype.Component; 13 | 14 | import javax.annotation.Resource; 15 | import java.util.List; 16 | import java.util.Map; 17 | import java.util.Optional; 18 | 19 | /** 20 | * @author wangchao 21 | * @version 1.0 22 | * @since 2017-08-26 22:32:00 23 | */ 24 | @Component 25 | public class InsertCanalListener extends AbstractCanalListener { 26 | private static final Logger logger = LoggerFactory.getLogger(InsertCanalListener.class); 27 | 28 | @Resource 29 | private MappingService mappingService; 30 | 31 | @Resource 32 | private ElasticsearchService elasticsearchService; 33 | 34 | @Override 35 | protected void doSync(String database, String table, String index, String type, RowData rowData) { 36 | List columns = rowData.getAfterColumnsList(); 37 | String primaryKey = Optional.ofNullable(mappingService.getTablePrimaryKeyMap().get(database + "." + table)).orElse("id"); 38 | Column idColumn = columns.stream().filter(column -> column.getIsKey() && primaryKey.equals(column.getName())).findFirst().orElse(null); 39 | if (idColumn == null || StringUtils.isBlank(idColumn.getValue())) { 40 | logger.warn("insert_column_find_null_warn insert从column中找不到主键,database=" + database + ",table=" + table); 41 | return; 42 | } 43 | logger.debug("insert_column_id_info insert主键id,database=" + database + ",table=" + table + ",id=" + idColumn.getValue()); 44 | Map dataMap = parseColumnsToMap(columns); 45 | elasticsearchService.insertById(index, type, idColumn.getValue(), dataMap); 46 | logger.debug("insert_es_info 同步es插入操作成功！database=" + database + ",table=" + table + ",data=" + JsonUtil.toJson(dataMap)); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/service/impl/ElasticsearchServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.service.impl; 2 | 3 | import com.star.sync.elasticsearch.service.ElasticsearchService; 4 | import com.star.sync.elasticsearch.util.JsonUtil; 5 | import org.elasticsearch.action.bulk.BulkRequestBuilder; 6 | import org.elasticsearch.action.bulk.BulkResponse; 7 | import org.elasticsearch.client.transport.TransportClient; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.stereotype.Service; 11 | 12 | import javax.annotation.Resource; 13 | import java.util.Map; 14 | 15 | /** 16 | * @author wangchao 17 | * @version 1.0 18 | * @since 2017-08-27 12:38:00 19 | */ 20 | @Service 21 | public class ElasticsearchServiceImpl implements ElasticsearchService { 22 | private static final Logger logger = LoggerFactory.getLogger(ElasticsearchServiceImpl.class); 23 | 24 | @Resource 25 | private TransportClient transportClient; 26 | 27 | @Override 28 | public void insertById(String index, String type, String id, Map dataMap) { 29 | transportClient.prepareIndex(index, type, id).setSource(dataMap).get(); 30 | } 31 | 32 | @Override 33 | public void batchInsertById(String index, String type, Map> idDataMap) { 34 | BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); 35 | 36 | idDataMap.forEach((id, dataMap) -> bulkRequestBuilder.add(transportClient.prepareIndex(index, type, id).setSource(dataMap))); 37 | try { 38 | BulkResponse bulkResponse = bulkRequestBuilder.execute().get(); 39 | if (bulkResponse.hasFailures()) { 40 | logger.error("elasticsearch批量插入错误, index=" + index + ", type=" + type + ", data=" + JsonUtil.toJson(idDataMap) + ", cause:" + bulkResponse.buildFailureMessage()); 41 | } 42 | } catch (Exception e) { 43 | logger.error("elasticsearch批量插入错误, index=" + index + ", type=" + type + ", data=" + JsonUtil.toJson(idDataMap), e); 44 | } 45 | } 46 | 47 | @Override 48 | public void update(String index, String type, String id, Map dataMap) { 49 | this.insertById(index, type, id, dataMap); 50 | } 51 | 52 | @Override 53 | public void deleteById(String index, String type, String id) { 54 | transportClient.prepareDelete(index, type, id).get(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/resources/mybatis/mapper/BaseMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 12 | 13 | 23 | 24 | 35 | 36 | 42 | 43 | 51 | 52 | 60 | 61 | 69 | 70 | 79 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # canal_mysql_elasticsearch_sync 支持请star✨ 2 | # 交流Q群：733688083 3 | 4 | **canal于v1.1.2版本后，已支持自动同步到Elasticsearch。赞canal！[canal传送门](https://github.com/alibaba/canal)** 5 | 6 | 基于 *canal* 的 *Mysql* 与 *Elasticsearch* 实时同步的 *javaweb* 服务。 7 | canal是阿里巴巴mysql数据库binlog的增量订阅&消费组件。 8 | 9 | ## 工作原理 10 | ### 全量 11 | 暴露Http接口（接口定义见[wiki](https://github.com/starcwang/canal_mysql_elasticsearch_sync/wiki/HttpApi)），待调用后开启后台线程，通过主键分批同步指定数据库中数据到Elasticsearch 12 | > 读取数据库会加**读锁** 13 | > 主键必须为数字类型 14 | #### 过程 15 | 1. 首先会根据所给的数据库主键字段，拿到最大的主键数字max_id； 16 | 2. 设*pk*=min_id（默认是数据库中的主键最小值）； 17 | 2. 加读锁🔐，从数据库中取出*pk* —— *pk*+*stepSize* 大小的数据（默认500）的数据； 18 | 3. 插入到Elasticsearch中； 19 | 4. 释放读锁🔐，pk累加*stepSize*，循环3.操作，直到*pk*>*max_id* 20 | 21 | ### 增量 22 | 循环监听canal通过binlog同步过来的event事件，区别增删改进行与之对应的Elasticsearch的操作。 23 | > 目前只解析了 insert、update、delete，其它数据库操作会被忽略 24 | 25 | ## 默认相关字段映射 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 |
Mysql字段类型Elasticsearch类型
char{"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 256}}
text{"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 256}}
blob{"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 256}}
int{"type": "long"}
date{"type": "date"}
time{"type": "date"}
float{"type": "float"}
double{"type": "float"}
decimal{"type": "float"}
其它{"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 256}}
72 | 73 | ## 注意事项 74 | - Mysql的binlog格式必须为**ROW** 75 | - 因为有行锁，Mysql中table使用的存储引擎须为**InnoDB** 76 | - 由于使用binlog进行增量同步，和数据库主从类似，不可避免的会有一定的主从延迟，延迟时间取决于机房网络、机器负载、数据量大小等 77 | - Elasticsearch支持的版本为**5.x** 78 | - canal已测试版为**v1.0.24**，其他版本请自行测试 79 | - 增量同步只监听了 **INSERT、UPDATE、DELETE**，其它如建表、删表等尚未支持 80 | - 建议Elasticsearch的mapping手动来创建，因为默认的创建方式不能保证满足业务需求 81 | 82 | ## 相关文档 83 | - [wiki](https://github.com/starcwang/canal_mysql_elasticsearch_sync/wiki) 84 | - [HttpApi](https://github.com/starcwang/canal_mysql_elasticsearch_sync/wiki/HttpApi) 85 | - [QuickStart](https://github.com/starcwang/canal_mysql_elasticsearch_sync/wiki/QuickStart) 86 | 87 | ## 联系方式 88 | 如果有不合理的地方，还请不吝赐教。 89 | - QQ群：733688083 90 | 91 | **支持记得star✨** 92 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/listener/AbstractCanalListener.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.listener; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry.Column; 4 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 5 | import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; 6 | import com.alibaba.otter.canal.protocol.CanalEntry.RowData; 7 | import com.google.protobuf.InvalidProtocolBufferException; 8 | import com.star.sync.elasticsearch.event.AbstractCanalEvent; 9 | import com.star.sync.elasticsearch.model.DatabaseTableModel; 10 | import com.star.sync.elasticsearch.model.IndexTypeModel; 11 | import com.star.sync.elasticsearch.service.MappingService; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | import org.springframework.context.ApplicationListener; 15 | 16 | import javax.annotation.Resource; 17 | import java.util.HashMap; 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | /** 22 | * @author wangchao 23 | * @version 1.0 24 | * @since 2017-08-28 14:40:00 25 | */ 26 | public abstract class AbstractCanalListener implements ApplicationListener { 27 | private static final Logger logger = LoggerFactory.getLogger(AbstractCanalListener.class); 28 | 29 | @Resource 30 | private MappingService mappingService; 31 | 32 | @Override 33 | public void onApplicationEvent(EVENT event) { 34 | Entry entry = event.getEntry(); 35 | String database = entry.getHeader().getSchemaName(); 36 | String table = entry.getHeader().getTableName(); 37 | IndexTypeModel indexTypeModel = mappingService.getIndexType(new DatabaseTableModel(database, table)); 38 | if (indexTypeModel == null) { 39 | return; 40 | } 41 | String index = indexTypeModel.getIndex(); 42 | String type = indexTypeModel.getType(); 43 | RowChange change; 44 | try { 45 | change = RowChange.parseFrom(entry.getStoreValue()); 46 | } catch (InvalidProtocolBufferException e) { 47 | logger.error("canalEntry_parser_error,根据CanalEntry获取RowChange失败！", e); 48 | return; 49 | } 50 | change.getRowDatasList().forEach(rowData -> doSync(database, table, index, type, rowData)); 51 | } 52 | 53 | Map parseColumnsToMap(List columns) { 54 | Map jsonMap = new HashMap<>(); 55 | columns.forEach(column -> { 56 | if (column == null) { 57 | return; 58 | } 59 | jsonMap.put(column.getName(), column.getIsNull() ? null : mappingService.getElasticsearchTypeObject(column.getMysqlType(), column.getValue())); 60 | }); 61 | return jsonMap; 62 | } 63 | 64 | protected abstract void doSync(String database, String table, String index, String type, RowData rowData); 65 | } 66 | -------------------------------------------------------------------------------- /src/main/bin/startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | current_path=`pwd` 4 | case "`uname`" in 5 | Linux) 6 | bin_abs_path=$(readlink -f $(dirname $0)) 7 | ;; 8 | *) 9 | bin_abs_path=`cd $(dirname $0); pwd` 10 | ;; 11 | esac 12 | base=${bin_abs_path}/.. 13 | sync_conf=$base/conf/application.properties 14 | loader_path=$base/conf 15 | logback_configurationFile=$base/conf/logback.xml 16 | export LANG=en_US.UTF-8 17 | export BASE=$base 18 | 19 | if [ -f $base/bin/sync.pid ] ; then 20 | echo "found sync.pid , Please run stop.sh first ,then startup.sh" 2>&2 21 | exit 1 22 | fi 23 | 24 | if [ ! -d $base/logs/sync ] ; then 25 | mkdir -p $base/logs/sync 26 | fi 27 | 28 | ## set java path 29 | if [ -z "$JAVA" ] ; then 30 | JAVA=$(which java) 31 | fi 32 | 33 | if [ -z "$JAVA" ]; then 34 | echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2 35 | exit 1 36 | fi 37 | 38 | case "$#" 39 | in 40 | 0 ) 41 | ;; 42 | 1 ) 43 | var=$* 44 | if [ -f $var ] ; then 45 | sync_conf=$var 46 | else 47 | echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN." 48 | exit 49 | fi;; 50 | 2 ) 51 | var=$1 52 | if [ -f $var ] ; then 53 | sync_conf=$var 54 | else 55 | if [ "$1" = "debug" ]; then 56 | DEBUG_PORT=$2 57 | DEBUG_SUSPEND="n" 58 | JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND" 59 | fi 60 | fi;; 61 | * ) 62 | echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN." 63 | exit;; 64 | esac 65 | 66 | str=`file -L $JAVA | grep 64-bit` 67 | if [ -n "$str" ]; then 68 | JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError" 69 | else 70 | JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m " 71 | fi 72 | 73 | JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8" 74 | CANAL_OPTS="-Dloader.path=$loader_path -DappName=canal_mysql_elasticsearch_sync -Dlogback.configurationFile=$logback_configurationFile" 75 | MAIN_JAR=$base/lib/canal-mysql-elasticsearch-sync-*.jar 76 | 77 | if [ -e $sync_conf -a -e $logback_configurationFile ] 78 | then 79 | 80 | for i in $base/lib/*; 81 | do CLASSPATH=$i:"$CLASSPATH"; 82 | done 83 | CLASSPATH="$base/conf:$CLASSPATH"; 84 | 85 | echo "cd to $bin_abs_path for workaround relative path" 86 | cd $bin_abs_path 87 | 88 | echo LOG CONFIGURATION : $logback_configurationFile 89 | echo sync conf : $sync_conf 90 | echo CLASSPATH :$CLASSPATH 91 | $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH -jar $MAIN_JAR 1>>$base/logs/info.log 2>&1 & 92 | echo $! > $base/bin/sync.pid 93 | 94 | echo "cd to $current_path for continue" 95 | cd $current_path 96 | else 97 | echo "sync conf("$sync_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!" 98 | fi 99 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/scheduling/CanalScheduling.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.scheduling; 2 | 3 | import com.alibaba.otter.canal.client.CanalConnector; 4 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry; 5 | import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; 6 | import com.alibaba.otter.canal.protocol.CanalEntry.EventType; 7 | import com.alibaba.otter.canal.protocol.Message; 8 | import com.star.sync.elasticsearch.event.DeleteAbstractCanalEvent; 9 | import com.star.sync.elasticsearch.event.InsertAbstractCanalEvent; 10 | import com.star.sync.elasticsearch.event.UpdateAbstractCanalEvent; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | import org.springframework.beans.BeansException; 14 | import org.springframework.context.ApplicationContext; 15 | import org.springframework.context.ApplicationContextAware; 16 | import org.springframework.scheduling.annotation.Scheduled; 17 | import org.springframework.stereotype.Component; 18 | 19 | import javax.annotation.Resource; 20 | import java.util.List; 21 | 22 | /** 23 | * @author wangchao 24 | * @version 1.0 25 | * @since 2017-08-26 22:44:00 26 | */ 27 | @Component 28 | public class CanalScheduling implements Runnable, ApplicationContextAware { 29 | private static final Logger logger = LoggerFactory.getLogger(CanalScheduling.class); 30 | private ApplicationContext applicationContext; 31 | 32 | @Resource 33 | private CanalConnector canalConnector; 34 | 35 | @Scheduled(fixedDelay = 100) 36 | @Override 37 | public void run() { 38 | try { 39 | int batchSize = 1000; 40 | // Message message = connector.get(batchSize); 41 | Message message = canalConnector.getWithoutAck(batchSize); 42 | long batchId = message.getId(); 43 | logger.debug("scheduled_batchId=" + batchId); 44 | try { 45 | List entries = message.getEntries(); 46 | if (batchId != -1 && entries.size() > 0) { 47 | entries.forEach(entry -> { 48 | if (entry.getEntryType() == EntryType.ROWDATA) { 49 | publishCanalEvent(entry); 50 | } 51 | }); 52 | } 53 | canalConnector.ack(batchId); 54 | } catch (Exception e) { 55 | logger.error("发送监听事件失败！batchId回滚,batchId=" + batchId, e); 56 | canalConnector.rollback(batchId); 57 | } 58 | } catch (Exception e) { 59 | logger.error("canal_scheduled异常！", e); 60 | } 61 | } 62 | 63 | private void publishCanalEvent(Entry entry) { 64 | EventType eventType = entry.getHeader().getEventType(); 65 | switch (eventType) { 66 | case INSERT: 67 | applicationContext.publishEvent(new InsertAbstractCanalEvent(entry)); 68 | break; 69 | case UPDATE: 70 | applicationContext.publishEvent(new UpdateAbstractCanalEvent(entry)); 71 | break; 72 | case DELETE: 73 | applicationContext.publishEvent(new DeleteAbstractCanalEvent(entry)); 74 | break; 75 | default: 76 | break; 77 | } 78 | } 79 | 80 | @Override 81 | public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { 82 | this.applicationContext = applicationContext; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/service/impl/SyncServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.service.impl; 2 | 3 | import com.star.sync.elasticsearch.dao.BaseDao; 4 | import com.star.sync.elasticsearch.model.DatabaseTableModel; 5 | import com.star.sync.elasticsearch.model.IndexTypeModel; 6 | import com.star.sync.elasticsearch.model.request.SyncByTableRequest; 7 | import com.star.sync.elasticsearch.service.MappingService; 8 | import com.star.sync.elasticsearch.service.SyncService; 9 | import com.star.sync.elasticsearch.service.TransactionalService; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.beans.factory.DisposableBean; 13 | import org.springframework.beans.factory.InitializingBean; 14 | import org.springframework.stereotype.Service; 15 | 16 | import javax.annotation.Resource; 17 | import java.math.BigDecimal; 18 | import java.util.Optional; 19 | import java.util.concurrent.ExecutorService; 20 | import java.util.concurrent.LinkedBlockingQueue; 21 | import java.util.concurrent.ThreadFactory; 22 | import java.util.concurrent.ThreadPoolExecutor; 23 | import java.util.concurrent.TimeUnit; 24 | 25 | /** 26 | * @author wangchao 27 | * @version 1.0 28 | * @since 2017-08-31 17:48:00 29 | */ 30 | @Service 31 | public class SyncServiceImpl implements SyncService, InitializingBean, DisposableBean { 32 | private static final Logger logger = LoggerFactory.getLogger(SyncServiceImpl.class); 33 | /** 34 | * 使用线程池控制并发数量 35 | */ 36 | private ExecutorService cachedThreadPool; 37 | 38 | @Resource 39 | private BaseDao baseDao; 40 | 41 | @Resource 42 | private MappingService mappingService; 43 | 44 | @Resource 45 | private TransactionalService transactionalService; 46 | 47 | @Override 48 | public boolean syncByTable(SyncByTableRequest request) { 49 | IndexTypeModel indexTypeModel = mappingService.getIndexType(new DatabaseTableModel(request.getDatabase(), request.getTable())); 50 | String primaryKey = Optional.ofNullable(mappingService.getTablePrimaryKeyMap().get(request.getDatabase() + "." + request.getTable())).orElse("id"); 51 | if (indexTypeModel == null) { 52 | throw new IllegalArgumentException(String.format("配置文件中缺失database=%s和table=%s所对应的index和type的映射配置", request.getDatabase(), request.getTable())); 53 | } 54 | 55 | long minPK = Optional.ofNullable(request.getFrom()).orElse(baseDao.selectMinPK(primaryKey, request.getDatabase(), request.getTable())); 56 | long maxPK = Optional.ofNullable(request.getTo()).orElse(baseDao.selectMaxPK(primaryKey, request.getDatabase(), request.getTable())); 57 | cachedThreadPool.submit(() -> { 58 | try { 59 | for (long i = minPK; i <= maxPK + request.getStepSize(); i += request.getStepSize()) { 60 | transactionalService.batchInsertElasticsearch(request, primaryKey, i, i + request.getStepSize(), indexTypeModel); 61 | logger.info(String.format("当前同步pk=%s，总共total=%s，进度=%s%%", i, maxPK, new BigDecimal(i * 100).divide(new BigDecimal(maxPK), 3, BigDecimal.ROUND_HALF_UP))); 62 | } 63 | } catch (Exception e) { 64 | logger.error("批量转换并插入Elasticsearch异常", e); 65 | } 66 | }); 67 | return true; 68 | } 69 | 70 | @Override 71 | public void afterPropertiesSet() throws Exception { 72 | cachedThreadPool = new ThreadPoolExecutor(5, 5, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), (ThreadFactory) Thread::new); 73 | } 74 | 75 | @Override 76 | public void destroy() throws Exception { 77 | if (cachedThreadPool != null) { 78 | cachedThreadPool.shutdown(); 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/service/impl/MappingServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.service.impl; 2 | 3 | import com.google.common.collect.BiMap; 4 | import com.google.common.collect.HashBiMap; 5 | import com.google.common.collect.Maps; 6 | import com.star.sync.elasticsearch.model.DatabaseTableModel; 7 | import com.star.sync.elasticsearch.model.IndexTypeModel; 8 | import com.star.sync.elasticsearch.service.MappingService; 9 | import org.apache.commons.lang.StringUtils; 10 | import org.springframework.beans.factory.InitializingBean; 11 | import org.springframework.boot.context.properties.ConfigurationProperties; 12 | import org.springframework.context.annotation.PropertySource; 13 | import org.springframework.stereotype.Service; 14 | 15 | import java.time.LocalDateTime; 16 | import java.time.format.DateTimeFormatter; 17 | import java.util.Map; 18 | import java.util.Map.Entry; 19 | import java.util.Optional; 20 | 21 | /** 22 | * @author wangchao 23 | * @version 1.0 24 | * @since 2017-08-27 13:14:00 25 | */ 26 | @Service 27 | @PropertySource("classpath:mapping.properties") 28 | @ConfigurationProperties 29 | public class MappingServiceImpl implements MappingService, InitializingBean { 30 | private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); 31 | 32 | private Map dbEsMapping; 33 | private BiMap dbEsBiMapping; 34 | private Map tablePrimaryKeyMap; 35 | private Map mysqlTypeElasticsearchTypeMapping; 36 | 37 | @Override 38 | public Map getTablePrimaryKeyMap() { 39 | return tablePrimaryKeyMap; 40 | } 41 | 42 | @Override 43 | public void setTablePrimaryKeyMap(Map tablePrimaryKeyMap) { 44 | this.tablePrimaryKeyMap = tablePrimaryKeyMap; 45 | } 46 | 47 | @Override 48 | public IndexTypeModel getIndexType(DatabaseTableModel databaseTableModel) { 49 | return dbEsBiMapping.get(databaseTableModel); 50 | } 51 | 52 | @Override 53 | public DatabaseTableModel getDatabaseTableModel(IndexTypeModel indexTypeModel) { 54 | return dbEsBiMapping.inverse().get(indexTypeModel); 55 | } 56 | 57 | @Override 58 | public Object getElasticsearchTypeObject(String mysqlType, String data) { 59 | Optional> result = mysqlTypeElasticsearchTypeMapping.entrySet().parallelStream().filter(entry -> mysqlType.toLowerCase().contains(entry.getKey())).findFirst(); 60 | return (result.isPresent() ? result.get().getValue() : (Converter) data1 -> data1).convert(data); 61 | } 62 | 63 | @Override 64 | public void afterPropertiesSet() throws Exception { 65 | dbEsBiMapping = HashBiMap.create(); 66 | dbEsMapping.forEach((key, value) -> { 67 | String[] keyStrings = StringUtils.split(key, "."); 68 | String[] valueStrings = StringUtils.split(value, "."); 69 | dbEsBiMapping.put(new DatabaseTableModel(keyStrings[0], keyStrings[1]), new IndexTypeModel(valueStrings[0], valueStrings[1])); 70 | }); 71 | 72 | mysqlTypeElasticsearchTypeMapping = Maps.newHashMap(); 73 | mysqlTypeElasticsearchTypeMapping.put("char", data -> data); 74 | mysqlTypeElasticsearchTypeMapping.put("text", data -> data); 75 | mysqlTypeElasticsearchTypeMapping.put("blob", data -> data); 76 | mysqlTypeElasticsearchTypeMapping.put("int", Long::valueOf); 77 | mysqlTypeElasticsearchTypeMapping.put("date", data -> LocalDateTime.parse(data, FORMATTER)); 78 | mysqlTypeElasticsearchTypeMapping.put("time", data -> LocalDateTime.parse(data, FORMATTER)); 79 | mysqlTypeElasticsearchTypeMapping.put("float", Double::valueOf); 80 | mysqlTypeElasticsearchTypeMapping.put("double", Double::valueOf); 81 | mysqlTypeElasticsearchTypeMapping.put("decimal", Double::valueOf); 82 | } 83 | 84 | public Map getDbEsMapping() { 85 | return dbEsMapping; 86 | } 87 | 88 | public void setDbEsMapping(Map dbEsMapping) { 89 | this.dbEsMapping = dbEsMapping; 90 | } 91 | 92 | private interface Converter { 93 | Object convert(String data); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/com/star/sync/elasticsearch/util/JsonUtil.java: -------------------------------------------------------------------------------- 1 | package com.star.sync.elasticsearch.util; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude.Include; 4 | import com.fasterxml.jackson.core.type.TypeReference; 5 | import com.fasterxml.jackson.databind.DeserializationFeature; 6 | import com.fasterxml.jackson.databind.ObjectMapper; 7 | import com.fasterxml.jackson.databind.SerializationFeature; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.io.IOException; 12 | import java.text.SimpleDateFormat; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | 16 | /** 17 | * json格式转换通用类 18 | * 19 | * @author wangchao 20 | * @version 1.0 21 | * @since 2017-08-26 22:54:00 22 | */ 23 | public class JsonUtil { 24 | private static final Logger logger = LoggerFactory.getLogger(JsonUtil.class); 25 | private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; 26 | /** 27 | * 对象映射 28 | */ 29 | private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); 30 | 31 | static { 32 | OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 33 | OBJECT_MAPPER.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); 34 | OBJECT_MAPPER.setDateFormat(new SimpleDateFormat(DATE_FORMAT)); 35 | OBJECT_MAPPER.setSerializationInclusion(Include.NON_NULL); 36 | } 37 | 38 | 39 | /** 40 | * Java对象转换为Json串 41 | * 42 | * @param obj Java对象 43 | * @return Json串 44 | */ 45 | public static String toJson(Object obj) { 46 | String rst; 47 | if (obj == null || obj instanceof String) { 48 | return (String) obj; 49 | } 50 | try { 51 | rst = OBJECT_MAPPER.writeValueAsString(obj); 52 | } catch (Exception e) { 53 | logger.error("将Java对象转换成Json串出错！"); 54 | throw new RuntimeException("将Java对象转换成Json串出错！", e); 55 | } 56 | return rst; 57 | } 58 | 59 | /** 60 | * Json串转换为Java对象 61 | * 62 | * @param json Json串 63 | * @param type Java对象类型 64 | * @return Java对象 65 | */ 66 | public static T fromJson(String json, Class type) { 67 | T rst; 68 | try { 69 | rst = OBJECT_MAPPER.readValue(json, type); 70 | } catch (Exception e) { 71 | logger.error("Json串转换成对象出错：{}", json); 72 | throw new RuntimeException("Json串转换成对象出错!", e); 73 | } 74 | return rst; 75 | } 76 | 77 | /** 78 | * Json串转换为Java对象 79 | *
使用引用类型，适用于List<Object>、Set<Object> 这种无法直接获取class对象的场景 80 | *
使用方法：TypeReference ref = new TypeReference<List<Integer>>(){}; 81 | * 82 | * @param json Json串 83 | * @param typeRef Java对象类型引用 84 | * @return Java对象 85 | */ 86 | @SuppressWarnings("unchecked") 87 | public static T fromJson(String json, TypeReference typeRef) { 88 | T rst; 89 | try { 90 | rst = OBJECT_MAPPER.readValue(json, typeRef); 91 | } catch (Exception e) { 92 | logger.error("Json串转换成对象出错：{}", json); 93 | throw new RuntimeException("Json串转换成对象出错!", e); 94 | } 95 | return rst; 96 | } 97 | 98 | @SuppressWarnings("unchecked") 99 | public static HashMap fromJsonToMap(String json) { 100 | HashMap map = new HashMap(); 101 | try { 102 | map = OBJECT_MAPPER.readValue(json, map.getClass()); 103 | } catch (IOException e) { 104 | logger.error("Json串转换成对象出错：{}", json); 105 | } 106 | return map; 107 | } 108 | 109 | @SuppressWarnings("unchecked") 110 | public static HashMap toMap(String json) { 111 | HashMap map; 112 | try { 113 | map = OBJECT_MAPPER.readValue(json, HashMap.class); 114 | } catch (Exception e) { 115 | map = null; 116 | logger.error("Json串转换成对象出错：{}", json); 117 | } 118 | return map; 119 | } 120 | 121 | @SuppressWarnings("unchecked") 122 | public static List> fromJsonToList(String json) { 123 | List> list; 124 | try { 125 | list = OBJECT_MAPPER.readValue(json, List.class); 126 | } catch (IOException e) { 127 | logger.error("Json串转换成对象出错：{}", json); 128 | throw new RuntimeException("Json串转换成List出错!", e); 129 | } 130 | return list; 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.star.sync 7 | canal-mysql-elasticsearch-sync 8 | 1.0.5 9 | jar 10 | 11 | canal-mysql-elasticsearch-sync 12 | 基于canal的mysql和elasticsearch实时同步方案，支持增量同步和全量同步 13 | 14 | 15 | org.springframework.boot 16 | spring-boot-starter-parent 17 | 1.5.6.RELEASE 18 | 19 | 20 | 21 | 22 | 5.1.1 23 | UTF-8 24 | UTF-8 25 | 1.8 26 | 27 | 28 | 29 | 30 | org.springframework.boot 31 | spring-boot-starter-jdbc 32 | 33 | 34 | org.mybatis.spring.boot 35 | mybatis-spring-boot-starter 36 | 1.3.1 37 | 38 | 39 | org.springframework.boot 40 | spring-boot-starter-web 41 | 42 | 43 | org.springframework.boot 44 | spring-boot-configuration-processor 45 | true 46 | 47 | 48 | 49 | org.elasticsearch.client 50 | transport 51 | ${org.elasticsearch.version} 52 | 53 | 54 | org.elasticsearch 55 | elasticsearch 56 | ${org.elasticsearch.version} 57 | 58 | 59 | com.alibaba.otter 60 | canal.client 61 | 1.0.24 62 | 63 | 64 | spring 65 | org.springframework 66 | 67 | 68 | 69 | 70 | 71 | org.apache.logging.log4j 72 | log4j-to-slf4j 73 | 2.7 74 | 75 | 76 | 77 | mysql 78 | mysql-connector-java 79 | runtime 80 | 81 | 82 | org.springframework.boot 83 | spring-boot-starter-test 84 | test 85 | 86 | 87 | 88 | 89 | 90 | 91 | org.springframework.boot 92 | spring-boot-maven-plugin 93 | 94 | ZIP 95 | 96 | 97 | 98 | 99 | maven-jar-plugin 100 | 101 | 102 | true 103 | 104 | 105 | **/logback.xml 106 | **/application.properties 107 | 108 | 109 | 110 | 111 | 112 | org.apache.maven.plugins 113 | maven-assembly-plugin 114 | 115 | 2.2.1 116 | 117 | 118 | assemble 119 | 120 | single 121 | 122 | package 123 | 124 | 125 | 126 | false 127 | false 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | dev 136 | 137 | true 138 | 139 | env 140 | !release 141 | 142 | 143 | 144 | 145 | 146 | 147 | maven-assembly-plugin 148 | 149 | 150 | 151 | ${basedir}/src/main/assembly/dev.xml 152 | 153 | canal 154 | ${project.build.directory} 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | release 164 | 165 | 166 | env 167 | release 168 | 169 | 170 | 171 | 172 | 173 | 174 | maven-assembly-plugin 175 | 176 | 177 | 178 | ${basedir}/src/main/assembly/release.xml 179 | 180 | 181 | ${project.artifactId}-${project.version} 182 | 183 | ${project.parent.build.directory} 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /src/main/resources/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | 启动成功 11 | 1316 | 1317 | 1318 |

启动成功

1319 | 1320 |
1321 |
    1322 |
  1. 1323 |
    1324 |
    1325 |
    1326 |
    1327 |
    1328 |
    1329 |
    1330 |
    1331 |
    1332 |
    1333 |
    1334 |
    1335 |
    1336 |
    1337 |
    1338 |
    1339 |
    1340 |
    1341 |
    1342 |
    1343 |
    1344 |
    2. 1345 |
1346 |
1347 | 1348 | --------------------------------------------------------------------------------