├── .gitignore ├── README.md ├── mysql-binlog-demo ├── pom.xml └── src │ └── main │ ├── java │ └── net │ │ └── giafei │ │ └── tools │ │ └── demo │ │ ├── DemoApplication.java │ │ ├── entity │ │ └── TestTable1.java │ │ ├── sql │ │ ├── create.sql │ │ └── test.sql │ │ └── watcher │ │ └── TestTable1Watcher.java │ └── resources │ └── application.yml ├── mysql-binlog-starter ├── pom.xml └── src │ └── main │ └── java │ └── net │ └── giafei │ └── tools │ └── binlog │ └── mysql │ ├── annotation │ └── MysqlWatcher.java │ ├── config │ ├── MySqlHost.java │ └── MySqlHostProfile.java │ ├── core │ ├── BinLogBeanProcessor.java │ ├── BinlogDataDispatcher.java │ ├── DataListenerContainer.java │ ├── MySqlTable.java │ └── MysqlDataListenerData.java │ ├── listener │ ├── AbstractMysqlDataImporter.java │ └── IMysqlDataListener.java │ └── thread │ ├── BinLogListenerThread.java │ └── BinlogThreadStarter.java └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | # ---> JetBrains 2 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio 3 | 4 | *.iml 5 | 6 | ## Directory-based project format: 7 | .idea/ 8 | # if you remove the above rule, at least ignore the following: 9 | 10 | # User-specific stuff: 11 | # .idea/workspace.xml 12 | # .idea/tasks.xml 13 | # .idea/dictionaries 14 | 15 | # Sensitive or high-churn files: 16 | # .idea/dataSources.ids 17 | # .idea/dataSources.xml 18 | # .idea/sqlDataSources.xml 19 | # .idea/dynamic.xml 20 | # .idea/uiDesigner.xml 21 | 22 | # Gradle: 23 | # .idea/gradle.xml 24 | # .idea/libraries 25 | 26 | # Mongo Explorer plugin: 27 | # .idea/mongoSettings.xml 28 | 29 | ## File-based project format: 30 | *.ipr 31 | *.iws 32 | 33 | ## Plugin-specific files: 34 | 35 | # IntelliJ 36 | /out/ 37 | 38 | # mpeltonen/sbt-idea plugin 39 | .idea_modules/ 40 | 41 | # JIRA plugin 42 | atlassian-ide-plugin.xml 43 | 44 | # Crashlytics plugin (for Android Studio and IntelliJ) 45 | com_crashlytics_export_strings.xml 46 | crashlytics.properties 47 | crashlytics-build.properties 48 | 49 | # logs 50 | *.log 51 | logs/* 52 | 53 | # ---> Java 54 | *.class 55 | 56 | # Mobile Tools for Java (J2ME) 57 | .mtj.tmp/ 58 | 59 | # Package Files # 60 | *.jar 61 | *.war 62 | *.ear 63 | 64 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 65 | hs_err_pid* 66 | 67 | # ---> Maven 68 | target/ 69 | pom.xml.tag 70 | pom.xml.releaseBackup 71 | pom.xml.versionsBackup 72 | pom.xml.next 73 | release.properties 74 | dependency-reduced-pom.xml 75 | buildNumber.properties 76 | .mvn/timing.properties 77 | 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 基于Spring Boot的监听Mysql数据变动的工具 2 | 3 | 使用时需要在配置文件中配置数据库的连接信息并配置binlog格式为ROW 4 | 5 | 使用方式 6 | ```JAVA 7 | @MysqlWatcher(hostName = "test-db", database = "test_binlog", table = "test_table1") 8 | public class TestTable1Watcher implements IMysqlDataListener { 9 | 10 | private Logger logger = LoggerFactory.getLogger(TestTable1Watcher.class); 11 | 12 | @Override 13 | public void onUpdate(TestTable1 from, TestTable1 to) { 14 | logger.info("ID 为 {} 的条目数据变更", from.getId()); 15 | logger.info("\t变化前:" + JSON.toJSONString(from, SerializerFeature.WriteDateUseDateFormat)); 16 | logger.info("\t变化后:" + JSON.toJSONString(to, SerializerFeature.WriteDateUseDateFormat)); 17 | } 18 | 19 | @Override 20 | public void onInsert(TestTable1 data) { 21 | logger.info("插入ID为 {} 的数据", data.getId()); 22 | } 23 | 24 | @Override 25 | public void onDelete(TestTable1 data) { 26 | logger.info("ID 为 {} 的数据被删除", data.getId()); 27 | } 28 | } 29 | ``` 30 | 31 | 可以监听mysql的数据变化同步数据到Redis或其他缓存，从业务层解耦 32 | -------------------------------------------------------------------------------- /mysql-binlog-demo/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | net.giafei.tools 8 | mysql-binlog-demo 9 | 1.0.1-SNAPSHOT 10 | 11 | 12 | 1.8 13 | 1.8 14 | 15 | 16 | 17 | 18 | org.springframework.boot 19 | spring-boot-starter 20 | 2.0.1.RELEASE 21 | 22 | 23 | net.giafei.tools 24 | mysql-binlog-starter 25 | 1.0.1 26 | 27 | 28 | -------------------------------------------------------------------------------- /mysql-binlog-demo/src/main/java/net/giafei/tools/demo/DemoApplication.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.demo; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication(scanBasePackages = {"net.giafei"}) 7 | public class DemoApplication { 8 | public static void main(String[] args) { 9 | SpringApplication.run(DemoApplication.class, args); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /mysql-binlog-demo/src/main/java/net/giafei/tools/demo/entity/TestTable1.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.demo.entity; 2 | 3 | import java.util.Date; 4 | 5 | public class TestTable1 { 6 | private long id; 7 | private String strValue; 8 | private Date dateValue; 9 | private double doubleValue; 10 | 11 | public long getId() { 12 | return id; 13 | } 14 | 15 | public void setId(long id) { 16 | this.id = id; 17 | } 18 | 19 | public String getStrValue() { 20 | return strValue; 21 | } 22 | 23 | public void setStrValue(String strValue) { 24 | this.strValue = strValue; 25 | } 26 | 27 | public Date getDateValue() { 28 | return dateValue; 29 | } 30 | 31 | public void setDateValue(Date dateValue) { 32 | this.dateValue = dateValue; 33 | } 34 | 35 | public double getDoubleValue() { 36 | return doubleValue; 37 | } 38 | 39 | public void setDoubleValue(double doubleValue) { 40 | this.doubleValue = doubleValue; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /mysql-binlog-demo/src/main/java/net/giafei/tools/demo/sql/create.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA `test_binlog` DEFAULT CHARACTER SET utf8mb4 ; 2 | 3 | CREATE TABLE `test_binlog`.`test_table1` ( 4 | `id` INT NOT NULL AUTO_INCREMENT, 5 | `str_value` VARCHAR(45) NULL, 6 | `date_value` DATETIME NULL, 7 | `double_value` DOUBLE NULL, 8 | PRIMARY KEY (`id`)); 9 | -------------------------------------------------------------------------------- /mysql-binlog-demo/src/main/java/net/giafei/tools/demo/sql/test.sql: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO `test_binlog`.`test_table1` (`str_value`, `date_value`, `double_value`) VALUES ('aaa', '2018-05-06 11:22:33', '1.65'); 3 | UPDATE `test_binlog`.`test_table1` SET `str_value`='bbb', `date_value`='2018-05-09 11:22:33', `double_value`='2.4' WHERE `id`='1'; 4 | DELETE FROM `test_binlog`.`test_table1` WHERE `id`='1'; 5 | -------------------------------------------------------------------------------- /mysql-binlog-demo/src/main/java/net/giafei/tools/demo/watcher/TestTable1Watcher.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.demo.watcher; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | import com.alibaba.fastjson.serializer.SerializerFeature; 5 | import net.giafei.tools.binlog.mysql.annotation.MysqlWatcher; 6 | import net.giafei.tools.binlog.mysql.listener.IMysqlDataListener; 7 | import net.giafei.tools.demo.entity.TestTable1; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | @MysqlWatcher(hostName = "test-db", database = "test_binlog", table = "test_table1") 12 | public class TestTable1Watcher implements IMysqlDataListener { 13 | 14 | private Logger logger = LoggerFactory.getLogger(TestTable1Watcher.class); 15 | 16 | @Override 17 | public void onUpdate(TestTable1 from, TestTable1 to) { 18 | logger.info("ID 为 {} 的条目数据变更", from.getId()); 19 | logger.info("\t变化前:" + JSON.toJSONString(from, SerializerFeature.WriteDateUseDateFormat)); 20 | logger.info("\t变化后:" + JSON.toJSONString(to, SerializerFeature.WriteDateUseDateFormat)); 21 | } 22 | 23 | @Override 24 | public void onInsert(TestTable1 data) { 25 | logger.info("插入ID为 {} 的数据", data.getId()); 26 | } 27 | 28 | @Override 29 | public void onDelete(TestTable1 data) { 30 | logger.info("ID 为 {} 的数据被删除", data.getId()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /mysql-binlog-demo/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | binlog: 2 | mysql: 3 | hosts: 4 | - name: test-db 5 | host: localhost 6 | port: 3306 7 | username: root 8 | password: root 9 | timeOffset: 28800000 10 | -------------------------------------------------------------------------------- /mysql-binlog-starter/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | net.giafei.tools 8 | mysql-binlog-starter 9 | 1.0.1 10 | 11 | 12 | 1.8 13 | 1.8 14 | 15 | 16 | 17 | 18 | org.springframework.boot 19 | spring-boot-starter 20 | 2.0.1.RELEASE 21 | provided 22 | 23 | 24 | 25 | mysql 26 | mysql-connector-java 27 | 5.1.46 28 | 29 | 30 | 31 | com.github.shyiko 32 | mysql-binlog-connector-java 33 | 0.16.1 34 | 35 | 36 | com.alibaba 37 | fastjson 38 | 1.2.46 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/annotation/MysqlWatcher.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.annotation; 2 | 3 | import org.springframework.stereotype.Component; 4 | 5 | import java.lang.annotation.*; 6 | 7 | /** 8 | * //////////////////////////////////////////////////////////////////// 9 | * // _ooOoo_ 10 | * // o8888888o 11 | * // 88" . "88 12 | * // (| ^_^ |) 13 | * // O\ = /O 14 | * // ____/`---'\____ 15 | * // .' \\| |// `. 16 | * // / \\||| : |||// \ 17 | * // / _||||| -:- |||||- \ 18 | * // | | \\\ - /// | | 19 | * // | \_| ''\---/'' | | 20 | * // \ .-\__ `-` ___/-. / 21 | * // ___`. .' /--.--\ `. . ___ 22 | * // ."" '< `.___\_<|>_/___.' >'"". 23 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 24 | * // \ \ `-. \_ __\ /__ _/ .-` / / 25 | * // ========`-.____`-.___\_____/___.-`____.-'======== 26 | * // `=---=' 27 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 28 | * // 佛祖保佑 永无BUG 永不修改 29 | * //////////////////////////////////////////////////////////////////// 30 | * 31 | * @author xjf 32 | * @version 1.0 33 | * Date 2018/9/21 13:07 34 | */ 35 | 36 | @Target(ElementType.TYPE) 37 | @Retention(RetentionPolicy.RUNTIME) 38 | @Documented 39 | @Component 40 | public @interface MysqlWatcher { 41 | String hostName(); 42 | String database(); 43 | String table(); 44 | } 45 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/config/MySqlHost.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.config; 2 | 3 | /** 4 | * //////////////////////////////////////////////////////////////////// 5 | * // _ooOoo_ 6 | * // o8888888o 7 | * // 88" . "88 8 | * // (| ^_^ |) 9 | * // O\ = /O 10 | * // ____/`---'\____ 11 | * // .' \\| |// `. 12 | * // / \\||| : |||// \ 13 | * // / _||||| -:- |||||- \ 14 | * // | | \\\ - /// | | 15 | * // | \_| ''\---/'' | | 16 | * // \ .-\__ `-` ___/-. / 17 | * // ___`. .' /--.--\ `. . ___ 18 | * // ."" '< `.___\_<|>_/___.' >'"". 19 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 20 | * // \ \ `-. \_ __\ /__ _/ .-` / / 21 | * // ========`-.____`-.___\_____/___.-`____.-'======== 22 | * // `=---=' 23 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 24 | * // 佛祖保佑 永无BUG 永不修改 25 | * //////////////////////////////////////////////////////////////////// 26 | * 27 | * @author xjf 28 | * @version 1.0 29 | * Date 2018/9/20 17:08 30 | */ 31 | 32 | public class MySqlHost { 33 | private String name; 34 | private String host; 35 | private int port; 36 | private String username; 37 | private String password; 38 | private long timeOffset; 39 | 40 | public String getName() { 41 | return name; 42 | } 43 | 44 | public void setName(String name) { 45 | this.name = name; 46 | } 47 | 48 | public String getHost() { 49 | return host; 50 | } 51 | 52 | public void setHost(String host) { 53 | this.host = host; 54 | } 55 | 56 | public int getPort() { 57 | return port; 58 | } 59 | 60 | public void setPort(int port) { 61 | this.port = port; 62 | } 63 | 64 | public String getUsername() { 65 | return username; 66 | } 67 | 68 | public void setUsername(String username) { 69 | this.username = username; 70 | } 71 | 72 | public String getPassword() { 73 | return password; 74 | } 75 | 76 | public void setPassword(String password) { 77 | this.password = password; 78 | } 79 | 80 | public long getTimeOffset() { 81 | return timeOffset; 82 | } 83 | 84 | public void setTimeOffset(long timeOffset) { 85 | this.timeOffset = timeOffset; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/config/MySqlHostProfile.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.config; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | import org.springframework.context.annotation.Configuration; 5 | 6 | import java.util.List; 7 | import java.util.Optional; 8 | 9 | /** 10 | * //////////////////////////////////////////////////////////////////// 11 | * // _ooOoo_ 12 | * // o8888888o 13 | * // 88" . "88 14 | * // (| ^_^ |) 15 | * // O\ = /O 16 | * // ____/`---'\____ 17 | * // .' \\| |// `. 18 | * // / \\||| : |||// \ 19 | * // / _||||| -:- |||||- \ 20 | * // | | \\\ - /// | | 21 | * // | \_| ''\---/'' | | 22 | * // \ .-\__ `-` ___/-. / 23 | * // ___`. .' /--.--\ `. . ___ 24 | * // ."" '< `.___\_<|>_/___.' >'"". 25 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 26 | * // \ \ `-. \_ __\ /__ _/ .-` / / 27 | * // ========`-.____`-.___\_____/___.-`____.-'======== 28 | * // `=---=' 29 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 30 | * // 佛祖保佑 永无BUG 永不修改 31 | * //////////////////////////////////////////////////////////////////// 32 | * 33 | * @author xjf 34 | * @version 1.0 35 | * Date 2018/9/20 17:09 36 | */ 37 | 38 | @Configuration 39 | @ConfigurationProperties(prefix = "binlog.mysql") 40 | public class MySqlHostProfile { 41 | private List hosts; 42 | 43 | public List getHosts() { 44 | return hosts; 45 | } 46 | 47 | public void setHosts(List hosts) { 48 | this.hosts = hosts; 49 | } 50 | 51 | public Optional getByName(String name) { 52 | return hosts.stream().filter(v -> name.equals(v.getName())) 53 | .findAny(); 54 | } 55 | 56 | public MySqlHost getByNameAndThrow(String name) { 57 | return getByName(name).orElseThrow(() -> new RuntimeException("未配置名为 "+name+" 的 binlog 连接信息")); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/core/BinLogBeanProcessor.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.core; 2 | 3 | import net.giafei.tools.binlog.mysql.config.MySqlHostProfile; 4 | import net.giafei.tools.binlog.mysql.listener.IMysqlDataListener; 5 | import net.giafei.tools.binlog.mysql.thread.BinlogThreadStarter; 6 | import org.springframework.beans.factory.SmartInitializingSingleton; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.context.ApplicationContext; 9 | import org.springframework.stereotype.Component; 10 | 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.stream.Collectors; 14 | 15 | /** 16 | * //////////////////////////////////////////////////////////////////// 17 | * // _ooOoo_ 18 | * // o8888888o 19 | * // 88" . "88 20 | * // (| ^_^ |) 21 | * // O\ = /O 22 | * // ____/`---'\____ 23 | * // .' \\| |// `. 24 | * // / \\||| : |||// \ 25 | * // / _||||| -:- |||||- \ 26 | * // | | \\\ - /// | | 27 | * // | \_| ''\---/'' | | 28 | * // \ .-\__ `-` ___/-. / 29 | * // ___`. .' /--.--\ `. . ___ 30 | * // ."" '< `.___\_<|>_/___.' >'"". 31 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 32 | * // \ \ `-. \_ __\ /__ _/ .-` / / 33 | * // ========`-.____`-.___\_____/___.-`____.-'======== 34 | * // `=---=' 35 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 36 | * // 佛祖保佑 永无BUG 永不修改 37 | * //////////////////////////////////////////////////////////////////// 38 | * 39 | * @author xjf 40 | * @version 1.0 41 | * Date 2018/9/20 17:22 42 | */ 43 | 44 | @Component 45 | public class BinLogBeanProcessor implements SmartInitializingSingleton { 46 | private ApplicationContext context; 47 | 48 | @Autowired 49 | private MySqlHostProfile profile; 50 | 51 | public BinLogBeanProcessor(ApplicationContext context) { 52 | this.context = context; 53 | } 54 | 55 | @Override 56 | public void afterSingletonsInstantiated() { 57 | Map beans = context.getBeansOfType(IMysqlDataListener.class); 58 | 59 | Map> listeners = beans.values().stream() 60 | .map(MysqlDataListenerData::new) 61 | .collect(Collectors.groupingBy(MysqlDataListenerData::getHostName)); 62 | 63 | listeners.forEach((k, v) -> new BinlogThreadStarter().runThread(profile.getByNameAndThrow(k), v)); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/core/BinlogDataDispatcher.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.core; 2 | 3 | import com.github.shyiko.mysql.binlog.BinaryLogClient; 4 | import com.github.shyiko.mysql.binlog.event.*; 5 | 6 | import java.io.Serializable; 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | /** 12 | * //////////////////////////////////////////////////////////////////// 13 | * // _ooOoo_ 14 | * // o8888888o 15 | * // 88" . "88 16 | * // (| ^_^ |) 17 | * // O\ = /O 18 | * // ____/`---'\____ 19 | * // .' \\| |// `. 20 | * // / \\||| : |||// \ 21 | * // / _||||| -:- |||||- \ 22 | * // | | \\\ - /// | | 23 | * // | \_| ''\---/'' | | 24 | * // \ .-\__ `-` ___/-. / 25 | * // ___`. .' /--.--\ `. . ___ 26 | * // ."" '< `.___\_<|>_/___.' >'"". 27 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 28 | * // \ \ `-. \_ __\ /__ _/ .-` / / 29 | * // ========`-.____`-.___\_____/___.-`____.-'======== 30 | * // `=---=' 31 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 32 | * // 佛祖保佑 永无BUG 永不修改 33 | * //////////////////////////////////////////////////////////////////// 34 | * 35 | * @author xjf 36 | * @version 1.0 37 | * Date 2018/9/20 17:24 38 | */ 39 | 40 | public class BinlogDataDispatcher implements BinaryLogClient.EventListener { 41 | private Map tableNameMap = new HashMap<>(); 42 | private Map> listenerMap = new HashMap<>(); 43 | 44 | public void addListener(String database, String table, List listeners) { 45 | String key = database + "." + table; 46 | this.listenerMap.put(key, listeners); 47 | } 48 | 49 | @Override 50 | public void onEvent(Event event) { 51 | EventHeaderV4 header = event.getHeader(); 52 | 53 | EventType eventType = header.getEventType(); 54 | if (eventType == EventType.TABLE_MAP) { 55 | MySqlTable table = new MySqlTable(event.getData()); 56 | String key = table.getDatabase() + "." + table.getTable(); 57 | 58 | if (this.listenerMap.containsKey(key)) 59 | tableNameMap.put(table.getId(), table); 60 | } else if (eventType == EventType.EXT_UPDATE_ROWS) { 61 | UpdateRowsEventData data = event.getData(); 62 | if (!tableNameMap.containsKey(data.getTableId())) 63 | return; 64 | 65 | dispatchEvent(data); 66 | } else if (eventType == EventType.EXT_WRITE_ROWS) { 67 | WriteRowsEventData data = event.getData(); 68 | if (!tableNameMap.containsKey(data.getTableId())) 69 | return; 70 | dispatchEvent(data); 71 | } else if (eventType == EventType.EXT_DELETE_ROWS) { 72 | DeleteRowsEventData data = event.getData(); 73 | if (!tableNameMap.containsKey(data.getTableId())) 74 | return; 75 | 76 | dispatchEvent(data); 77 | } 78 | } 79 | 80 | private void dispatchEvent(UpdateRowsEventData data) { 81 | MySqlTable table = tableNameMap.get(data.getTableId()); 82 | String key = table.getDatabase() + "." + table.getTable(); 83 | 84 | List containers = listenerMap.get(key); 85 | List> rows = data.getRows(); 86 | containers.forEach(c -> c.invokeUpdate(rows)); 87 | } 88 | 89 | private void dispatchEvent(DeleteRowsEventData data) { 90 | MySqlTable table = tableNameMap.get(data.getTableId()); 91 | String key = table.getDatabase() + "." + table.getTable(); 92 | 93 | List containers = listenerMap.get(key); 94 | List rows = data.getRows(); 95 | containers.forEach(c -> c.invokeDelete(rows)); 96 | } 97 | 98 | private void dispatchEvent(WriteRowsEventData data) { 99 | MySqlTable table = tableNameMap.get(data.getTableId()); 100 | String key = table.getDatabase() + "." + table.getTable(); 101 | 102 | List containers = listenerMap.get(key); 103 | List rows = data.getRows(); 104 | containers.forEach(c -> c.invokeInsert(rows)); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/core/DataListenerContainer.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.core; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.alibaba.fastjson.PropertyNamingStrategy; 5 | import com.alibaba.fastjson.parser.ParserConfig; 6 | import com.alibaba.fastjson.util.TypeUtils; 7 | import net.giafei.tools.binlog.mysql.listener.IMysqlDataListener; 8 | 9 | import java.io.Serializable; 10 | import java.util.Date; 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.function.Consumer; 15 | 16 | /** 17 | * //////////////////////////////////////////////////////////////////// 18 | * // _ooOoo_ 19 | * // o8888888o 20 | * // 88" . "88 21 | * // (| ^_^ |) 22 | * // O\ = /O 23 | * // ____/`---'\____ 24 | * // .' \\| |// `. 25 | * // / \\||| : |||// \ 26 | * // / _||||| -:- |||||- \ 27 | * // | | \\\ - /// | | 28 | * // | \_| ''\---/'' | | 29 | * // \ .-\__ `-` ___/-. / 30 | * // ___`. .' /--.--\ `. . ___ 31 | * // ."" '< `.___\_<|>_/___.' >'"". 32 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 33 | * // \ \ `-. \_ __\ /__ _/ .-` / / 34 | * // ========`-.____`-.___\_____/___.-`____.-'======== 35 | * // `=---=' 36 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 37 | * // 佛祖保佑 永无BUG 永不修改 38 | * //////////////////////////////////////////////////////////////////// 39 | * 40 | * @author xjf 41 | * @version 1.0 42 | * Date 2018/9/21 9:57 43 | */ 44 | 45 | public class DataListenerContainer { 46 | private Class entityClass; 47 | private IMysqlDataListener listener; 48 | private String[] columnName; 49 | 50 | //从binlog读取的时间与真实值有偏差 与 服务端的时区有关 51 | private long timeOffset = 0; 52 | 53 | private static final ParserConfig snakeCase; 54 | static { 55 | snakeCase = new ParserConfig(); 56 | snakeCase.propertyNamingStrategy = PropertyNamingStrategy.SnakeCase; 57 | } 58 | 59 | public DataListenerContainer(Class entityClass, IMysqlDataListener listener, String[] columnName) { 60 | this.entityClass = entityClass; 61 | this.listener = listener; 62 | this.columnName = columnName; 63 | } 64 | 65 | public DataListenerContainer(Class entityClass, IMysqlDataListener listener, String[] columnName, long timeOffset) { 66 | this.entityClass = entityClass; 67 | this.listener = listener; 68 | this.columnName = columnName; 69 | this.timeOffset = timeOffset; 70 | } 71 | 72 | public void setTimeOffset(long timeOffset) { 73 | this.timeOffset = timeOffset; 74 | } 75 | 76 | public void invokeInsert(List data) { 77 | invokeSingle(data, listener::onInsert); 78 | } 79 | 80 | public void invokeDelete(List data) { 81 | invokeSingle(data, listener::onDelete); 82 | } 83 | 84 | public void invokeUpdate(List> data) { 85 | data.forEach(row -> { 86 | listener.onUpdate(toEntity(row.getKey()), toEntity(row.getValue())); 87 | }); 88 | } 89 | 90 | private T toEntity(Serializable[] data) { 91 | for (int i = 0; i < data.length; i++) { 92 | Serializable da = data[i]; 93 | if (da instanceof Date) { 94 | data[i] = new Date(((Date)da).getTime() + timeOffset); 95 | } 96 | } 97 | 98 | Map b = new HashMap<>(data.length); 99 | for (int i = 0; i < data.length; i++) { 100 | b.put(columnName[i], data[i]); 101 | } 102 | 103 | return TypeUtils.cast(b, entityClass, snakeCase); 104 | } 105 | 106 | private void invokeSingle(List data, Consumer consumer) { 107 | data.stream().map(this::toEntity) 108 | .forEach(consumer); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/core/MySqlTable.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.core; 2 | 3 | import com.github.shyiko.mysql.binlog.event.TableMapEventData; 4 | 5 | /** 6 | * //////////////////////////////////////////////////////////////////// 7 | * // _ooOoo_ 8 | * // o8888888o 9 | * // 88" . "88 10 | * // (| ^_^ |) 11 | * // O\ = /O 12 | * // ____/`---'\____ 13 | * // .' \\| |// `. 14 | * // / \\||| : |||// \ 15 | * // / _||||| -:- |||||- \ 16 | * // | | \\\ - /// | | 17 | * // | \_| ''\---/'' | | 18 | * // \ .-\__ `-` ___/-. / 19 | * // ___`. .' /--.--\ `. . ___ 20 | * // ."" '< `.___\_<|>_/___.' >'"". 21 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 22 | * // \ \ `-. \_ __\ /__ _/ .-` / / 23 | * // ========`-.____`-.___\_____/___.-`____.-'======== 24 | * // `=---=' 25 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 26 | * // 佛祖保佑 永无BUG 永不修改 27 | * //////////////////////////////////////////////////////////////////// 28 | * 29 | * @author xjf 30 | * @version 1.0 31 | * Date 2018/9/20 17:28 32 | */ 33 | 34 | public class MySqlTable { 35 | private long id; 36 | private String database; 37 | private String table; 38 | 39 | public MySqlTable(TableMapEventData data) { 40 | this.id = data.getTableId(); 41 | this.database = data.getDatabase(); 42 | this.table = data.getTable(); 43 | } 44 | 45 | public long getId() { 46 | return id; 47 | } 48 | 49 | public String getDatabase() { 50 | return database; 51 | } 52 | 53 | public String getTable() { 54 | return table; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/core/MysqlDataListenerData.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.core; 2 | 3 | import net.giafei.tools.binlog.mysql.annotation.MysqlWatcher; 4 | import net.giafei.tools.binlog.mysql.listener.IMysqlDataListener; 5 | import org.springframework.aop.support.AopUtils; 6 | import org.springframework.core.annotation.AnnotationUtils; 7 | 8 | import java.lang.reflect.ParameterizedType; 9 | import java.lang.reflect.Type; 10 | 11 | /** 12 | * //////////////////////////////////////////////////////////////////// 13 | * // _ooOoo_ 14 | * // o8888888o 15 | * // 88" . "88 16 | * // (| ^_^ |) 17 | * // O\ = /O 18 | * // ____/`---'\____ 19 | * // .' \\| |// `. 20 | * // / \\||| : |||// \ 21 | * // / _||||| -:- |||||- \ 22 | * // | | \\\ - /// | | 23 | * // | \_| ''\---/'' | | 24 | * // \ .-\__ `-` ___/-. / 25 | * // ___`. .' /--.--\ `. . ___ 26 | * // ."" '< `.___\_<|>_/___.' >'"". 27 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 28 | * // \ \ `-. \_ __\ /__ _/ .-` / / 29 | * // ========`-.____`-.___\_____/___.-`____.-'======== 30 | * // `=---=' 31 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 32 | * // 佛祖保佑 永无BUG 永不修改 33 | * //////////////////////////////////////////////////////////////////// 34 | * 35 | * @author xjf 36 | * @version 1.0 37 | * Date 2018/9/26 9:15 38 | */ 39 | 40 | public class MysqlDataListenerData { 41 | private IMysqlDataListener listener; 42 | private String hostName; 43 | private String database; 44 | private String table; 45 | private Class entityClass; 46 | 47 | public MysqlDataListenerData(IMysqlDataListener listener) { 48 | this.listener = listener; 49 | 50 | Class targetClass = AopUtils.getTargetClass(listener); 51 | MysqlWatcher annotation = AnnotationUtils.findAnnotation(targetClass, MysqlWatcher.class); 52 | if (annotation == null) 53 | throw new RuntimeException("Mysql binlog listener必须添加 MysqlWatcher 注解"); 54 | 55 | hostName = annotation.hostName(); 56 | database = annotation.database(); 57 | table = annotation.table(); 58 | entityClass = getGenericClass(targetClass); 59 | } 60 | 61 | public IMysqlDataListener getListener() { 62 | return listener; 63 | } 64 | 65 | public String getHostName() { 66 | return hostName; 67 | } 68 | 69 | public String getDatabase() { 70 | return database; 71 | } 72 | 73 | public String getTable() { 74 | return table; 75 | } 76 | 77 | public Class getEntityClass() { 78 | return entityClass; 79 | } 80 | 81 | private Class getGenericClass(Class targetClass) { 82 | if (targetClass == Object.class) 83 | return null; 84 | 85 | Type[] types = targetClass.getGenericInterfaces(); 86 | if (types.length == 0) { 87 | types = new Type[] {targetClass.getGenericSuperclass()}; 88 | } 89 | 90 | for (Type type : types) { 91 | if (type instanceof ParameterizedType) { 92 | ParameterizedType t = (ParameterizedType) type; 93 | Type[] array = t.getActualTypeArguments(); 94 | return (Class) array[0]; 95 | } 96 | } 97 | 98 | return getGenericClass(targetClass.getSuperclass()); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/listener/AbstractMysqlDataImporter.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.listener; 2 | 3 | /** 4 | * //////////////////////////////////////////////////////////////////// 5 | * // _ooOoo_ 6 | * // o8888888o 7 | * // 88" . "88 8 | * // (| ^_^ |) 9 | * // O\ = /O 10 | * // ____/`---'\____ 11 | * // .' \\| |// `. 12 | * // / \\||| : |||// \ 13 | * // / _||||| -:- |||||- \ 14 | * // | | \\\ - /// | | 15 | * // | \_| ''\---/'' | | 16 | * // \ .-\__ `-` ___/-. / 17 | * // ___`. .' /--.--\ `. . ___ 18 | * // ."" '< `.___\_<|>_/___.' >'"". 19 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 20 | * // \ \ `-. \_ __\ /__ _/ .-` / / 21 | * // ========`-.____`-.___\_____/___.-`____.-'======== 22 | * // `=---=' 23 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 24 | * // 佛祖保佑 永无BUG 永不修改 25 | * //////////////////////////////////////////////////////////////////// 26 | * 27 | * @author xjf 28 | * @version 1.0 29 | * Date 2018/9/21 9:08 30 | */ 31 | 32 | public abstract class AbstractMysqlDataImporter implements IMysqlDataListener { 33 | @Override 34 | public void onUpdate(T from, T to) { 35 | onData(to); 36 | } 37 | 38 | @Override 39 | public void onInsert(T data) { 40 | onData(data); 41 | } 42 | 43 | protected abstract void onData(T data); 44 | } 45 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/listener/IMysqlDataListener.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.listener; 2 | 3 | /** 4 | * //////////////////////////////////////////////////////////////////// 5 | * // _ooOoo_ 6 | * // o8888888o 7 | * // 88" . "88 8 | * // (| ^_^ |) 9 | * // O\ = /O 10 | * // ____/`---'\____ 11 | * // .' \\| |// `. 12 | * // / \\||| : |||// \ 13 | * // / _||||| -:- |||||- \ 14 | * // | | \\\ - /// | | 15 | * // | \_| ''\---/'' | | 16 | * // \ .-\__ `-` ___/-. / 17 | * // ___`. .' /--.--\ `. . ___ 18 | * // ."" '< `.___\_<|>_/___.' >'"". 19 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 20 | * // \ \ `-. \_ __\ /__ _/ .-` / / 21 | * // ========`-.____`-.___\_____/___.-`____.-'======== 22 | * // `=---=' 23 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 24 | * // 佛祖保佑 永无BUG 永不修改 25 | * //////////////////////////////////////////////////////////////////// 26 | * 27 | * @author xjf 28 | * @version 1.0 29 | * Date 2018/9/21 9:03 30 | */ 31 | 32 | public interface IMysqlDataListener { 33 | void onUpdate(T from, T to); 34 | void onInsert(T data); 35 | void onDelete(T data); 36 | } 37 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/thread/BinLogListenerThread.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.thread; 2 | 3 | import com.github.shyiko.mysql.binlog.BinaryLogClient; 4 | import net.giafei.tools.binlog.mysql.config.MySqlHost; 5 | import net.giafei.tools.binlog.mysql.core.BinlogDataDispatcher; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.io.IOException; 10 | 11 | /** 12 | * //////////////////////////////////////////////////////////////////// 13 | * // _ooOoo_ 14 | * // o8888888o 15 | * // 88" . "88 16 | * // (| ^_^ |) 17 | * // O\ = /O 18 | * // ____/`---'\____ 19 | * // .' \\| |// `. 20 | * // / \\||| : |||// \ 21 | * // / _||||| -:- |||||- \ 22 | * // | | \\\ - /// | | 23 | * // | \_| ''\---/'' | | 24 | * // \ .-\__ `-` ___/-. / 25 | * // ___`. .' /--.--\ `. . ___ 26 | * // ."" '< `.___\_<|>_/___.' >'"". 27 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 28 | * // \ \ `-. \_ __\ /__ _/ .-` / / 29 | * // ========`-.____`-.___\_____/___.-`____.-'======== 30 | * // `=---=' 31 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 32 | * // 佛祖保佑 永无BUG 永不修改 33 | * //////////////////////////////////////////////////////////////////// 34 | * 35 | * @author xjf 36 | * @version 1.0 37 | * Date 2018/9/21 13:35 38 | */ 39 | 40 | public class BinLogListenerThread implements Runnable { 41 | private MySqlHost host; 42 | private BinlogDataDispatcher listener; 43 | 44 | private Logger logger = LoggerFactory.getLogger(BinLogListenerThread.class); 45 | 46 | public BinLogListenerThread(MySqlHost host, BinlogDataDispatcher listener) { 47 | this.host = host; 48 | this.listener = listener; 49 | } 50 | 51 | @Override 52 | public void run() { 53 | BinaryLogClient client = new BinaryLogClient(host.getHost(), host.getPort(), host.getUsername(), host.getPassword()); 54 | 55 | client.registerEventListener(listener); 56 | 57 | while (true) { 58 | try { 59 | client.connect(); 60 | } catch (IOException e) { 61 | logger.error("{}:{}监听器错误", host.getHost(), host.getPort(), e); 62 | } 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /mysql-binlog-starter/src/main/java/net/giafei/tools/binlog/mysql/thread/BinlogThreadStarter.java: -------------------------------------------------------------------------------- 1 | package net.giafei.tools.binlog.mysql.thread; 2 | 3 | import net.giafei.tools.binlog.mysql.config.MySqlHost; 4 | import net.giafei.tools.binlog.mysql.core.BinlogDataDispatcher; 5 | import net.giafei.tools.binlog.mysql.core.DataListenerContainer; 6 | import net.giafei.tools.binlog.mysql.core.MysqlDataListenerData; 7 | 8 | import java.sql.*; 9 | import java.util.ArrayList; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.stream.Collectors; 14 | 15 | /** 16 | * //////////////////////////////////////////////////////////////////// 17 | * // _ooOoo_ 18 | * // o8888888o 19 | * // 88" . "88 20 | * // (| ^_^ |) 21 | * // O\ = /O 22 | * // ____/`---'\____ 23 | * // .' \\| |// `. 24 | * // / \\||| : |||// \ 25 | * // / _||||| -:- |||||- \ 26 | * // | | \\\ - /// | | 27 | * // | \_| ''\---/'' | | 28 | * // \ .-\__ `-` ___/-. / 29 | * // ___`. .' /--.--\ `. . ___ 30 | * // ."" '< `.___\_<|>_/___.' >'"". 31 | * // | | : `- \`.;`\ _ /`;.`/ - ` : | | 32 | * // \ \ `-. \_ __\ /__ _/ .-` / / 33 | * // ========`-.____`-.___\_____/___.-`____.-'======== 34 | * // `=---=' 35 | * // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 36 | * // 佛祖保佑 永无BUG 永不修改 37 | * //////////////////////////////////////////////////////////////////// 38 | * 39 | * @author xjf 40 | * @version 1.0 41 | * Date 2018/9/21 13:40 42 | */ 43 | 44 | public class BinlogThreadStarter { 45 | static { 46 | try { 47 | Class.forName("com.mysql.jdbc.Driver"); 48 | } catch (ClassNotFoundException e) { 49 | e.printStackTrace(); 50 | throw new RuntimeException(e); 51 | } 52 | } 53 | 54 | //设计失误 其实只需要一个connection 已经写成这样了 就先不改了 55 | private Map connectionPool = new HashMap<>(); 56 | private Connection getConnection(MySqlHost host) throws SQLException { 57 | String key = host.getHost() + ":" + host.getPort(); 58 | Connection connection = connectionPool.get(key); 59 | 60 | if (connection == null) { 61 | String url = "jdbc:mysql://" + key + 62 | "/INFORMATION_SCHEMA?useUnicode=true&characterEncoding=UTF-8&useSSL=false"; 63 | 64 | connection = DriverManager.getConnection(url, host.getUsername(), host.getPassword()); 65 | connectionPool.put(key, connection); 66 | } 67 | 68 | return connection; 69 | } 70 | 71 | private void releaseConnection() { 72 | for (Map.Entry entry : connectionPool.entrySet()) { 73 | try { 74 | entry.getValue().close(); 75 | } catch (SQLException e) { 76 | //不用管 77 | } 78 | } 79 | 80 | connectionPool.clear(); 81 | } 82 | 83 | public void runThread(MySqlHost host, List listeners) { 84 | Map> map = listeners.stream() 85 | .collect(Collectors.groupingBy(l -> l.getDatabase() + ":" + l.getTable())); 86 | 87 | BinlogDataDispatcher logListener = new BinlogDataDispatcher(); 88 | 89 | map.forEach((k, v) -> { 90 | String[] arr = k.split(":"); 91 | String[] columns = getColumns(host, arr[0], arr[1]); 92 | 93 | List containers = v.stream() 94 | .map(l -> new DataListenerContainer(l.getEntityClass(), l.getListener(), columns, host.getTimeOffset())) 95 | .collect(Collectors.toList()); 96 | 97 | logListener.addListener(arr[0], arr[1], containers); 98 | }); 99 | 100 | new Thread(new BinLogListenerThread(host, logListener)).start(); 101 | 102 | releaseConnection(); 103 | } 104 | 105 | private String[] getColumns(MySqlHost host, String db, String table) { 106 | try { 107 | Connection connection = getConnection(host); 108 | Statement statement = connection.createStatement(); 109 | 110 | String sql = "select COLUMN_NAME from INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA='" 111 | + db + "' and TABLE_NAME='" + table + "' order by ORDINAL_POSITION asc;"; 112 | 113 | ResultSet resultSet = statement.executeQuery(sql); 114 | List buf = new ArrayList<>(); 115 | 116 | while (resultSet.next()) { 117 | buf.add(resultSet.getString(1)); 118 | } 119 | 120 | return buf.toArray(new String[0]); 121 | } catch (SQLException e) { 122 | throw new RuntimeException(e); 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | net.giafei.tools 8 | mysql-binlog-watcher 9 | pom 10 | 1.0-SNAPSHOT 11 | 12 | mysql-binlog-starter 13 | mysql-binlog-demo 14 | 15 | 16 | 17 | --------------------------------------------------------------------------------