├── src
    ├── main
    │   ├── .DS_Store
    │   ├── resources
    │   │   ├── binlog2Hive_conf.properties
    │   │   ├── mysql.properties
    │   │   ├── core-site.xml
    │   │   ├── hdfs-site.xml
    │   │   └── log4j.properties
    │   └── java
    │   │   └── com
    │   │       ├── config
    │   │           ├── DatabaseConnection.java
    │   │           └── Config.java
    │   │       ├── common
    │   │           ├── ExecCMD.java
    │   │           ├── BinlogOption.java
    │   │           └── FSUtils.java
    │   │       ├── main
    │   │           └── Binlog2Hive.java
    │   │       └── bixuange
    │   │           └── BinlogClient.java
    └── test
    │   └── java
    │       └── com
    │           └── bixuange
    │               ├── OrcTest.java
    │               ├── FileSystemTest.java
    │               ├── BinlogClientTest.java
    │               └── AppTest.java
├── doc
    └── CREATE_TABLE.sql
├── README.md
├── pom.xml
└── Binlog2Hive.iml


/src/main/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MOBIN-F/Binlog2Hive/HEAD/src/main/.DS_Store


--------------------------------------------------------------------------------
/src/main/resources/binlog2Hive_conf.properties:
--------------------------------------------------------------------------------
1 | ##hive表的路径
2 | default_table1_path = /DATA/PUBLIC/TABLE1


--------------------------------------------------------------------------------
/doc/CREATE_TABLE.sql:
--------------------------------------------------------------------------------
 1 | ##记录Binlog同步游标的表
 2 | CREATE TABLE binglogDB.t_position(nextposition LONG ,binlogfilename VARCHAR(20))
 3 | 
 4 | CREATE EXTERNAL TABLE IF NOT EXISTS TABLE1(
 5 | KEYID bigint comment '记录序号',
 6 | DDATETIME string comment '数据时间',
 7 | DRP double comment 'xxx',
 8 | DHP double comment 'xxx',
 9 | DVP double comment 'xxx',
10 | YYP double comment 'xxx',
11 | )partitioned by (day int)
12 | ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
13 | LINES TERMINATED BY '\n'
14 | location '/DATA/PUBLIC/TABLE1/';


--------------------------------------------------------------------------------
/src/main/resources/mysql.properties:
--------------------------------------------------------------------------------
 1 | driverClassName = com.mysql.jdbc.Driver
 2 | url = jdbc:mysql://localhost:3306/binlogdb?characterEncoding=utf-8&rewriteBatchedStatements=true
 3 | username = root
 4 | password = root
 5 | 
 6 | initialSize = 5
 7 | maxActive = 10
 8 | minIdle = 3
 9 | maxWait = 600000
10 | removeAbandoned = true
11 | removeAbandonedTimeout = 180
12 | timeBetweenEvictionRunsMillis = 60000
13 | minEvictableIdleTimeMillis = 300000
14 | validationQuery = SELECT 1 FROM DUAL
15 | testWhileIdle = true 
16 | testOnBorrow = false
17 | testOnReturn = false
18 | poolPreparedStatements = false
19 | maxPoolPreparedStatementPerConnectionSize = 50
20 | filters = stat


--------------------------------------------------------------------------------
/src/main/resources/core-site.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | 
 3 | <configuration>
 4 |   <property>
 5 |     <name>fs.defaultFS</name>
 6 |       <value>hdfs://localhost:9001</value>
 7 |   </property>
 8 |   <property>
 9 |     <name>io.compression.codecs</name>
10 |     <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec</value>
11 |   </property>
12 |   <property>
13 |     <name>io.compression.codec.lzo.class</name>
14 |     <value>com.hadoop.compression.lzo.LzoCodec</value>
15 |   </property>
16 | </configuration>
17 | 


--------------------------------------------------------------------------------
/src/main/java/com/config/DatabaseConnection.java:
--------------------------------------------------------------------------------
 1 | package com.config;
 2 | import com.alibaba.druid.pool.DruidDataSourceFactory;
 3 | 
 4 | import javax.sql.DataSource;
 5 | import java.io.InputStream;
 6 | import java.sql.Connection;
 7 | import java.sql.SQLException;
 8 | import java.util.Properties;
 9 | 
10 | /**
11 |  * Created by Mobin on 2017/9/12.
12 |  */
13 | public class DatabaseConnection {
14 |     private static DataSource MysqlDataSource;
15 |     public DatabaseConnection(){}
16 |     static {
17 |         try{
18 |             InputStream in = DatabaseConnection.class.getClassLoader()
19 |                     .getResourceAsStream("mysql.properties");
20 |             Properties props = new Properties();
21 |             props.load(in);
22 |             MysqlDataSource = DruidDataSourceFactory.createDataSource(props);
23 |         }catch(Exception ex){
24 |             ex.printStackTrace();
25 |         }
26 |     }
27 |     public static Connection getMysqlConnection() throws SQLException{
28 |         return MysqlDataSource.getConnection();
29 |     }
30 | 
31 | }
32 | 


--------------------------------------------------------------------------------
/src/main/resources/hdfs-site.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | 
 3 | <configuration>
 4 |     <property>
 5 |         <name>dfs.client.block.write.replace-datanode-on-failure.enable</name>
 6 |         <value>true</value>
 7 |     </property>
 8 |     <property>
 9 |         <name>dfs.client.block.write.replace-datanode-on-failure.policy</name>
10 |         <value>NEVER</value>
11 |     </property>
12 |     <property>
13 |         <name>dfs.replication</name>
14 |         <value>1</value>
15 |     </property>
16 |     <property>
17 |         <name>dfs.datanode.fsdataset.volume.choosing.policy</name>
18 |         <value>org.apache.hadoop.hdfs.server.datanode.fsdataset.AvailableSpaceVolumeChoosingPolicy</value>
19 |     </property>
20 |     <property>
21 |         <name>dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold</name>
22 |         <value>10737418240</value>
23 |     </property>
24 |     <property>
25 |         <name>dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction</name>
26 |         <value>0.75</value>
27 |     </property>
28 | </configuration>
29 | 


--------------------------------------------------------------------------------
/src/main/java/com/config/Config.java:
--------------------------------------------------------------------------------
 1 | package com.config;
 2 | 
 3 | import java.io.IOException;
 4 | import java.io.InputStream;
 5 | import java.util.Properties;
 6 | 
 7 | /**
 8 |  * Created with IDEA
 9 |  * Creater: MOBIN
10 |  * Date: 2018/5/17
11 |  * Time: 上午10:20
12 |  */
13 | public class Config {
14 | 
15 |     private static final Properties config = loadConfig();
16 | 
17 |     private static Properties loadConfig(){
18 |         InputStream in = Config.class.getResourceAsStream("/binlog2Hive_conf.properties");
19 |         Properties config = new Properties();
20 |         try {
21 |             config.load(in);
22 |             in.close();
23 |         } catch (IOException e) {
24 |             throw new RuntimeException("Failed to load binlog2Hive_conf.properties");
25 |         }
26 |         return config;
27 |     }
28 | 
29 |     public static String getStringProperty(String key){
30 |         String value = config.getProperty(key);
31 |         if (value == null)
32 |             return null;
33 |         return value.trim();
34 |     }
35 | 
36 |     public static String getPath(String tableName){
37 |         String path = getStringProperty("default_" + tableName + "_path");
38 |         return path;
39 |     }
40 | }
41 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ## 项目背景
 2 | >RDS的数据实时同步到HDFS下，并映射到Hive
 3 | 
 4 | ## 原理
 5 | >通过解析RDS的binlog将RDS的增量数据同步到HDFS下，并映射加载到Hive外部分区表
 6 | 
 7 | 由于RDS表中的第二个字段都为datetime字段，所以刚才以该字段作为Hive的分区字段
 8 | 
 9 | ### 配置文件介绍
10 | * doc/creat table.sql：Hive表的建表语句，除了静态表外，其他全部为天级别外部分区表
11 | * binglog2Hive_conf.properties:里面为所有全部需要同步到HDFS的表
12 | * mysql.properties:Mysql druid连接池配置
13 | 
14 | ### 程序说明
15 | binlog解析框架：https://github.com/shyiko/mysql-binlog-connector-java
16 | 
17 | 核心类为BinlogClient
18 | 1. 程序主要序列化以下几个事件
19 | * TABLE_MAP：包括表名，数据库名
20 | * WRITE_ROWS：包含增量的业务记录
21 | 
22 | 2. 
23 | * 程序启动时会先从t_position表中获取上次的同步状态，根据上次的同步状态来确定binlog的读取位置
24 | * BinaryLogClient首先对TABLE_MAP事件进行序列化，再结合binlog2Hive_conf.propertiesd配置过滤出我们需要同步的表再对WRITE_ROW事件进行序列化
25 | * 解析WRITE_ROWS时，将</DATA/PUBLIC/表名，记录>存储到ConcurrentHashMap<String, List<Serializable[]>> mapA中
26 | * 解析的记录超过一定阀值option.countInterval后再统一写HDFS文件
27 | * 写HDFS文件时，遍历mapA，根据表名分类，整理成</DATA/PUBLIC/表名/day=xxx,记录>存储到
28 | ConcurrentHashMap<String, ArrayList<Serializable[]>> mapB，最后再统一遍历mapB将数据写入到HDFS，写到哪个文件中是根据mapB的key来确定的
29 | * 文件操作类在FSUtils中，写文件时以下三种情况
30 |   1. 如果目录不存在就创建文件并将Hive表的分区映射到这个路径下，
31 |   2. 文件已存在且文件大小小于250MB就以追加的方式写文件
32 |   3. 文件大小超250MB就重新写成另一个新文件，以HDFS_BLOCK_SIZE为标准
33 | * 文件写入成功后将当前的同步状态(binlogfilename,nextposition)更新到t_position表中
34 | 
35 | **项目已经去掉敏感业务信息**
36 | 
37 | 
38 | 
39 | 


--------------------------------------------------------------------------------
/src/main/java/com/common/ExecCMD.java:
--------------------------------------------------------------------------------
 1 | package com.common;
 2 | 
 3 | import com.google.common.io.LineReader;
 4 | import org.slf4j.Logger;
 5 | import org.slf4j.LoggerFactory;
 6 | 
 7 | import java.io.IOException;
 8 | import java.io.InputStreamReader;
 9 | 
10 | /**
11 |  * Created with IDEA
12 |  * Creater: MOBIN
13 |  * Date: 2018/5/18
14 |  * Time: 下午9:56
15 |  */
16 | public class ExecCMD {
17 |     private static final Logger log = LoggerFactory.getLogger(ExecCMD.class);
18 | 
19 |     public static String exec(String cmd){
20 |         try {
21 |             String[] cmds = {"/bin/sh", "-c", cmd};
22 |             Process process = Runtime.getRuntime().exec(cmds);
23 |             LineReader lineReader = new LineReader(new InputStreamReader(process.getInputStream()));
24 |             StringBuffer sb = new StringBuffer();
25 |             String line;
26 |             while ((line = lineReader.readLine()) != null){
27 |                 sb.append(line).append("\n");
28 |             }
29 |             String rs = sb.toString();
30 |             if (!cmd.isEmpty())
31 |                 log.info("cmd executed, result: " + rs);
32 |             return rs;
33 |         } catch (IOException e) {
34 |             log.error("Failed to exec cmd: " + cmd, e);
35 |         }
36 |         return null;
37 |     }
38 | }
39 | 


--------------------------------------------------------------------------------
/src/main/java/com/main/Binlog2Hive.java:
--------------------------------------------------------------------------------
 1 | package com.main;
 2 | 
 3 | import com.common.BinlogOption;
 4 | import com.bixuange.BinlogClient;
 5 | import com.common.FSUtils;
 6 | import org.apache.hadoop.fs.FileSystem;
 7 | import org.slf4j.Logger;
 8 | import org.slf4j.LoggerFactory;
 9 | 
10 | import java.io.IOException;
11 | 
12 | /**
13 |  * Created with IDEA
14 |  * Creater: MOBIN
15 |  * Date: 2018/5/20
16 |  * Time: 下午10:09
17 |  */
18 | public class Binlog2Hive {
19 |     private static final Logger log = LoggerFactory.getLogger(Binlog2Hive.class);
20 |     static {
21 |         //-DHADOOP_USER_NAME=hdfs
22 |         System.setProperty("HADOOP_USER_NAME","hdfs");
23 |     }
24 |     //227527999，binlogFileName文件为mysql-bin.000043
25 |     //Users/mobin/Downloads/RDS-binlog/mysql-bin.000642
26 |     public static void main(String[] args) throws Exception {
27 |         BinlogClient client = new BinlogClient();
28 |         BinlogOption option = new BinlogOption(args);
29 |         FileSystem fs = FSUtils.getFileSystem();
30 |         registerCloseFSHook(fs);
31 |         try {
32 |             if (option.syncByBinlogFile) {
33 |                 client.readDataFromBinlogFile(option.binlogFilePath, fs, option);
34 |             } else {
35 |                 client.replicationStream(fs, option);
36 |             }
37 |         } catch (Exception e) {
38 |             e.printStackTrace();
39 |         }
40 | 
41 |     }
42 | 
43 | 
44 |     public static void registerCloseFSHook(FileSystem fs) {
45 |         log.info("注册FileSystem钩子");
46 |         //只有JVM退出时才关闭fs
47 |         Runtime.getRuntime().addShutdownHook(new Thread() {
48 |             @Override
49 |             public void run() {
50 |                 try {
51 |                     log.info("关闭FS");
52 |                     fs.close();
53 |                 } catch (IOException e) {
54 |                     e.printStackTrace();
55 |                 }
56 |             }
57 |         });
58 |     }
59 | }
60 | 


--------------------------------------------------------------------------------
/src/test/java/com/bixuange/OrcTest.java:
--------------------------------------------------------------------------------
 1 | //package com.bixuange;
 2 | //
 3 | //import org.apache.hadoop.conf.Configuration;
 4 | //import org.apache.hadoop.fs.Path;
 5 | //import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
 6 | //import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
 7 | //import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 8 | //import org.apache.orc.OrcFile;
 9 | //import org.apache.orc.Writer;
10 | //import org.junit.Test;
11 | //import org.apache.orc.TypeDescription;
12 | //
13 | //import java.io.IOException;
14 | //import java.nio.charset.Charset;
15 | //
16 | //
17 | ///**
18 | // * Created with IDEA
19 | // * Creater: MOBIN
20 | // * Date: 2018/5/25
21 | // * Time: 上午10:11
22 | // */
23 | //public class OrcTest {
24 | //    @Test
25 | //    public void orcTest() throws IOException {
26 | //        Configuration conf = new Configuration();
27 | //        Path path = new Path("/mobin1.orc");
28 | //        //typeName不能有空格
29 | //        TypeDescription schema = TypeDescription.fromString("struct<id:int,name:string>");
30 | //        Writer writer = OrcFile.createWriter(path, OrcFile.writerOptions(conf).setSchema(schema));
31 | //        VectorizedRowBatch batch = schema.createRowBatch();
32 | //        LongColumnVector id = (LongColumnVector) batch.cols[0];
33 | //        BytesColumnVector name = (BytesColumnVector) batch.cols[1];
34 | //        final int BATCH_SIZE = batch.getMaxSize();
35 | //        System.out.println(BATCH_SIZE);
36 | //        for (int i = 0; i < 10; i ++){
37 | //            int row = batch.size ++;
38 | //            id.vector[row] = i;
39 | //            byte[] buffer = ("name" + i).getBytes(Charset.forName("UTF-8"));
40 | //            name.setRef(row, buffer, 0, buffer.length);
41 | //            if (row == BATCH_SIZE - 1){
42 | //                writer.addRowBatch(batch);
43 | //                batch.reset();
44 | //            }
45 | //        }
46 | //        if (batch.size != 0){
47 | //            writer.addRowBatch(batch);
48 | //            batch.reset();
49 | //        }
50 | //        writer.close();
51 | //    }
52 | //}
53 | 


--------------------------------------------------------------------------------
/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
 1 | # Define some default values that can be overridden by system properties
 2 | Binlog.root.logger=INFO,console,DRFA
 3 | Binlog.security.logger=INFO,console
 4 | Binlog.log.dir=.
 5 | Binlog.log.file=Binlog.log
 6 | 
 7 | # Define the root logger to the system property Binlog.root.logger.
 8 | log4j.rootLogger=${Binlog.root.logger}
 9 | 
10 | # Logging Threshold
11 | log4j.threshold=ALL
12 | 
13 | #
14 | # Daily Rolling File Appender
15 | #
16 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
17 | log4j.appender.DRFA.Encoding=UTF-8
18 | log4j.appender.DRFA.File=${Binlog.log.dir}/${Binlog.log.file}
19 | 
20 | # Rollver at midnight
21 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
22 | 
23 | # 30-day backup
24 | #log4j.appender.DRFA.MaxBackupIndex=30
25 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
26 | 
27 | # Pattern format: Date LogLevel LoggerName LogMessage
28 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
29 | 
30 | # Rolling File Appender properties
31 | Binlog.log.maxfilesize=256MB
32 | Binlog.log.maxbackupindex=20
33 | 
34 | # Rolling File Appender
35 | log4j.appender.RFA=org.apache.log4j.RollingFileAppender
36 | log4j.appender.RFA.File=${Binlog.log.dir}/${Binlog.log.file}
37 | 
38 | log4j.appender.RFA.MaxFileSize=${Binlog.log.maxfilesize}
39 | log4j.appender.RFA.MaxBackupIndex=${Binlog.log.maxbackupindex}
40 | 
41 | log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
42 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
43 | 
44 | #
45 | # Security audit appender
46 | #
47 | Binlog.security.log.file=SecurityAuth.audit
48 | Binlog.security.log.maxfilesize=256MB
49 | Binlog.security.log.maxbackupindex=20
50 | log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
51 | log4j.appender.RFAS.File=${Binlog.log.dir}/${Binlog.security.log.file}
52 | log4j.appender.RFAS.MaxFileSize=${Binlog.security.log.maxfilesize}
53 | log4j.appender.RFAS.MaxBackupIndex=${Binlog.security.log.maxbackupindex}
54 | log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
55 | log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
56 | log4j.category.SecurityLogger=${Binlog.security.logger}
57 | log4j.additivity.SecurityLogger=false
58 | #log4j.logger.SecurityLogger.org.apache.hadoop.Binlog.security.access.AccessController=TRACE
59 | 
60 | #
61 | # Null Appender
62 | #
63 | log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
64 | 
65 | #
66 | # console
67 | # Add console to rootlogger above if you want to use this 
68 | #
69 | log4j.appender.console=org.apache.log4j.ConsoleAppender
70 | log4j.appender.console.target=System.err
71 | log4j.appender.console.layout=org.apache.log4j.PatternLayout
72 | log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
73 | 
74 | # Custom Logging levels
75 | 
76 | log4j.com.wellcommsoft.noce=INFO
77 | 
78 | 


--------------------------------------------------------------------------------
/src/test/java/com/bixuange/FileSystemTest.java:
--------------------------------------------------------------------------------
 1 | package com.bixuange;
 2 | 
 3 | import com.common.FSUtils;
 4 | import org.apache.hadoop.conf.Configuration;
 5 | import org.apache.hadoop.fs.*;
 6 | import org.apache.hadoop.io.IOUtils;
 7 | import sun.rmi.runtime.Log;
 8 | 
 9 | import java.io.ByteArrayInputStream;
10 | import java.io.IOException;
11 | import java.io.OutputStream;
12 | import java.util.concurrent.Semaphore;
13 | 
14 | public class FileSystemTest {
15 | 
16 |     public static void main(String[] args) throws Exception {
17 | //        System.setProperty("HADOOP_USER_NAME", "hdfs");
18 |         FileSystem fs = null;
19 |         try {
20 |             //1527386243433
21 |             fs = FSUtils.getNewFileSystem();
22 |             System.out.println(fs.exists(new Path("/DATA/PUBLIC/TABLE1/day=20180519/")));
23 |             Long lastModeifTime = Long.MIN_VALUE;
24 |             Path path = null;
25 |             for (FileStatus fileStatus: fs.listStatus(new Path("/DATA/PUBLIC/TABLE1/day=20180519/"))){
26 |                 Long time = fileStatus.getModificationTime();
27 |                 if (time > lastModeifTime){
28 |                     lastModeifTime = time;
29 |                     path = fileStatus.getPath();
30 |                 }
31 |             }
32 | 
33 |             System.out.println(path.toString());
34 |             System.out.println(fs.getFileStatus(new Path("/DATA/PUBLIC/TABLE1/day=20180519/000000_0")).getLen());
35 | 
36 | //            System.out.println(fs.getFileStatus(new Path("/mobin.txt")).getModificationTime());
37 | //            OutputStream out = FSUtils.openOutputStream(null, "/test/2.txt","","");
38 | //            out.write(("test。。。。。" + "\n").getBytes());
39 | //            out.flush();
40 |         } finally {
41 | //            fs.close();
42 |         }
43 |     }
44 | 
45 |     static class MyThread extends Thread {
46 |         OutputStream out;
47 |         Semaphore semaphore;
48 | 
49 |         public MyThread() {
50 |             super("MyThread");
51 |         }
52 | 
53 |         @Override
54 |         public void run() {
55 |             try {
56 |                 for (int i = 0; i < 100; i++) {
57 |                     try {
58 |                         synchronized (out) {
59 |                             out.write((Thread.currentThread().getName() + "aaaaaaaaaaaaaaaaa" + i + "\n").getBytes());
60 |                         }
61 |                     } catch (IOException e) {
62 |                         // TODO Auto-generated catch block
63 |                         e.printStackTrace();
64 |                     }
65 |                 }
66 |                 try {
67 |                     synchronized (out) {
68 |                         out.flush();
69 |                     }
70 |                 } catch (IOException e) {
71 |                     // TODO Auto-generated catch block
72 |                     e.printStackTrace();
73 |                 }
74 |             } finally {
75 |                 semaphore.release();
76 |             }
77 |         }
78 |     }
79 | }
80 | 


--------------------------------------------------------------------------------
/src/test/java/com/bixuange/BinlogClientTest.java:
--------------------------------------------------------------------------------
 1 | package com.bixuange;
 2 | 
 3 | import com.common.BinlogOption;
 4 | import com.common.FSUtils;
 5 | import com.github.shyiko.mysql.binlog.BinaryLogFileReader;
 6 | import com.github.shyiko.mysql.binlog.event.*;
 7 | import com.github.shyiko.mysql.binlog.event.deserialization.ChecksumType;
 8 | import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
 9 | import com.github.shyiko.mysql.binlog.event.deserialization.NullEventDataDeserializer;
10 | import org.apache.hadoop.fs.FileSystem;
11 | import org.junit.Test;
12 | 
13 | import java.io.File;
14 | import java.io.IOException;
15 | import java.sql.SQLException;
16 | 
17 | /**
18 |  * Created with IDEA
19 |  * Creater: MOBIN
20 |  * Date: 2018/5/17
21 |  * Time: 上午11:16
22 |  */
23 | public class BinlogClientTest {
24 |     @Test
25 |     public void binlogClinet() throws IOException {
26 |         BinlogClient client = new BinlogClient();
27 |         FileSystem fs = null;
28 |         try {
29 |             fs = FSUtils.getFileSystem();
30 |             BinlogOption option = new BinlogOption(null);
31 |             client.replicationStream(fs,option);
32 |         } catch (SQLException e) {
33 |             e.printStackTrace();
34 |         } finally {
35 |             fs.close();
36 |         }
37 | 
38 |     }
39 | 
40 | 
41 |     @Test
42 |     public void readBinlog() throws IOException {
43 |         File binlogFile = new File("/Users/mobin/Downloads/RDS-binlog/mysql-bin.000043");
44 |         EventDeserializer eventDeserializer = new EventDeserializer();
45 | //        eventDeserializer.setEventDataDeserializer(EventType.WRITE_ROWS, new ByteArrayEventDataDeserializer());
46 |         eventDeserializer.setEventDataDeserializer(EventType.UPDATE_ROWS, new NullEventDataDeserializer());
47 |         eventDeserializer.setChecksumType(ChecksumType.CRC32);
48 | 
49 |         BinaryLogFileReader reader = new BinaryLogFileReader(binlogFile,eventDeserializer);
50 |         try {
51 |             for (Event event; (event = reader.readEvent()) != null; ) {
52 |                 // && !EventHeaderV4.class.isInstance(event.getHeader())
53 |                 EventType eventType = event.getHeader().getEventType();//                TableMapEventData tableMapEventData = event.getData();
54 |                 if (eventType == EventType.TABLE_MAP) {  //判断是哪张表
55 |                     TableMapEventData tableMapEventData = event.getData();
56 |                     System.out.println(tableMapEventData.getTable());
57 |                 }
58 | //
59 | //                if (event.getHeader().getEventType() == EventType.WRITE_ROWS){
60 | //                    WriteRowsEventData writeRowsEventData = new WriteRowsEventData();
61 | //                    writeRowsEventData.setRows(event.getData());
62 | ////                    ByteArrayEventData byteArrayEventData = event.getData();
63 | ////                    String s = new String(byteArrayEventData.getData());
64 | ////                    System.out.println(s);
65 | //                }
66 |             }
67 |         } finally {
68 |             reader.close();
69 |         }
70 |     }
71 | }
72 | 


--------------------------------------------------------------------------------
/src/main/java/com/common/BinlogOption.java:
--------------------------------------------------------------------------------
 1 | package com.common;
 2 | 
 3 | import com.config.DatabaseConnection;
 4 | import org.apache.hadoop.fs.FileSystem;
 5 | import org.slf4j.Logger;
 6 | import org.slf4j.LoggerFactory;
 7 | import java.sql.Connection;
 8 | import java.sql.PreparedStatement;
 9 | import java.sql.ResultSet;
10 | import java.sql.SQLException;
11 | 
12 | /**
13 |  * Created with IDEA
14 |  * Creater: MOBIN
15 |  * Date: 2018/5/17
16 |  * Time: 下午2:46
17 |  */
18 | public class BinlogOption {
19 |     private static final Logger log = LoggerFactory.getLogger(BinlogOption.class);
20 |     public String dirPath = null;
21 |     public int count = 0;
22 |     public int countInterval = 50000;  //每50000写入一次
23 |     public static String database = "mobinDB";
24 |     public Long binlogPosition = null;  //自上次同步的游标开始同步
25 |     public String binlogFileName = null;
26 |     public FileSystem fs = null;
27 |     public boolean isSyncTable = true;  //是否同步该表
28 | 
29 |     public boolean syncByBinlogFile = false;  //通过指定binlog进行同步,默认从数据库获取上次同步的游标
30 |     public String binlogFilePath = null;
31 | 
32 |     public BinlogOption(String[] args){
33 |         for (int i = 0; i < args.length; i ++){
34 |             String arg = args[i];
35 |             switch (arg){
36 |                 case "-syncByBinlogFile":
37 |                     syncByBinlogFile = Boolean.parseBoolean(args[++ i]);
38 |                     break;
39 |                 case "-countInterval":
40 |                     countInterval = Integer.parseInt(args[++ i]);
41 |                     break;
42 |                 case "-binlogFilePath":
43 |                     binlogFilePath = args[++ i];
44 |                     break;
45 |                 default:
46 |                     log.error("参数无效：" + arg);
47 |                     System.exit(-1);
48 |             }
49 |         }
50 |         if (!syncByBinlogFile){
51 |             getNextPosition();
52 |         }else {
53 |             checkBinlogFilePath(binlogFilePath);
54 |         }
55 |     }
56 | 
57 |     private void checkBinlogFilePath(String binlogFilePath) {
58 |         if (binlogFilePath == null){
59 |             log.error("binlogFilePath路径无效：{}",binlogFilePath);
60 |             System.exit(-1);
61 |         }
62 |     }
63 | 
64 | 
65 |     public  void getNextPosition(){
66 |         Connection conn = null;
67 |         PreparedStatement ps = null;
68 |         ResultSet rs = null;
69 |         try {
70 |             conn = DatabaseConnection.getMysqlConnection();
71 |             String SQL = "SELECT nextposition,binlogfilename FROM t_position";
72 |             ps = conn.prepareStatement(SQL);
73 |             rs = ps.executeQuery();
74 |             while (rs.next()) {
75 |                 binlogPosition = rs.getLong("nextposition");
76 |                 binlogFileName = rs.getString("binlogfilename");
77 |             }
78 |             log.info("数据库最新nextposition={},binlogfilename={}",binlogPosition,binlogFileName);
79 |         } catch (SQLException e) {
80 |             e.printStackTrace();
81 |         }finally {
82 |             try {
83 |                 ps.close();
84 |                 conn.close();
85 |             } catch (SQLException e) {
86 |                 e.printStackTrace();
87 |             }
88 |         }
89 |     }
90 | }
91 | 


--------------------------------------------------------------------------------
/src/test/java/com/bixuange/AppTest.java:
--------------------------------------------------------------------------------
  1 | package com.bixuange;
  2 | 
  3 | import static org.junit.Assert.assertTrue;
  4 | 
  5 | import com.common.ExecCMD;
  6 | import com.common.FSUtils;
  7 | import com.config.DatabaseConnection;
  8 | import com.main.Binlog2Hive;
  9 | import org.apache.hadoop.fs.FileSystem;
 10 | import org.apache.hadoop.fs.Path;
 11 | import org.apache.hadoop.metrics.spi.OutputRecord;
 12 | import org.junit.Test;
 13 | 
 14 | import javax.swing.text.html.parser.Entity;
 15 | import java.io.IOException;
 16 | import java.io.OutputStream;
 17 | import java.sql.SQLException;
 18 | import java.util.*;
 19 | import java.util.concurrent.Semaphore;
 20 | 
 21 | /**
 22 |  * Unit test for simple App.
 23 |  */
 24 | public class AppTest {
 25 |     @Test
 26 |     public void shouldAnswerWithTrue() throws IOException, SQLException {
 27 | //        System.out.println(FSUtils.getFileSystem());
 28 |         System.out.println(DatabaseConnection.getMysqlConnection());
 29 |     }
 30 | 
 31 |     @Test
 32 |     public void append() throws IOException {
 33 |         OutputStream out = null;
 34 |         FileSystem fs = null;
 35 |         try {
 36 |             fs = FSUtils.getNewFileSystem();
 37 |             final Semaphore semaphore = new Semaphore(2);
 38 |             for (int i=0; i<=3 ; i ++){
 39 |                 out = FSUtils.openOutputStream(null,"","");
 40 |                 for (int j = 1000; j < 2000; j++) {
 41 |                     out.write(("aaaaaaaaaaaaaaaaa" + j + "\n").getBytes());
 42 |                 }
 43 |                 out.flush();
 44 |                 out.close();
 45 |                 out.close();
 46 |             }
 47 |         } catch (IOException e) {
 48 |             e.printStackTrace();
 49 |         } catch (Exception e) {
 50 |             e.printStackTrace();
 51 |         } finally {
 52 |             out.close();
 53 |             fs.close();
 54 |         }
 55 |     }
 56 | 
 57 |     @Test
 58 |     public void split(){
 59 |         String str = "/DATA/PUBLIC/TABLE1/day=20180514/000000_0";
 60 |         String str1 = "T_ALLAWSMIND_20180518";
 61 |         String tableName = str1.split("_",-1)[2];
 62 |         System.out.println(tableName);
 63 | 
 64 |         System.out.println(new java.util.Date());
 65 | 
 66 |         String fileName = "/DATA/PUBLIC/TABLE1/day=20180522/000000_0";
 67 |         System.out.println(fileName.split("_",-1)[0]);
 68 |         System.out.println(Integer.parseInt(fileName.split("_",-1)[2]));
 69 |     }
 70 | 
 71 |     @Test
 72 |     public void process() throws IOException {
 73 |         Process proc=Runtime.getRuntime().exec("/opt/hive-1.2.2/bin/hive -e 'create table yy(id int)'");
 74 | //        String SQL = "hive -e "  + "USE met_office;ALTER TABLE T_HNAWSMIND  add IF NOT EXISTS partition (day=20180519) location '/DATA/PUBLIC/METOFFICE/T_HNAWSMIND/day=20180519/'";
 75 | //        String SQL = "open .";
 76 | //        System.out.println(SQL);
 77 | //        ExecCMD.exec(SQL);
 78 |     }
 79 | 
 80 |     @Test
 81 |     public void getNextPosition() throws SQLException {
 82 | //        System.out.println(Binlog2Hive.getNextPosition(null));
 83 |         HashMap<String,Integer> map = new HashMap<>();
 84 |         map.put("2",1);
 85 |         map.put("1",4);
 86 |         map.put("3",2);
 87 |         Set<Map.Entry<String,Integer>> set = map.entrySet();
 88 |         List<Map.Entry<String,Integer>> list = new ArrayList<>(set);
 89 |         Collections.sort(list, new Comparator<Map.Entry<String,Integer>>() {
 90 |             @Override
 91 |             public int compare(Map.Entry<String,Integer> o1, Map.Entry<String,Integer> o2) {
 92 |                 return o1.getValue() - o2.getValue();
 93 |             }
 94 |         });
 95 | 
 96 |         LinkedHashMap<String,Integer> linkedHashMap = new LinkedHashMap<>();
 97 |         for (Map.Entry<String,Integer> entry : list){
 98 |             linkedHashMap.put(entry.getKey(),entry.getValue());
 99 |         }
100 | 
101 |         System.out.println(linkedHashMap);
102 |     }
103 | }
104 | 


--------------------------------------------------------------------------------
/src/main/java/com/common/FSUtils.java:
--------------------------------------------------------------------------------
  1 | package com.common;
  2 | 
  3 | import org.apache.hadoop.conf.Configuration;
  4 | import org.apache.hadoop.fs.FileStatus;
  5 | import org.apache.hadoop.fs.FileSystem;
  6 | import org.apache.hadoop.fs.Path;
  7 | import org.apache.hadoop.io.IOUtils;
  8 | import org.slf4j.Logger;
  9 | import org.slf4j.LoggerFactory;
 10 | 
 11 | import java.io.*;
 12 | 
 13 | public class FSUtils {
 14 |     private static final Logger log = LoggerFactory.getLogger(FSUtils.class);
 15 | 
 16 |     public static final int BLOCK_SIZE = 240;
 17 |     public static FileSystem getFileSystem() throws IOException {
 18 |         Configuration conf = new Configuration();
 19 |         conf.set("fs.hdfs.impl",org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
 20 |         return FileSystem.get(conf);
 21 |     }
 22 | 
 23 |     public static FileSystem getNewFileSystem() throws IOException {
 24 |         Configuration conf = new Configuration();
 25 |         conf.set("fs.hdfs.impl",org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
 26 |         return FileSystem.newInstance(conf);
 27 |     }
 28 | 
 29 | //    public static OutputStream createLzoOutputStream(FileSystem fs, Path lzoFile) throws IOException {
 30 | //        OutputStream os = fs.create(lzoFile);
 31 | //        return createLzoOutputStream(fs, os);
 32 | //    }
 33 | //
 34 | //    public static OutputStream createLzoOutputStream(FileSystem fs, OutputStream os) throws IOException {
 35 | //
 36 | ////        OutputStream os = fs.create(lzoFile);
 37 | //        return createLzoOutputStream(fs, os);
 38 | //    }
 39 | 
 40 |     public static OutputStream openOutputStream(FileSystem fs, String day, String tableNameDayPath) throws Exception {
 41 |         OutputStream os = null;
 42 |         Path path = getLastModifiedFile(fs,new Path(tableNameDayPath));  //获取最近写入的那个文件
 43 |         if (fs.exists(path) && fs.getFileStatus(path).getLen()/(1<<20) <= BLOCK_SIZE) {
 44 |             log.info("append内容到{}",path);
 45 |             try {
 46 |                 os = fs.append(path);
 47 |                 log.info(path+"文件append完成");
 48 |             } catch (Exception e) {
 49 |                 log.error("文件append失败：" + e);
 50 |                 //在不支持append的情况下，先将数据读出来，再写回去
 51 |                 byte[] oldBytes = FSUtils.readDataFile(fs, path);  //追加的文件内容
 52 |                 os = fs.create(path);  //被追加文件内容
 53 |                 os.write(oldBytes);
 54 |             }
 55 |         } else if (!fs.exists(path)) {
 56 |             log.info("创建{}文件", path); //DATA/PUBLIC/TABLE1/day=20180522/000000_0
 57 |             os = fs.create(path);
 58 |             log.info("开始对{}进行分区", tableNameDayPath);
 59 |             addHivePartition(path.toString(), day, tableNameDayPath);
 60 |         } else{
 61 |             int fileName_suffix = Integer.parseInt(path.getName().split("_",-1)[1]) + 1;
 62 |             String newFilePath = tableNameDayPath + "/" + "000000_" + fileName_suffix;
 63 |             log.info("创建{}文件",newFilePath);
 64 |             os = fs.create(new Path(newFilePath));
 65 |         }
 66 |         return os;
 67 |     }
 68 | 
 69 |     public static Path getLastModifiedFile(FileSystem fs,Path path) throws IOException {
 70 |         if (!fs.exists(path) || !fs.exists(new Path(path.toString() + "/000000_0"))){
 71 |             return new Path(path.toString() + "/000000_0");
 72 |         }
 73 |         Long lastModifiedTime = Long.MIN_VALUE;
 74 |         Path filePath =  null;
 75 |         for (FileStatus fileStatus: fs.listStatus(path)){
 76 |             Long lastTime = fileStatus.getModificationTime();
 77 |             if (lastTime > lastModifiedTime){
 78 |                 lastModifiedTime = lastTime;
 79 |                 filePath = fileStatus.getPath();
 80 |             }
 81 |         }
 82 |         return filePath;
 83 |     }
 84 | 
 85 | 
 86 |     public static void addHivePartition(String path, String day,String tableName_day) {
 87 |         StringBuffer sb = new StringBuffer();
 88 |         String tableName = path.split("/", -1)[4];
 89 |         String location = tableName_day +"/";
 90 |         sb.append("USE mobinDB;");
 91 |         sb.append("ALTER TABLE ")
 92 |                 .append(tableName)
 93 |                 .append("  add IF NOT EXISTS partition (");
 94 |         sb.append("day=").append(day).append(")");
 95 |         sb.append(" location '").append(location).append("'");
 96 |         String HiveSQL = String.format("hive -e \"%s\"", sb.toString());
 97 |         System.out.println(HiveSQL);
 98 |         log.info(HiveSQL);
 99 |         ExecCMD.exec(HiveSQL);
100 |     }
101 | 
102 |     public static byte[] readDataFile(FileSystem fs, Path dataFile) throws IOException {
103 |         ByteArrayOutputStream bos = new ByteArrayOutputStream(2 * 1024 * 1024);
104 |         InputStream in = null;
105 |         try {
106 |             in = fs.open(dataFile);
107 |             IOUtils.copyBytes(in, bos, 4096, false);
108 |         } finally {
109 |             IOUtils.closeStream(in);
110 |         }
111 |         return bos.toByteArray();
112 |     }
113 | 
114 | 
115 | }
116 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | 
  3 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4 |   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5 |   <modelVersion>4.0.0</modelVersion>
  6 | 
  7 |   <repositories>
  8 |     <repository>
  9 |       <id>cloudera</id>
 10 |       <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
 11 |     </repository>
 12 |   </repositories>
 13 | 
 14 |   <groupId>com.bixuange</groupId>
 15 |   <artifactId>Binlog2Hive</artifactId>
 16 |   <version>1.0-SNAPSHOT</version>
 17 | 
 18 |   <name>Binlog2Hive</name>
 19 |   <!-- FIXME change it to the project's website -->
 20 |   <url>http://www.example.com</url>
 21 | 
 22 |   <properties>
 23 |     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 24 |     <maven.compiler.source>1.7</maven.compiler.source>
 25 |     <maven.compiler.target>1.7</maven.compiler.target>
 26 |     <hadoop.version>2.6.0-cdh5.7.2</hadoop.version>
 27 |   </properties>
 28 | 
 29 |   <dependencies>
 30 |     <dependency>
 31 |       <groupId>junit</groupId>
 32 |       <artifactId>junit</artifactId>
 33 |       <version>4.11</version>
 34 |       <scope>test</scope>
 35 |     </dependency>
 36 | 
 37 |     <!--<dependency>-->
 38 |       <!--<groupId>org.apache.orc</groupId>-->
 39 |       <!--<artifactId>orc</artifactId>-->
 40 |       <!--<version>1.5.0</version>-->
 41 |     <!--</dependency>-->
 42 | 
 43 |     <!--<dependency>-->
 44 |       <!--<groupId>org.apache.orc</groupId>-->
 45 |       <!--<artifactId>orc-core</artifactId>-->
 46 |       <!--<version>1.5.0</version>-->
 47 |     <!--</dependency>-->
 48 | 
 49 |     <dependency>
 50 |       <groupId>mysql</groupId>
 51 |       <artifactId>mysql-connector-java</artifactId>
 52 |       <version>5.1.32</version>
 53 |     </dependency>
 54 | 
 55 |     <dependency>
 56 |       <groupId>com.alibaba</groupId>
 57 |       <artifactId>druid</artifactId>
 58 |       <version>1.1.5</version>
 59 |     </dependency>
 60 | 
 61 |     <dependency>
 62 |       <groupId>com.github.shyiko</groupId>
 63 |       <artifactId>mysql-binlog-connector-java</artifactId>
 64 |       <version>0.16.1</version>
 65 |     </dependency>
 66 | 
 67 |     <dependency>
 68 |       <groupId>org.apache.hadoop</groupId>
 69 |       <artifactId>hadoop-common</artifactId>
 70 |       <version>${hadoop.version}</version>
 71 |     </dependency>
 72 |     <dependency>
 73 |       <groupId>org.apache.hadoop</groupId>
 74 |       <artifactId>hadoop-hdfs</artifactId>
 75 |       <version>${hadoop.version}</version>
 76 |     </dependency>
 77 |     <dependency>
 78 |       <groupId>org.apache.hadoop</groupId>
 79 |       <artifactId>hadoop-client</artifactId>
 80 |       <version>${hadoop.version}</version>
 81 |     </dependency>
 82 |   </dependencies>
 83 | 
 84 |   <build>
 85 |     <plugins>
 86 |       <plugin>
 87 |         <groupId>org.springframework.boot</groupId>
 88 |         <artifactId>spring-boot-maven-plugin</artifactId>
 89 |       </plugin>
 90 |       <plugin>
 91 |         <groupId>org.apache.maven.plugins</groupId>
 92 |         <artifactId>maven-assembly-plugin</artifactId>
 93 |         <version>2.5.5</version>
 94 |         <configuration>
 95 |           <archive>
 96 |             <manifest>
 97 |               <mainClass>Binlog2Hive</mainClass>
 98 |             </manifest>
 99 |           </archive>
100 |           <descriptorRefs>
101 |             <descriptorRef>jar-with-dependencies</descriptorRef>
102 |           </descriptorRefs>
103 |         </configuration>
104 |       </plugin>
105 |       <plugin>
106 |         <artifactId>maven-compiler-plugin</artifactId>
107 |         <configuration>
108 |           <source>1.8</source>
109 |           <target>1.8</target>
110 |           <encoding>UTF-8</encoding>
111 |           <compilerArguments>
112 |             <extdirs>${project.basedir}/lib</extdirs>
113 |           </compilerArguments>
114 |         </configuration>
115 |       </plugin>
116 |     </plugins>
117 |     <pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
118 |       <plugins>
119 |         <plugin>
120 |           <artifactId>maven-clean-plugin</artifactId>
121 |           <version>3.0.0</version>
122 |         </plugin>
123 |         <!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
124 |         <plugin>
125 |           <artifactId>maven-resources-plugin</artifactId>
126 |           <version>3.0.2</version>
127 |         </plugin>
128 |         <plugin>
129 |           <artifactId>maven-compiler-plugin</artifactId>
130 |           <version>3.7.0</version>
131 |         </plugin>
132 |         <plugin>
133 |           <artifactId>maven-surefire-plugin</artifactId>
134 |           <version>2.20.1</version>
135 |         </plugin>
136 |         <plugin>
137 |           <artifactId>maven-jar-plugin</artifactId>
138 |           <version>3.0.2</version>
139 |         </plugin>
140 |         <plugin>
141 |           <artifactId>maven-install-plugin</artifactId>
142 |           <version>2.5.2</version>
143 |         </plugin>
144 |         <plugin>
145 |           <artifactId>maven-deploy-plugin</artifactId>
146 |           <version>2.8.2</version>
147 |         </plugin>
148 |       </plugins>
149 |     </pluginManagement>
150 |   </build>
151 | 
152 | 
153 | </project>
154 | 


--------------------------------------------------------------------------------
/src/main/java/com/bixuange/BinlogClient.java:
--------------------------------------------------------------------------------
  1 | package com.bixuange;
  2 | 
  3 | import com.common.BinlogOption;
  4 | import com.common.FSUtils;
  5 | import com.config.Config;
  6 | import com.config.DatabaseConnection;
  7 | import com.github.shyiko.mysql.binlog.BinaryLogClient;
  8 | import com.github.shyiko.mysql.binlog.BinaryLogFileReader;
  9 | import com.github.shyiko.mysql.binlog.event.*;
 10 | import com.github.shyiko.mysql.binlog.event.deserialization.ChecksumType;
 11 | import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
 12 | import com.github.shyiko.mysql.binlog.event.deserialization.NullEventDataDeserializer;
 13 | import org.apache.hadoop.fs.FileSystem;
 14 | import org.slf4j.Logger;
 15 | import org.slf4j.LoggerFactory;
 16 | 
 17 | import java.io.*;
 18 | import java.nio.charset.Charset;
 19 | import java.sql.Connection;
 20 | import java.sql.PreparedStatement;
 21 | import java.sql.SQLException;
 22 | import java.text.SimpleDateFormat;
 23 | import java.util.*;
 24 | import java.util.concurrent.ConcurrentHashMap;
 25 | 
 26 | /**
 27 |  * Created with IDEA
 28 |  * Creater: MOBIN
 29 |  * Date: 2018/5/17
 30 |  * Time: 上午10:08
 31 |  */
 32 | public class BinlogClient {
 33 |     private static final Logger log = LoggerFactory.getLogger(BinlogClient.class);
 34 |     private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 35 |     private SimpleDateFormat sfDay = new SimpleDateFormat("yyyyMMdd");
 36 | 
 37 |     public ConcurrentHashMap<String, List<Serializable[]>> tableRows = new ConcurrentHashMap();  //key是表前缀路径
 38 |     private static final String HOSTNAME = "localhost";
 39 |     private static final int PORT = 3306;
 40 |     private static final String USERNAME = "root";
 41 |     private static final String PASSWD = "root";
 42 | 
 43 |     public void replicationStream(FileSystem fs, final BinlogOption option) throws IOException, SQLException {
 44 |         BinaryLogClient client = new BinaryLogClient(HOSTNAME, PORT, USERNAME, PASSWD);
 45 |         if (!option.syncByBinlogFile){
 46 |             client.setBinlogFilename(option.binlogFileName);
 47 |             client.setBinlogPosition(option.binlogPosition);
 48 |         }
 49 |         log.info("即将解析{}文件，起始游标为：{}", option.binlogFileName, option.binlogPosition);
 50 |         client.registerEventListener(new BinaryLogClient.EventListener() {
 51 |             @Override
 52 |             public void onEvent(Event event) {
 53 |                 analysisEvent(event, option, fs, true);
 54 |             }
 55 |         });
 56 |         client.connect();
 57 |     }
 58 | 
 59 | 
 60 |     public void readDataFromBinlogFile(String filePath, FileSystem fs, final BinlogOption option) throws IOException {
 61 |         log.info("正在解析{}文件",filePath);
 62 |         File binlogFile = new File(filePath);
 63 |         EventDeserializer eventDeserializer = new EventDeserializer();
 64 |         eventDeserializer.setEventDataDeserializer(EventType.UPDATE_ROWS, new NullEventDataDeserializer());
 65 |         eventDeserializer.setEventDataDeserializer(EventType.DELETE_ROWS, new NullEventDataDeserializer());
 66 |         eventDeserializer.setChecksumType(ChecksumType.CRC32);
 67 |         BinaryLogFileReader reader = new BinaryLogFileReader(binlogFile, eventDeserializer);
 68 |         try {
 69 |             for (Event event; (event = reader.readEvent()) != null; ) {
 70 |                 analysisEvent(event, option, fs, false);
 71 |             }
 72 |         } finally {
 73 |             reader.close();
 74 |         }
 75 |     }
 76 | 
 77 |     private void analysisEvent(Event event, BinlogOption option, FileSystem fs, boolean isUpdatePosition) {
 78 |         EventType eventType = event.getHeader().getEventType();
 79 |         if (eventType == EventType.ROTATE) {
 80 |             RotateEventData rotateEventData = event.getData();
 81 |             option.binlogFileName = rotateEventData.getBinlogFilename();
 82 |         }
 83 |         if (eventType == EventType.TABLE_MAP) {  //判断是哪张表
 84 |             TableMapEventData tableMapEventData = event.getData();
 85 |             String tableName = tableMapEventData.getTable().toLowerCase();
 86 |             String database = tableMapEventData.getDatabase().toLowerCase();
 87 |             String dirPath = Config.getPath(tableName);
 88 |             option.isSyncTable = dirPath == null ? false : true;
 89 |             option.database = database;
 90 |             option.dirPath = dirPath;
 91 |         }
 92 |         if (option.isSyncTable && "mobinDB".equals(option.database) && eventType == EventType.WRITE_ROWS ) {
 93 |             List<Serializable[]> writeRowsEventDataList = tableRows.get(option.dirPath);
 94 |             if (writeRowsEventDataList == null) {
 95 |                 writeRowsEventDataList = new ArrayList<>();
 96 |                 tableRows.put(option.dirPath, writeRowsEventDataList);
 97 |             }
 98 |             WriteRowsEventData writeRowsEventData = event.getData();
 99 |             option.count = option.count + writeRowsEventData.getRows().size();
100 |             writeRowsEventDataList.addAll(writeRowsEventData.getRows());
101 |         }
102 |         if (option.count > option.countInterval) {
103 |             ConcurrentHashMap<String, ArrayList<Serializable[]>> map = new ConcurrentHashMap<>();//key是表名+分区目录名(day)，value是一条数据
104 |             try {
105 |                 Iterator<String> iterators = tableRows.keySet().iterator();
106 |                 //不同表且分区字段不一样打开一次文件流
107 |                 while (iterators.hasNext()) {  //遍历次数少,根据表前缀路径来遍历
108 |                     traverseTable(map, iterators, option);   //将数据按天进行分区</DATA/PUBLIC/XXX/day=20180520,row>
109 |                 }
110 |                 log.info("数据整理完毕，准备写入HDFS");
111 |                 Iterator<String> its = map.keySet().iterator();
112 |                 while (its.hasNext()) {   //遍历次数少,根据表的分区字段day遍历
113 |                     traverseDay(event, map, its, fs, option, isUpdatePosition);
114 |                 }
115 |             } catch (IOException e) {
116 |                 e.printStackTrace();
117 |             } catch (Exception e) {
118 |                 e.printStackTrace();
119 |             }
120 |             log.info("本次同步{}条数据", option.count);
121 |             option.count = 0;
122 |             tableRows.clear();
123 |         }
124 |     }
125 | 
126 | 
127 |     private void traverseDay(Event event, ConcurrentHashMap<String, ArrayList<Serializable[]>> map,
128 |                              Iterator<String> its, FileSystem fs,
129 |                              BinlogOption option, boolean isUpdatePosition) throws Exception {
130 |         OutputStream out;
131 |         String tableName_day = its.next();
132 |         String day = tableName_day.split("=", -1)[1];
133 |         out = FSUtils.openOutputStream(fs, day, tableName_day);
134 |         for (Serializable[] row : map.get(tableName_day)) {
135 |             StringBuffer sb = new StringBuffer();
136 |             for (int i = 0; i < row.length; i++) {
137 |                 if (i == row.length - 1) {
138 |                     sb.append(row[i]);
139 |                 } else if (i == 1) {
140 |                     synchronized (sdf) {
141 |                         sdf.setTimeZone(TimeZone.getTimeZone("GTM"));
142 |                         sb.append(sdf.format(row[1]) + "|");
143 |                     }
144 |                 } else {
145 |                     sb.append(row[i] + "|");
146 |                 }
147 |             }
148 |             sb.append("\n");
149 |             out.write(sb.toString().getBytes(Charset.forName("UTF-8")));
150 |         }
151 |         out.flush();
152 |         out.close();
153 |         if (isUpdatePosition) {
154 |             EventHeaderV4 eventHeaderV4 = event.getHeader();
155 |             option.binlogPosition = eventHeaderV4.getNextPosition();
156 |             updateBinlogNextPosition(option);
157 |         }
158 | 
159 |     }
160 | 
161 |     private void traverseTable(ConcurrentHashMap<String, ArrayList<Serializable[]>> map, Iterator<String> iterators, BinlogOption option) {
162 |         String table_path_prefix = iterators.next();
163 |         String tableName = table_path_prefix.split("/", -1)[4];
164 |         List<Serializable[]> rows = tableRows.get(table_path_prefix);
165 |         for (Serializable[] row : rows) {
166 |             String datetime;
167 |             synchronized (sfDay) {
168 |                 sfDay.setTimeZone(TimeZone.getTimeZone("GTM"));
169 |                 datetime = sfDay.format(row[1]);
170 |             }
171 |             String key = table_path_prefix + "/day=" + datetime;
172 |             ArrayList<Serializable[]> rowList = map.get(key);
173 |             if (rowList == null) {
174 |                 rowList = new ArrayList<>();
175 |                 map.put(key, rowList);
176 |             }
177 |             rowList.add(row);
178 |         }
179 |     }
180 | 
181 |     public static void updateBinlogNextPosition(BinlogOption option) {
182 |         Connection conn = null;
183 |         PreparedStatement ps = null;
184 |         try {
185 |             conn = DatabaseConnection.getMysqlConnection();
186 |             String SQL = "UPDATE t_position SET nextposition=?,binlogfilename=?";
187 |             ps = conn.prepareStatement(SQL);
188 |             ps.setLong(1, option.binlogPosition);
189 |             ps.setString(2, option.binlogFileName);
190 |             ps.executeUpdate();
191 |             ps.close();
192 |             log.info("更新游标为：{}，binlogFileName文件为{}", option.binlogPosition, option.binlogFileName);
193 |         } catch (SQLException e) {
194 |             e.printStackTrace();
195 |         } finally {
196 |             try {
197 |                 if (conn != null) {
198 |                     conn.close();
199 |                 }
200 |             } catch (SQLException e) {
201 |                 e.printStackTrace();
202 |             }
203 |         }
204 |     }
205 | }
206 | 


--------------------------------------------------------------------------------
/Binlog2Hive.iml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
  3 |   <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
  4 |     <output url="file://$MODULE_DIR$/target/classes" />
  5 |     <output-test url="file://$MODULE_DIR$/target/test-classes" />
  6 |     <content url="file://$MODULE_DIR$">
  7 |       <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
  8 |       <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
  9 |       <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
 10 |       <excludeFolder url="file://$MODULE_DIR$/target" />
 11 |     </content>
 12 |     <orderEntry type="inheritedJdk" />
 13 |     <orderEntry type="sourceFolder" forTests="false" />
 14 |     <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.11" level="project" />
 15 |     <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
 16 |     <orderEntry type="library" name="Maven: mysql:mysql-connector-java:5.1.32" level="project" />
 17 |     <orderEntry type="library" name="Maven: com.alibaba:druid:1.1.5" level="project" />
 18 |     <orderEntry type="library" name="Maven: com.github.shyiko:mysql-binlog-connector-java:0.16.1" level="project" />
 19 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-common:2.6.0-cdh5.7.2" level="project" />
 20 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-annotations:2.6.0-cdh5.7.2" level="project" />
 21 |     <orderEntry type="library" name="Maven: com.google.guava:guava:11.0.2" level="project" />
 22 |     <orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
 23 |     <orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.1.1" level="project" />
 24 |     <orderEntry type="library" name="Maven: xmlenc:xmlenc:0.52" level="project" />
 25 |     <orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.1" level="project" />
 26 |     <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.4" level="project" />
 27 |     <orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
 28 |     <orderEntry type="library" name="Maven: commons-net:commons-net:3.1" level="project" />
 29 |     <orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" />
 30 |     <orderEntry type="library" name="Maven: javax.servlet:servlet-api:2.5" level="project" />
 31 |     <orderEntry type="library" name="Maven: org.mortbay.jetty:jetty:6.1.26.cloudera.4" level="project" />
 32 |     <orderEntry type="library" name="Maven: org.mortbay.jetty:jetty-util:6.1.26.cloudera.4" level="project" />
 33 |     <orderEntry type="library" name="Maven: com.sun.jersey:jersey-core:1.9" level="project" />
 34 |     <orderEntry type="library" name="Maven: com.sun.jersey:jersey-json:1.9" level="project" />
 35 |     <orderEntry type="library" name="Maven: org.codehaus.jettison:jettison:1.1" level="project" />
 36 |     <orderEntry type="library" name="Maven: com.sun.xml.bind:jaxb-impl:2.2.3-1" level="project" />
 37 |     <orderEntry type="library" name="Maven: javax.xml.bind:jaxb-api:2.2.2" level="project" />
 38 |     <orderEntry type="library" name="Maven: javax.xml.stream:stax-api:1.0-2" level="project" />
 39 |     <orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
 40 |     <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.8.3" level="project" />
 41 |     <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-xc:1.8.3" level="project" />
 42 |     <orderEntry type="library" name="Maven: com.sun.jersey:jersey-server:1.9" level="project" />
 43 |     <orderEntry type="library" name="Maven: asm:asm:3.1" level="project" />
 44 |     <orderEntry type="library" scope="RUNTIME" name="Maven: tomcat:jasper-compiler:5.5.23" level="project" />
 45 |     <orderEntry type="library" name="Maven: tomcat:jasper-runtime:5.5.23" level="project" />
 46 |     <orderEntry type="library" name="Maven: javax.servlet.jsp:jsp-api:2.1" level="project" />
 47 |     <orderEntry type="library" name="Maven: commons-el:commons-el:1.0" level="project" />
 48 |     <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.3" level="project" />
 49 |     <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
 50 |     <orderEntry type="library" name="Maven: net.java.dev.jets3t:jets3t:0.9.0" level="project" />
 51 |     <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.1.2" level="project" />
 52 |     <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.1.2" level="project" />
 53 |     <orderEntry type="library" name="Maven: com.jamesmurty.utils:java-xmlbuilder:0.4" level="project" />
 54 |     <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
 55 |     <orderEntry type="library" name="Maven: commons-configuration:commons-configuration:1.6" level="project" />
 56 |     <orderEntry type="library" name="Maven: commons-digester:commons-digester:1.8" level="project" />
 57 |     <orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.7.0" level="project" />
 58 |     <orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" level="project" />
 59 |     <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.5" level="project" />
 60 |     <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.5" level="project" />
 61 |     <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.8.8" level="project" />
 62 |     <orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.8.8" level="project" />
 63 |     <orderEntry type="library" name="Maven: org.apache.avro:avro:1.7.6-cdh5.7.2" level="project" />
 64 |     <orderEntry type="library" name="Maven: com.thoughtworks.paranamer:paranamer:2.3" level="project" />
 65 |     <orderEntry type="library" name="Maven: org.xerial.snappy:snappy-java:1.0.4.1" level="project" />
 66 |     <orderEntry type="library" name="Maven: com.google.protobuf:protobuf-java:2.5.0" level="project" />
 67 |     <orderEntry type="library" name="Maven: com.google.code.gson:gson:2.2.4" level="project" />
 68 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-auth:2.6.0-cdh5.7.2" level="project" />
 69 |     <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" />
 70 |     <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" />
 71 |     <orderEntry type="library" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" />
 72 |     <orderEntry type="library" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" />
 73 |     <orderEntry type="library" name="Maven: org.apache.curator:curator-framework:2.7.1" level="project" />
 74 |     <orderEntry type="library" name="Maven: com.jcraft:jsch:0.1.42" level="project" />
 75 |     <orderEntry type="library" name="Maven: org.apache.curator:curator-client:2.7.1" level="project" />
 76 |     <orderEntry type="library" name="Maven: org.apache.curator:curator-recipes:2.7.1" level="project" />
 77 |     <orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:3.0.0" level="project" />
 78 |     <orderEntry type="library" name="Maven: org.apache.htrace:htrace-core4:4.0.1-incubating" level="project" />
 79 |     <orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.5-cdh5.7.2" level="project" />
 80 |     <orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" />
 81 |     <orderEntry type="library" name="Maven: org.tukaani:xz:1.0" level="project" />
 82 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-hdfs:2.6.0-cdh5.7.2" level="project" />
 83 |     <orderEntry type="library" name="Maven: commons-daemon:commons-daemon:1.0.13" level="project" />
 84 |     <orderEntry type="library" name="Maven: io.netty:netty:3.6.2.Final" level="project" />
 85 |     <orderEntry type="library" name="Maven: io.netty:netty-all:4.0.23.Final" level="project" />
 86 |     <orderEntry type="library" name="Maven: xerces:xercesImpl:2.9.1" level="project" />
 87 |     <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.3.04" level="project" />
 88 |     <orderEntry type="library" name="Maven: org.fusesource.leveldbjni:leveldbjni-all:1.8" level="project" />
 89 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-client:2.6.0-cdh5.7.2" level="project" />
 90 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-app:2.6.0-cdh5.7.2" level="project" />
 91 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-common:2.6.0-cdh5.7.2" level="project" />
 92 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-client:2.6.0-cdh5.7.2" level="project" />
 93 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-server-common:2.6.0-cdh5.7.2" level="project" />
 94 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.6.0-cdh5.7.2" level="project" />
 95 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-api:2.6.0-cdh5.7.2" level="project" />
 96 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-core:2.6.0-cdh5.7.2" level="project" />
 97 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-yarn-common:2.6.0-cdh5.7.2" level="project" />
 98 |     <orderEntry type="library" name="Maven: com.sun.jersey:jersey-client:1.9" level="project" />
 99 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.6.0-cdh5.7.2" level="project" />
100 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-aws:2.6.0-cdh5.7.2" level="project" />
101 |     <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.2.3" level="project" />
102 |     <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.2.3" level="project" />
103 |     <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.2.3" level="project" />
104 |     <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-s3:1.10.6" level="project" />
105 |     <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-kms:1.10.6" level="project" />
106 |     <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk-core:1.10.6" level="project" />
107 |   </component>
108 | </module>


--------------------------------------------------------------------------------