├── README.md
├── pom.xml
├── process.png
├── script
    ├── snappy_file.py
    └── synch_file.py
└── src
    ├── main
        ├── java
        │   ├── META-INF
        │   │   └── MANIFEST.MF
        │   └── com
        │   │   └── yiguan
        │   │       └── kafka
        │   │           ├── App202Consumer.java
        │   │           ├── Consumer.java
        │   │           ├── ProducerFromFile.java
        │   │           ├── Start.java
        │   │           ├── service
        │   │               └── TimeService.java
        │   │           └── util
        │   │               ├── Constants.java
        │   │               ├── DistinctAST.java
        │   │               ├── OZBase64.java
        │   │               └── ProducerConstants.java
        ├── main.iml
        └── resources
        │   ├── beijingidc-resources
        │       ├── conf.properties
        │       └── log4j.properties
        │   └── ucloud-resources
        │       ├── conf.properties
        │       └── log4j.properties
    └── test
        └── test.iml


/README.md:
--------------------------------------------------------------------------------
 1 | # Backquarter (Kafka Message Synchronization)
 2 | 
 3 | ![](process.png)
 4 | 
 5 | 
 6 | 同步Ucloud上的Kafka消息到IDC机房：消费者消费Ucloud上Kafka集群的消息，生成文件，实时同步文件至IDC机房。生产者实时监控文件，解析文件发送至IDC的Kafka集群。
 7 | 
 8 | ------
 9 | 
10 | ## 说明
11 |  - src/java: 程序源码
12 |  - src/resources: 配置文件
13 |  - Consumer.java: 消费者
14 |  - ProducerFromFile.java: 生产者
15 | 
16 | ## 脚本
17 |  - script/snappy_file.py: 文件压缩/解压脚本
18 |  - script/synch_file.py: 文件传输脚本
19 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
  3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5 |     <modelVersion>4.0.0</modelVersion>
  6 |     <groupId>com.analysys</groupId>
  7 |     <artifactId>data_sync</artifactId>
  8 |     <version>0.1-SNAPSHOT</version>
  9 | 
 10 |     <properties>
 11 |         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 12 |     </properties>
 13 | 
 14 |     <dependencies>
 15 |         <dependency>
 16 |             <groupId>junit</groupId>
 17 |             <artifactId>junit</artifactId>
 18 |             <version>4.12</version>
 19 |             <scope>test</scope>
 20 |         </dependency>
 21 |         <!--<dependency>
 22 |             <groupId>log4j</groupId>
 23 |             <artifactId>log4j</artifactId>
 24 |             <version>1.2.17</version>
 25 |         </dependency>-->
 26 |         <dependency>
 27 |             <groupId>commons-beanutils</groupId>
 28 |             <artifactId>commons-beanutils-core</artifactId>
 29 |             <version>1.8.3</version>
 30 |         </dependency>
 31 |         <dependency>
 32 |             <groupId>commons-beanutils</groupId>
 33 |             <artifactId>commons-beanutils</artifactId>
 34 |             <version>1.8.3</version>
 35 |         </dependency>
 36 |         <dependency>
 37 |             <groupId>commons-collections</groupId>
 38 |             <artifactId>commons-collections</artifactId>
 39 |             <version>3.1</version>
 40 |         </dependency>
 41 |         <dependency>
 42 |             <groupId>commons-lang</groupId>
 43 |             <artifactId>commons-lang</artifactId>
 44 |             <version>2.6</version>
 45 |         </dependency>
 46 |         <dependency>
 47 |             <groupId>commons-logging</groupId>
 48 |             <artifactId>commons-logging</artifactId>
 49 |             <version>1.1.3</version>
 50 |         </dependency>
 51 |         <dependency>
 52 |             <groupId>net.sf.ezmorph</groupId>
 53 |             <artifactId>ezmorph</artifactId>
 54 |             <version>1.0.6</version>
 55 |         </dependency>
 56 |         <dependency>
 57 |             <groupId>net.sf.json-lib</groupId>
 58 |             <artifactId>json-lib</artifactId>
 59 |             <version>2.4</version>
 60 |             <classifier>jdk15</classifier>
 61 |         </dependency>
 62 |         <dependency>
 63 |             <groupId>mysql</groupId>
 64 |             <artifactId>mysql-connector-java</artifactId>
 65 |             <version>5.1.34</version>
 66 |         </dependency>
 67 |         <dependency>
 68 |             <groupId>javax.mail</groupId>
 69 |             <artifactId>mail</artifactId>
 70 |             <version>1.4.7</version>
 71 |         </dependency>
 72 |         <dependency>
 73 |             <groupId>com-analysys-common</groupId>
 74 |             <artifactId>com-analysys-common</artifactId>
 75 |             <version>0.0.2</version>
 76 |         </dependency>
 77 |         <dependency>
 78 |             <groupId>com.alibaba</groupId>
 79 |             <artifactId>fastjson</artifactId>
 80 |             <version>[1.2.68,)</version>
 81 |         </dependency>
 82 |         <dependency>
 83 |             <groupId>org.apache.kafka</groupId>
 84 |             <artifactId>kafka-clients</artifactId>
 85 |             <version>0.8.2.2</version>
 86 |         </dependency>
 87 |         <dependency>
 88 |             <groupId>com.yammer.metrics</groupId>
 89 |             <artifactId>metrics-core</artifactId>
 90 |             <version>2.2.0</version>
 91 |         </dependency>
 92 |         <dependency>
 93 |             <groupId>org.apache.zookeeper</groupId>
 94 |             <artifactId>zookeeper</artifactId>
 95 |             <version>[3.4.14,)</version>
 96 |         </dependency>
 97 |         <dependency>
 98 |             <groupId>org.apache.kafka</groupId>
 99 |             <artifactId>kafka_2.10</artifactId>
100 |             <version>0.8.2.2</version>
101 |         </dependency>
102 |         <dependency>
103 |             <groupId>org.scala-lang</groupId>
104 |             <artifactId>scala-library</artifactId>
105 |             <version>2.10.4</version>
106 |         </dependency>
107 |         <dependency>
108 |             <groupId>com.kafka</groupId>
109 |             <artifactId>slf4j-api</artifactId>
110 |             <version>1.7.6</version>
111 |         </dependency>
112 |         <dependency>
113 |             <groupId>com.analysys.http.util</groupId>
114 |             <artifactId>http.util</artifactId>
115 |             <version>0.0.1-SNAPSHOT</version>
116 |         </dependency>
117 |     </dependencies>
118 | 
119 |     <build>
120 |         <resources>
121 |             <resource>
122 |                 <directory>src/main/java</directory>
123 |             </resource>
124 |             <resource>
125 |                 <directory>src/test/java</directory>
126 |             </resource>
127 |             <resource>
128 |                 <directory>${file-directory}</directory>
129 |                 <includes>
130 |                     <include>**/*.properties</include>
131 |                     <include>**/*.xml</include>
132 |                 </includes>
133 |             </resource>
134 |         </resources>
135 |         <plugins>
136 |             <plugin>
137 |                 <groupId>org.apache.maven.plugins</groupId>
138 |                 <artifactId>maven-shade-plugin</artifactId>
139 |                 <version>1.2.1</version>
140 |                 <executions>
141 |                     <execution>
142 |                         <phase>package</phase>
143 |                         <goals>
144 |                             <goal>shade</goal>
145 |                         </goals>
146 |                         <configuration>
147 |                             <transformers>
148 |                                 <transformer
149 |                                         implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
150 |                                     <mainClass>com.yiguan.kafka.Start</mainClass>
151 |                                 </transformer>
152 |                             </transformers>
153 |                         </configuration>
154 |                     </execution>
155 |                 </executions>
156 |             </plugin>
157 |             <plugin>
158 |                 <artifactId>maven-compiler-plugin</artifactId>
159 |                 <configuration>
160 |                     <source>1.7</source>
161 |                     <target>1.7</target>
162 |                 </configuration>
163 |             </plugin>
164 |         </plugins>
165 |     </build>
166 |     <profiles>
167 |         <profile>
168 |             <id>ucloud</id>
169 |             <properties>
170 |                 <file-directory>src/main/resources/ucloud-resources</file-directory>
171 |             </properties>
172 |         </profile>
173 |         <profile>
174 |             <id>beijingidc</id>
175 |             <properties>
176 |                 <file-directory>src/main/resources/beijingidc-resources</file-directory>
177 |             </properties>
178 |         </profile>
179 |     </profiles>
180 | </project>


--------------------------------------------------------------------------------
/process.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/analysys/Backquarter/c5f425610083ed055db4f049b8ac9dd714b13095/process.png


--------------------------------------------------------------------------------
/script/snappy_file.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import os
  3 | import datetime as dt
  4 | import time
  5 | 
  6 | print dt.datetime.now()
  7 | currentTimestamp = dt.datetime.strftime(dt.datetime.now(),"%m%d%H%M%S")
  8 | snappyPostfix = "tmp_nappy"
  9 | checkSnappyPostfix = "tmp_check_nappy"
 10 | 
 11 | if(len(sys.argv)!=4):
 12 |     print "python snappy_file.py [sourceFilePath]  [targetFilePath]  [c|d]"
 13 |     print "c compress \t  d decompress"
 14 |     sys.exit()
 15 | 
 16 | def intFileSizeThreshold(fileMap): 
 17 |     fileMap["h5jssdk"]=1024*200
 18 |     fileMap["app_2_0_HDFS"]=1024*1024*10
 19 |     fileMap["default"]=1024*1024*500
 20 | 
 21 | def getFileSizeThreshold(fileMap,str1):
 22 |     count1= str1.find('.')
 23 |     queue1=str1[0:count1]
 24 |     threshold=0
 25 |     day=""
 26 |     if(str1.find(".")>0):
 27 |         day=queue1[queue1.rfind("_")+1:len(queue1)]
 28 |         isday=day.isdigit()
 29 |         if (isday == True and len(day) == 8):
 30 |             queue1=queue1[:queue1.rfind("_")]
 31 |     if(fileMap.has_key("%s"%(queue1))==False):    
 32 |         threshold=fileMap["default"]
 33 |     else:
 34 |         threshold=fileMap[queue1]
 35 |     return threshold
 36 | 
 37 | def getFileLineCntFromName(fileName):
 38 |     return fileName.split(".")[2]
 39 | 
 40 | def getFileLineCnt(tmpAbspath):
 41 |     return len(open(tmpAbspath).readlines())
 42 | 
 43 | def getTmpSnappyFileName(filepath, fileName):
 44 |     lineCnt = getFileLineCnt("%s/%s"%(filepath, fileName))
 45 |     currentTime = "%s%s"%(dt.datetime.strftime(dt.datetime.today(), "%m%d"), dt.datetime.strftime(dt.datetime.now(), "%H%M%S"))
 46 |     return "%s.%s.%s.%s"%(fileName, lineCnt, currentTime,snappyPostfix)
 47 | 
 48 | def cleanTmpFile(sourceFilePath):
 49 |     files = os.listdir(sourceFilePath) 
 50 |     for tmpfile in files:
 51 |         if(tmpfile.find(checkSnappyPostfix)>0):
 52 |             os.remove("%s/%s"%(sourceFilePath, tmpfile))
 53 |         if(tmpfile.find(snappyPostfix)>0):
 54 |             os.remove("%s/%s"%(sourceFilePath, tmpfile))
 55 | 
 56 | def isCompressOk(sourceFilePath, tmpSnappyFileName):
 57 |     tmpCheckFile = "tmp."+checkSnappyPostfix
 58 |     os.system("python -m snappy -d %s/%s  %s/%s"%(sourceFilePath, tmpSnappyFileName, sourceFilePath, tmpCheckFile))
 59 |     lineCnt = getFileLineCnt("%s/%s"%(sourceFilePath, tmpCheckFile))
 60 |     fileLineCnt = getFileLineCntFromName(tmpSnappyFileName)
 61 |     return int(lineCnt) == int(fileLineCnt)
 62 |     
 63 | def doCompressFile(sourceFilePath, fileName, targetFilePath):
 64 |     tmpSnappyFileName = getTmpSnappyFileName(sourceFilePath, fileName)
 65 |     os.system("python -m snappy -c %s/%s %s/%s"%(sourceFilePath, fileName, sourceFilePath, tmpSnappyFileName))
 66 |     isOk = isCompressOk(sourceFilePath, tmpSnappyFileName)
 67 |     if(isOk):
 68 |         targetFileName=tmpSnappyFileName[:tmpSnappyFileName.rfind(".")]
 69 |         topicandday=tmpSnappyFileName[0:tmpSnappyFileName.find('.')]
 70 |         day=topicandday[topicandday.rfind("_")+1:len(topicandday)]
 71 |         isday=day.isdigit()
 72 |         if (isday == True and len(day) == 8):
 73 |             snames=tmpSnappyFileName.split(".")
 74 |             targetFileName="%s.%s.%s.%s.%s"%(topicandday[:topicandday.rfind("_")],snames[1],day,snames[2],snames[3])
 75 |         fileSize = os.path.getsize("%s/%s"%(sourceFilePath, tmpSnappyFileName))
 76 |         targetFileName = "%s.%s.%s"%(targetFileName,fileSize,"snappy")
 77 |         print "-----------%s"%(targetFileName)
 78 |         os.system("mv %s/%s  %s/%s"%(sourceFilePath,tmpSnappyFileName, targetFilePath, targetFileName))
 79 |         os.remove("%s/%s"%(sourceFilePath, fileName))
 80 |     cleanTmpFile(sourceFilePath)
 81 |     return isOk
 82 | 
 83 | def doDecompressFile(sourceFilePath, fileName):
 84 |     tmpCheckFile = "tmp."+checkSnappyPostfix
 85 |     os.system("python -m snappy -d %s/%s %s/%s"%(sourceFilePath, fileName, sourceFilePath, tmpCheckFile))
 86 |     lineCnt = getFileLineCnt("%s/%s"%(sourceFilePath, tmpCheckFile))
 87 |     fileLineCnt = getFileLineCntFromName(fileName)
 88 |     isOk = int(lineCnt) == int(fileLineCnt)
 89 |     if(isOk):
 90 |         os.system("mv %s/%s  %s/%s"%(sourceFilePath, tmpCheckFile, sourceFilePath, fileName[:fileName.rfind(".")]) )
 91 |         os.remove("%s/%s"%(sourceFilePath, fileName))
 92 |     cleanTmpFile(sourceFilePath)  
 93 |     return isOk
 94 | 
 95 | def log(content):
 96 |     v_logfile="/tmp/snappy_file.log"
 97 |     if(sys.argv[3] == "c"):
 98 |         v_logfile="/data/logs/snappy_file.log"
 99 |     currentTime = "%s%s"%(dt.datetime.strftime(dt.datetime.today(), "%Y%m%d"), dt.datetime.strftime(dt.datetime.now(), "%H%M%S"))
100 |     if(True):
101 |         fileHandle = open ( v_logfile, 'a+' ) 
102 |         fileHandle.write("%s    %s\n"%(currentTime, content))
103 |         fileHandle.close() 
104 | 
105 | def compressFile(sourceFilePath,fileSizeThresholdMap,targetFilePath):
106 |     while(1==1):
107 |         try:
108 |             time.sleep(1)
109 |             files = os.listdir(sourceFilePath) 
110 |             for tmpfile in files:
111 |                 fileSize = os.path.getsize("%s/%s"%(sourceFilePath, tmpfile))
112 |                 threshold=getFileSizeThreshold(fileSizeThresholdMap,tmpfile)
113 |                 if(tmpfile.find(".")>0 and tmpfile.find(".snappy")==-1 and fileSize>= threshold):
114 |                     currentTime = dt.datetime.now()
115 |                     for i in range(5):
116 |                         log("%s begin compress"%(tmpfile))
117 |                         isOk = doCompressFile(sourceFilePath, tmpfile, targetFilePath)
118 |                         if(isOk):
119 |                             break;
120 |                     log("%s is compressed [isOk:%s] usetime:%s"%(tmpfile, isOk, dt.datetime.now() - currentTime))
121 |         except Exception, e:
122 |             print "error", log(str(e))
123 |             pass;
124 | 
125 | def discompressFile(sourceFilePath, targetFilePath):
126 |     while(1==1):
127 |         try:
128 |             time.sleep(1)
129 |             files = os.listdir(sourceFilePath) 
130 |             for tmpfile in files:
131 |                 if( tmpfile.find(".")>0  and os.path.exists("%s/%s.done"%(sourceFilePath, str(tmpfile))) ):
132 |                     currentTime = dt.datetime.now()
133 |                     for i in range(5):
134 |                         log("%s begin decompress"%(tmpfile))
135 |                         isOk = doDecompressFile(sourceFilePath, tmpfile)
136 |                         if(isOk):
137 |                             os.system("rm %s/%s.done"%(sourceFilePath, tmpfile))
138 |                             os.system("mv %s/%s  %s"%(sourceFilePath,tmpfile[:tmpfile.rfind(".")], targetFilePath) )
139 |                             break;
140 |                     log("%s is compressed [isOk:%s] usetime:%s"%(tmpfile, isOk, dt.datetime.now() - currentTime))
141 |         except Exception, e:
142 |             print "error", (e)
143 |             pass;
144 | 
145 | def main():
146 |     sourceFilePath = sys.argv[1]
147 |     targetFilePath = sys.argv[2]
148 |     processType = sys.argv[3]
149 |     fileSizeThresholdMap={}
150 |     intFileSizeThreshold(fileSizeThresholdMap)    
151 |     if(processType == "c"):
152 |         print sourceFilePath
153 |         compressFile(sourceFilePath,fileSizeThresholdMap,targetFilePath)
154 |     if(processType == "d"):
155 |         discompressFile(sourceFilePath, targetFilePath)
156 | main()
157 | 


--------------------------------------------------------------------------------
/script/synch_file.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | import os
 3 | import datetime as dt
 4 | import time
 5 | 
 6 | print dt.datetime.now()
 7 | currentTimestamp = dt.datetime.strftime(dt.datetime.now(),"%m%d%H%M%S")
 8 | 
 9 | fortest = False
10 | def getModifyFileName(fname, timestamp):
11 |     return "%s.%s%s"%(fname[:fname.find(".")],fname[fname.find(".")+1:],timestamp)
12 | 
13 | def log(content):
14 |     v_logfile="/data/logsynch.log"
15 |     currentTime = "%s%s"%(dt.datetime.strftime(dt.datetime.today(), "%Y%m%d"), dt.datetime.strftime(dt.datetime.now(), "%H%M%S"))
16 |     if(True):
17 |         print content
18 |         fileHandle = open ( v_logfile, 'a+' ) 
19 |         fileHandle.write("%s    %s\n"%(currentTime, content))
20 |         fileHandle.close() 
21 | 
22 | def getFileLineCnt(filepath):
23 |     cnt = len(open(filepath).readlines())
24 |     return cnt
25 | 
26 | def main():
27 |     filepath = "/data/log"
28 |     bakFilepath = "/data/baklog"
29 |     targetFilepath = "root@192.168.0.15:/data/tmplog"
30 |     scpPort ="  "
31 |     while(1==1):
32 |     #if(1==1):
33 |         try:
34 |             time.sleep(1)
35 |             files = os.listdir(filepath) 
36 |             for tmpfile in files:
37 |                 if( tmpfile.find(".snappy")>0 ):
38 |                     currentDate = dt.datetime.strftime(dt.datetime.now(),"%Y%m%d")
39 |                     currentTime = dt.datetime.now()
40 |                     modifyFileName = tmpfile
41 |                     modifyFileNameDone = modifyFileName + ".done"
42 |                     fileSize = os.path.getsize("%s/%s"%(filepath, tmpfile))
43 |                     #scpCmd = ("scp %s/%s  %s"%(filepath, modifyFileName, targetFilepath))
44 |                     scpCmd = ("sudo scp %s %s/%s  %s"%(scpPort, filepath, modifyFileName, targetFilepath))
45 |                     mkdirCmd = "mkdir -p %s/%s"%(bakFilepath, currentDate)
46 |                     mvFileCmd = ("mv %s/%s  %s/%s"%(filepath, modifyFileName, bakFilepath, currentDate))
47 |                     echoModifyFileNameDone = ("echo ''>%s/%s"%(filepath, modifyFileNameDone))
48 |                     scpModifyFileNameDone = "sudo scp %s %s/%s  %s"%(scpPort, filepath, modifyFileNameDone, targetFilepath)
49 |                     rmModifyFileNameDone = "rm -f %s/%s"%(filepath, modifyFileNameDone)
50 |                     if(fortest==False and len(modifyFileName)>0):
51 |                         log(scpCmd)
52 |                         os.system(scpCmd)
53 |                         log(echoModifyFileNameDone)
54 |                         os.system(echoModifyFileNameDone)
55 |                         log(scpModifyFileNameDone)
56 |                         os.system(scpModifyFileNameDone)
57 |                         log(rmModifyFileNameDone)
58 |                         os.system(rmModifyFileNameDone)
59 |                         log(mkdirCmd)
60 |                         os.system(mkdirCmd)
61 |                         log(mvFileCmd)
62 |                         os.system(mvFileCmd)
63 |                         syncFileInfo("%s/%s"%(bakFilepath, currentDate), modifyFileName)
64 |                     log( "%s\t%sm\t%ss"%(modifyFileName, (float(fileSize))/(1024*1024), dt.datetime.now() - currentTime))
65 |         except Exception, e:
66 |             print str(e)
67 |             pass;
68 | main()
69 | 


--------------------------------------------------------------------------------
/src/main/java/META-INF/MANIFEST.MF:
--------------------------------------------------------------------------------
1 | Manifest-Version: 1.0
2 | Main-Class: com.yiguan.kafka.Start
3 | 
4 | 


--------------------------------------------------------------------------------
/src/main/java/com/yiguan/kafka/App202Consumer.java:
--------------------------------------------------------------------------------
  1 | package com.yiguan.kafka;
  2 | 
  3 | import java.io.File;
  4 | import java.util.HashMap;
  5 | import java.util.List;
  6 | import java.util.Map;
  7 | import java.util.Properties;
  8 | import java.util.concurrent.atomic.AtomicInteger;
  9 | 
 10 | import kafka.consumer.ConsumerConfig;
 11 | import kafka.consumer.ConsumerIterator;
 12 | import kafka.consumer.KafkaStream;
 13 | import kafka.javaapi.consumer.ConsumerConnector;
 14 | 
 15 | import org.apache.log4j.Logger;
 16 | 
 17 | import com.yiguan.kafka.service.TimeService;
 18 | import com.yiguan.kafka.util.Constants;
 19 | import com.yiguan.kafka.util.DistinctAST;
 20 | import com.yiguan.kafka.util.OZBase64;
 21 | 
 22 | /**
 23 |  * 这个线程只负责读取卡夫卡中的数据
 24 |  * @author Europe
 25 |  */
 26 | public class App202Consumer implements Runnable {
 27 | 	private final ConsumerConnector consumer;
 28 | 	private final String topic;
 29 | 	private String group;
 30 | 	private Logger logger;
 31 | 	private Logger consumerLogger = Logger.getLogger("ConsumeLogs");
 32 | 	private Map<String, AtomicInteger> map = new HashMap<String, AtomicInteger>();
 33 | 
 34 | 	public App202Consumer(String topic, String group, Logger logger) throws Exception {
 35 | 		this.topic = topic + "_" + TimeService.getInstance().getCurrentTimeYYYYMMDD();
 36 | 		this.logger = logger;
 37 | 		this.group = group;
 38 | 		consumer = kafka.consumer.Consumer.createJavaConsumerConnector(createConsumerConfig());
 39 | 		map.put("sdk_2_0_1_hdfs_8089_kafka", Start.SDK201);
 40 | 		map.put("app_2_0_2", Start.APP202);
 41 | 		map.put("app_3_0", Start.APP300);
 42 | 		map.put("app_all_CASSANDRA", Start.ALLAPP);
 43 | 		map.put("sdk_HDFS_168", Start.SDK168);
 44 | 		map.put("app_2_0_HDFS", Start.APP200);
 45 | 		map.put(this.topic, new AtomicInteger(0));
 46 | 	}
 47 | 
 48 | 	/*
 49 | 	 * 从kafka读取用
 50 | 	 */
 51 | 	private ConsumerConfig createConsumerConfig() {
 52 | 		Properties props = new Properties();
 53 | 		props.put("zookeeper.connect", Constants.ZK_CONNECT);
 54 | 		props.put("group.id", group);
 55 | 		props.put("zookeeper.session.timeout.ms", "40000");
 56 | 		props.put("zookeeper.sync.time.ms", "200");
 57 | 		props.put("auto.commit.interval.ms", "1000");
 58 | 		props.put("fetch.message.max.bytes", "104857400");
 59 | 		props.put("auto.offset.reset", "smallest");
 60 | 		return new ConsumerConfig(props);
 61 | 	}
 62 | 
 63 | 	public void run() {
 64 | 		readKafak();
 65 | 	}
 66 | 
 67 | 	public void readKafak() {
 68 | 		Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
 69 | 		topicCountMap.put(topic, new Integer(1));
 70 | 		Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);
 71 | 		KafkaStream<byte[], byte[]> stream = consumerMap.get(topic).get(0);
 72 | 		ConsumerIterator<byte[], byte[]> it = stream.iterator();
 73 | 		// 获取卡夫卡中的数据
 74 | 		int count = 0;
 75 | 		while (true) {
 76 | 			String[] files = getFileLists();
 77 | 			if (files.length > 50) {
 78 | 				System.out.println(topic + " i will sleep times:" + (1000 * 60 * 5L));
 79 | 				try {
 80 | 					Thread.sleep(1000 * 60 * 5L);
 81 | 					continue;
 82 | 				} catch (Exception e) {
 83 | 					e.printStackTrace();
 84 | 				}
 85 | 			}
 86 | 			while (it.hasNext()) {
 87 | 				byte[] dataByte = it.next().message();
 88 | 				try {
 89 | 					String distinctJSON = DistinctAST.distinctAll(new String(dataByte));
 90 | 					logger.info(OZBase64.encode(distinctJSON.getBytes()));
 91 | 					count++;
 92 | 					if (count % 50000 == 0) {
 93 | 						if (getFileLists().length > 50) {
 94 | 							count = 0;
 95 | 							break;
 96 | 						}
 97 | 					}
 98 | 				} catch (Exception e) {
 99 | 					e.printStackTrace();
100 | 				}
101 | 				logCnt(topic);
102 | 			}
103 | 		}
104 | 	}
105 | 
106 | 	private String[] getFileLists() {
107 | 		File file = new File(Constants.WRITE_DATA_FILE_DIR);
108 | 		String[] files = file.list();
109 | 		return files;
110 | 	}
111 | 
112 | 	private void logCnt(String topic) {
113 | 		TimeService service = TimeService.getInstance();
114 | 		long currentTime = System.currentTimeMillis();
115 | 		AtomicInteger counter = map.get(topic);
116 | 		synchronized (map) {
117 | 			if (!(currentTime >= service.getCurrentTimeHourLowerLimit() && currentTime < service.getCurrentTimeHourUpperLimit())) {
118 | 				synchronized (map) {
119 | 					if (!(currentTime >= service.getCurrentTimeHourLowerLimit() && currentTime < service.getCurrentTimeHourUpperLimit())) {
120 | 						consumerLogger.info(String.format("H\t%s\t%s\t%s", topic, counter.get(), service.getCurrentTimeYYYYMMDDHHMMSS()));
121 | 						counter.set(0);
122 | 						service.resetHourLimits();
123 | 					}
124 | 				}
125 | 			}
126 | 		}
127 | 		synchronized (map) {
128 | 			if (!(currentTime >= service.getCurrentTimeDayLowerLimit() && currentTime < service.getCurrentTimeDayUpperLimit())) {
129 | 				synchronized (map) {
130 | 					if (!(currentTime >= service.getCurrentTimeDayLowerLimit() && currentTime < service.getCurrentTimeDayUpperLimit())) {
131 | 						consumerLogger.info(String.format("D\t%s\t%s\t%s", topic, counter.get(), service.getCurrentTimeYYYYMMDDHHMMSS()));
132 | 						counter.set(0);
133 | 						service.resetDayLimits();
134 | 					}
135 | 				}
136 | 			}
137 | 		}
138 | 		counter.addAndGet(1);
139 | 		if (counter.get() % 1000 == 0) {
140 | 			System.out.println(String.format("%s\t%s\t%s", topic, counter.get(), service.getCurrentTimeHourUpperLimit()));
141 | 		}
142 | 	}
143 | }
144 | 


--------------------------------------------------------------------------------
/src/main/java/com/yiguan/kafka/Consumer.java:
--------------------------------------------------------------------------------
  1 | package com.yiguan.kafka;
  2 | 
  3 | import java.io.File;
  4 | import java.util.HashMap;
  5 | import java.util.List;
  6 | import java.util.Map;
  7 | import java.util.Properties;
  8 | import java.util.concurrent.atomic.AtomicInteger;
  9 | 
 10 | import kafka.consumer.ConsumerConfig;
 11 | import kafka.consumer.ConsumerIterator;
 12 | import kafka.consumer.KafkaStream;
 13 | import kafka.javaapi.consumer.ConsumerConnector;
 14 | 
 15 | import org.apache.log4j.Logger;
 16 | 
 17 | import com.yiguan.kafka.service.TimeService;
 18 | import com.yiguan.kafka.util.Constants;
 19 | import com.yiguan.kafka.util.OZBase64;
 20 | 
 21 | /**
 22 |  * 这个线程只负责读取卡夫卡中的数据
 23 |  * @author Europe
 24 |  */
 25 | public class Consumer implements Runnable {
 26 | 	private final ConsumerConnector consumer;
 27 | 	private final String topic;
 28 | 	private String group;
 29 | 	private Logger logger;
 30 | 	private Logger consumerLogger = Logger.getLogger("ConsumeLogs");
 31 | 	private Map<String, AtomicInteger> map = new HashMap<String, AtomicInteger>();
 32 | 
 33 | 	public Consumer(String topic, String group, Logger logger) throws Exception {
 34 | 		this.topic = topic + "_" + TimeService.getInstance().getCurrentTimeYYYYMMDD();
 35 | 		this.logger = logger;
 36 | 		this.group = group;
 37 | 		consumer = kafka.consumer.Consumer.createJavaConsumerConnector(createConsumerConfig());
 38 | 		map.put("sdk_2_0_1_hdfs_8089_kafka", Start.SDK201);
 39 | 		map.put("app_2_0_2", Start.APP202);
 40 | 		map.put("app_3_0", Start.APP300);
 41 | 		map.put("app_all_CASSANDRA", Start.ALLAPP);
 42 | 		map.put("sdk_HDFS_168", Start.SDK168);
 43 | 		map.put("app_2_0_HDFS", Start.APP200);
 44 | 		map.put("h5jssdk", Start.H5SDK);
 45 | 		map.put("sdk_2_0_1_hdfs_8089_kafka_2", Start.SDK2012);
 46 | 		map.put(this.topic, new AtomicInteger(0));
 47 | 	}
 48 | 
 49 | 	/*
 50 | 	 * 从kafka读取用
 51 | 	 */
 52 | 	private ConsumerConfig createConsumerConfig() {
 53 | 		Properties props = new Properties();
 54 | 		props.put("zookeeper.connect", Constants.ZK_CONNECT);
 55 | 		props.put("group.id", group);
 56 | 		props.put("zookeeper.session.timeout.ms", "40000");
 57 | 		props.put("zookeeper.sync.time.ms", "200");
 58 | 		props.put("auto.commit.interval.ms", "1000");
 59 | 		props.put("fetch.message.max.bytes", "104857400");
 60 | 		props.put("auto.offset.reset", "smallest");
 61 | 		return new ConsumerConfig(props);
 62 | 	}
 63 | 
 64 | 	public void run() {
 65 | 		readKafak();
 66 | 	}
 67 | 
 68 | 	public void readKafak() {
 69 | 		Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
 70 | 		topicCountMap.put(topic, new Integer(1));
 71 | 		Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);
 72 | 		KafkaStream<byte[], byte[]> stream = consumerMap.get(topic).get(0);
 73 | 		ConsumerIterator<byte[], byte[]> it = stream.iterator();
 74 | 		// 获取卡夫卡中的数据
 75 | 		int count = 0;
 76 | 		while (true) {
 77 | 			String[] files = getFileLists();
 78 | 			if (files.length > 50) {
 79 | 				System.out.println(topic + " i will sleep times:" + (1000 * 60 * 5L));
 80 | 				try {
 81 | 					Thread.sleep(1000 * 60 * 5L);
 82 | 					continue;
 83 | 				} catch (Exception e) {
 84 | 					e.printStackTrace();
 85 | 				}
 86 | 			}
 87 | 			while (it.hasNext()) {
 88 | 				byte[] dataByte = it.next().message();
 89 | 				try {
 90 | 					logger.info(OZBase64.encode(dataByte));
 91 | 					count++;
 92 | 					if (count % 50000 == 0) {
 93 | 						if (getFileLists().length > 50) {
 94 | 							count = 0;
 95 | 							break;
 96 | 						}
 97 | 					}
 98 | 				} catch (Exception e) {
 99 | 					e.printStackTrace();
100 | 				}
101 | 				logCnt(topic);
102 | 			}
103 | 		}
104 | 	}
105 | 
106 | 	private String[] getFileLists() {
107 | 		File file = new File(Constants.WRITE_DATA_FILE_DIR);
108 | 		String[] files = file.list();
109 | 		return files;
110 | 	}
111 | 
112 | 	private void logCnt(String topic) {
113 | 		TimeService service = TimeService.getInstance();
114 | 		long currentTime = System.currentTimeMillis();
115 | 		AtomicInteger counter = map.get(topic);
116 | 		synchronized (map) {
117 | 			if (!(currentTime >= service.getCurrentTimeHourLowerLimit() && currentTime < service.getCurrentTimeHourUpperLimit())) {
118 | 				synchronized (map) {
119 | 					if (!(currentTime >= service.getCurrentTimeHourLowerLimit() && currentTime < service.getCurrentTimeHourUpperLimit())) {
120 | 						consumerLogger.info(String.format("H\t%s\t%s\t%s", topic, counter.get(), service.getCurrentTimeYYYYMMDDHHMMSS()));
121 | 						counter.set(0);
122 | 						service.resetHourLimits();
123 | 					}
124 | 				}
125 | 			}
126 | 		}
127 | 		synchronized (map) {
128 | 			if (!(currentTime >= service.getCurrentTimeDayLowerLimit() && currentTime < service.getCurrentTimeDayUpperLimit())) {
129 | 				synchronized (map) {
130 | 					if (!(currentTime >= service.getCurrentTimeDayLowerLimit() && currentTime < service.getCurrentTimeDayUpperLimit())) {
131 | 						consumerLogger.info(String.format("D\t%s\t%s\t%s", topic, counter.get(), service.getCurrentTimeYYYYMMDDHHMMSS()));
132 | 						counter.set(0);
133 | 						service.resetDayLimits();
134 | 					}
135 | 				}
136 | 			}
137 | 		}
138 | 		counter.addAndGet(1);
139 | 		if (counter.get() % 1000 == 0) {
140 | 			System.out.println(String.format("%s\t%s\t%s", topic, counter.get(), service.getCurrentTimeHourUpperLimit()));
141 | 		}
142 | 	}
143 | }
144 | 


--------------------------------------------------------------------------------
/src/main/java/com/yiguan/kafka/ProducerFromFile.java:
--------------------------------------------------------------------------------
  1 | package com.yiguan.kafka;
  2 | 
  3 | import java.io.BufferedReader;
  4 | import java.io.File;
  5 | import java.io.FileInputStream;
  6 | import java.io.InputStreamReader;
  7 | import java.io.UnsupportedEncodingException;
  8 | import java.text.ParseException;
  9 | import java.text.SimpleDateFormat;
 10 | import java.util.ArrayList;
 11 | import java.util.Arrays;
 12 | import java.util.Comparator;
 13 | import java.util.Date;
 14 | import java.util.HashMap;
 15 | import java.util.List;
 16 | import java.util.Map;
 17 | import java.util.Properties;
 18 | import java.util.Random;
 19 | import java.util.concurrent.atomic.AtomicInteger;
 20 | import java.util.regex.Pattern;
 21 | 
 22 | import kafka.producer.KeyedMessage;
 23 | import kafka.producer.ProducerConfig;
 24 | 
 25 | import org.apache.log4j.Logger;
 26 | 
 27 | import com.yiguan.kafka.service.TimeService;
 28 | import com.yiguan.kafka.util.Constants;
 29 | import com.yiguan.kafka.util.OZBase64;
 30 | import com.yiguan.kafka.util.ProducerConstants;
 31 | 
 32 | /**
 33 |  * 读取文件,向IDC的Kafka写入数据
 34 |  * @author Administrator
 35 |  */
 36 | public class ProducerFromFile implements Runnable {
 37 | 	public static ProducerConfig config;
 38 | 	private Logger logger = Logger.getLogger("ProducerLogs");
 39 | 	private Pattern pattern = Pattern.compile("[0-9]*");
 40 | 	private SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
 41 | 	private String workSpace;
 42 | 	private String index = "1";
 43 | 	private Integer cache_num = 1000;
 44 | 
 45 | 	static {
 46 | 		Properties pro = new Properties();
 47 | 		pro.put("metadata.broker.list", ProducerConstants.KAFKA_BROKER);
 48 | 		pro.put("zookeeper.connect", ProducerConstants.ZK_CONNECT);
 49 | 		pro.put("serializer.class", ProducerConstants.serializerClass);
 50 | 		pro.put("partitioner.class", ProducerConstants.partitionerClass);
 51 | 		pro.put("request.required.acks", ProducerConstants.acks);
 52 | 		pro.put("compression.codec", ProducerConstants.compressionCodec);
 53 | 		pro.put("compressed.topics", ProducerConstants.compressedTopics);
 54 | 		pro.put("request.timeout.ms", ProducerConstants.requestTimeoutMS);
 55 | 		pro.put("queue.buffering.max.ms", ProducerConstants.queueBufferingMaxMS);
 56 | 		pro.put("queue.buffering.max.messages", ProducerConstants.queueBufferingMaxMessages);
 57 | 		pro.put("batch.num.messages", ProducerConstants.batchNumMessages);
 58 | 		pro.put("queue.enqueue.timeout.ms", ProducerConstants.queueEnqueueTimeoutMS);
 59 | 		config = new ProducerConfig(pro);
 60 | 	}
 61 | 
 62 | 	@SuppressWarnings("rawtypes")
 63 | 	public kafka.javaapi.producer.Producer producer = new kafka.javaapi.producer.Producer(config);
 64 | 	public KeyedMessage<byte[], byte[]> data;
 65 | 	public int MSCOUNT = 0;
 66 | 	private String topicName;
 67 | 	Map<String, AtomicInteger> map = new HashMap<String, AtomicInteger>();
 68 | 
 69 | 	public ProducerFromFile(String topicName, String index) {
 70 | 		this.topicName = topicName;
 71 | 		this.index = index;
 72 | 		this.workSpace = Constants.DEAL_DATA_FILE_DIR + "/" + topicName + "/" + this.index;
 73 | 		File folder = new File(this.workSpace);
 74 | 		if(!folder.exists())
 75 | 			folder.mkdirs();
 76 | 		map.put(topicName, new AtomicInteger(0));
 77 | 	}
 78 | 
 79 | 	@SuppressWarnings("rawtypes")
 80 | 	private KeyedMessage getMsg(String topic, String key, byte[] bytes) {
 81 | 		return new KeyedMessage<byte[], byte[]>(topic, key.getBytes(), bytes);
 82 | 	}
 83 | 
 84 | 	public void close() {
 85 | 		producer.close();
 86 | 	}
 87 | 
 88 | 	/**
 89 | 	 * 获取待处理文件
 90 | 	 * @param topic
 91 | 	 * @return
 92 | 	 */
 93 | 	private String getCanReadFile(String topic) {
 94 | 		String targetFilePath = "";
 95 | 		//先轮询自己的工作目录
 96 | 		if(map.get(topicName).get() == 0) {
 97 | 			File myFolder = new File(this.workSpace);
 98 | 			File[] myFiles = myFolder.listFiles();
 99 | 			if(myFiles.length > 0){
100 | 				for(File oldFile : myFiles){
101 | 					moveFile(oldFile.getAbsolutePath(), Constants.READ_DATA_FILE_DIR);
102 | 				}
103 | 			}
104 | 		}
105 | 		//再去待处理目录抓取文件(生成时间排序)
106 | 		File folder = new File(Constants.READ_DATA_FILE_DIR);
107 | 		File[] files = folder.listFiles();
108 | 		if(files != null && files.length > 1){
109 | 			try {
110 | 				Arrays.sort(files, new Comparator<File>() {
111 | 					@Override
112 | 					public int compare(File f1, File f2) {
113 | 						long diff = f1.lastModified() - f2.lastModified();
114 | 						if (diff > 0) {
115 | 							return 1;
116 | 						} else if (diff == 0) {
117 | 							return 0;
118 | 						} else {
119 | 							return -1;
120 | 						}
121 | 					}
122 | 				});
123 | 			} catch (Exception e) {}
124 | 		}
125 | 		for(File file : files) {
126 | 			if(file.isDirectory())
127 | 				continue;
128 | 			String topicName = topic + ".";
129 | 			if(file.getName().startsWith(topicName)){
130 | 				boolean success = moveFile(file.getAbsolutePath(), this.workSpace);
131 | 				if(success){
132 | 					targetFilePath = this.workSpace + "/" + file.getName();
133 | 					break; 
134 | 				}
135 | 			}
136 | 		}
137 | 		return targetFilePath;
138 | 	}
139 | 	
140 | 	@SuppressWarnings({ "unchecked", "rawtypes" })
141 | 	private void send(List<KeyedMessage> ls, List<String> tmpTopicName, int lineCount, File file){
142 | 		int errNum = 0;
143 | 		for (int i = 0; i < 5; i++) {
144 | 			try {
145 | 				producer.send(ls);
146 | 				System.out.println(String.format("read %s|%s\t%s|%s\t%s\t%s", tmpTopicName, (topicName), lineCount, map.get(topicName).get(), file, TimeService.getInstance().getCurrentTimeYYYYMMDDHHMMSS()));
147 | 				return;
148 | 			} catch (Exception e) {
149 | 				errNum++;
150 | 				int msg_length = 0;
151 | 				try {
152 | 					for(KeyedMessage m : ls)
153 | 						msg_length += ((byte[])m.message()).length;
154 | 				} catch (Exception e2) {}
155 | 				System.out.println(String.format("readerr %s\t%s|%s\t%s|%s", tmpTopicName, lineCount, msg_length, file, TimeService.getInstance().getCurrentTimeYYYYMMDDHHMMSS()));
156 | 			}
157 | 			if (errNum == 3) {
158 | 				try {
159 | 					producer.close();
160 | 					producer = new kafka.javaapi.producer.Producer(config);
161 | 					producer.send(ls);
162 | 					return;
163 | 				} catch (Exception e) {
164 | 					System.out.println("###" + e.getMessage());
165 | 				}
166 | 			}
167 | 		}
168 | 	}
169 | 	
170 | 	@SuppressWarnings("rawtypes")
171 | 	@Override
172 | 	public void run() {
173 | 		try {
174 | 			String configFileName = "";
175 | 			byte[] message = null;
176 | 			String lineTxt = null;
177 | 			List<KeyedMessage> ls = null;
178 | 			int lineCount = 0;
179 | 			InputStreamReader read = null;
180 | 			BufferedReader bufferedReader = null;
181 | 			while (true) {
182 | 				configFileName = getCanReadFile(topicName);
183 | 				if(configFileName != null && !"".equals(configFileName)){
184 | 					File _file = new File(configFileName);
185 | 					if (!(_file.isFile() && _file.exists())) { // 判断文件是否存在
186 | 						Thread.sleep(60000L);
187 | 						continue;
188 | 					}
189 | 				} else {
190 | 					System.out.println(TimeService.getInstance().getCurrentTimeYYYYMMDDHHMMSS() + "   " + topicName + "=== NoFile to deal, wait 1m...");
191 | 					Thread.sleep(60000L);
192 | 					continue;
193 | 				}
194 | 				File file = new File(configFileName);
195 | 				if (file.isFile() && file.exists()) {
196 | 					try {
197 | 						System.out.println(topicName + "===Read datafile:" + (configFileName) + " " + (file.isFile() && file.exists()));
198 | 						long startTime = System.currentTimeMillis();
199 | 						read = new InputStreamReader(new FileInputStream(file));// 考虑到编码格式
200 | 						bufferedReader = new BufferedReader(read);
201 | 						lineCount = 0;
202 | 						ls = new ArrayList<KeyedMessage>(cache_num);
203 | 						List<String> tmpTopicName = getTopicNameWithTimestamp(topicName, file.getName());
204 | 						while ((lineTxt = bufferedReader.readLine()) != null) {
205 | 							lineCount++;
206 | 							map.get(topicName).addAndGet(1);
207 | 							try {
208 | 								message = OZBase64.decode(lineTxt);
209 | 								for (String tmp : tmpTopicName) {
210 | 									ls.add(getMsg(tmp, tmp, message));
211 | 								}
212 | 								if (ls.size() >= cache_num) {
213 | 									send(ls, tmpTopicName, lineCount, file);
214 | 									ls = new ArrayList<KeyedMessage>(cache_num);
215 | 								}
216 | 							} catch (UnsupportedEncodingException e) {
217 | 								logger.error(e.getMessage(), e);
218 | 							}
219 | 						}
220 | 						if (ls.size() > 0) {
221 | 							send(ls, tmpTopicName, lineCount, file);
222 | 							ls = new ArrayList<KeyedMessage>(1);
223 | 						}
224 | 						System.out.println(topicName + "===Read datafile:" + (configFileName) + " use time:" + (System.currentTimeMillis() - startTime) + " linecnt:" + lineCount);
225 | 					} catch (Exception e) {
226 | 						e.printStackTrace();
227 | 						logger.error(e);
228 | 					} finally {
229 | 						if (bufferedReader != null) {
230 | 							bufferedReader.close();
231 | 							bufferedReader = null;
232 | 						}
233 | 						if (read != null) {
234 | 							read.close();
235 | 							read = null;
236 | 						}
237 | 					}
238 | 					processComplete(topicName, new File(configFileName));
239 | 				}
240 | 			}
241 | 		} catch (Exception e) {
242 | 			e.printStackTrace();
243 | 			logger.error(e);
244 | 		}
245 | 	}
246 | 
247 | 	/**
248 | 	 * 获取队列名称(根据文件名称)
249 | 	 * @param topicName
250 | 	 * @param fileName
251 | 	 * @return
252 | 	 */
253 | 	private List<String> getTopicNameWithTimestamp(String topicName, String fileName) {
254 | 		boolean isEnd = System.currentTimeMillis() > TimeService.getInstance().getEndTime();
255 | 		int cnt = isEnd ? 1 : 2;
256 | 		ArrayList<String> topics = new ArrayList<String>(cnt);
257 | 		if (!isEnd) {
258 | 			topics.add(topicName);
259 | 		}
260 | 		try {
261 | 			String topicDay = fileName.split("\\.")[2];
262 | 			format.parse(topicDay);
263 | 			if (topicDay != null && topicDay.length() == 8 && pattern.matcher(topicDay).matches()) {
264 | 				topics.add(new StringBuilder().append(topicName).append("_").append(topicDay).toString());
265 | 			} else {
266 | 				topics.add(new StringBuilder().append(topicName).append("_").append(TimeService.getInstance().getCurrentTimeYYYYMMDD()).toString());
267 | 			}
268 | 		} catch (Exception e) {
269 | 			topics.add(new StringBuilder().append(topicName).append("_").append(TimeService.getInstance().getCurrentTimeYYYYMMDD()).toString());
270 | 		}
271 | 		return topics;
272 | 	}
273 | 	
274 | 	/**
275 | 	 * 备份文件
276 | 	 * @param topic
277 | 	 * @param file
278 | 	 */
279 | 	private void processComplete(String topic, File file) {
280 | 		String fileDay = "";
281 | 		try {
282 | 			fileDay = file.getName().split("\\.")[2];
283 | 			format.parse(fileDay);
284 | 			if (fileDay == null || fileDay.length() != 8 || !pattern.matcher(fileDay).matches()) {
285 | 				fileDay = new SimpleDateFormat("yyyyMMdd").format(new Date());
286 | 			}
287 | 		} catch (Exception e) {
288 | 			fileDay = new SimpleDateFormat("yyyyMMdd").format(new Date());
289 | 		}
290 | 		String bakFullDir = getBakDir(fileDay, topic);
291 | 		moveFile(file.getAbsolutePath(), bakFullDir);
292 | 	}
293 | 	
294 | 	/**
295 | 	 * 移动文件
296 | 	 * @param srcFile
297 | 	 * @param destPath
298 | 	 * @return
299 | 	 */
300 | 	private boolean moveFile(String srcFile, String destPath){
301 | 		File fileFrom = new File(srcFile);
302 | 		if(!fileFrom.exists())
303 | 			return false;
304 | 		File toDir = new File(destPath);
305 | 		try {
306 | 			boolean success = fileFrom.renameTo(new File(toDir, fileFrom.getName()));
307 | 			return success;
308 | 		} catch (Exception e) {
309 | 			return false;
310 | 		}
311 | 	}
312 | 	
313 | 	private String getBakDir(String fileDay, String topic){
314 | 		String backUpDir = Constants.BACKUP_DATA_FILE_DIR;
315 | 		String bakFullDir = backUpDir + "/" + fileDay + "/" + topic;
316 | 		File folder = new File(bakFullDir);
317 | 		if(!folder.exists())
318 | 			folder.mkdirs();
319 | 		return bakFullDir;
320 | 	}
321 | 
322 | }
323 | 


--------------------------------------------------------------------------------
/src/main/java/com/yiguan/kafka/Start.java:
--------------------------------------------------------------------------------
  1 | package com.yiguan.kafka;
  2 | 
  3 | import java.io.IOException;
  4 | import java.util.HashMap;
  5 | import java.util.Map;
  6 | import java.util.concurrent.atomic.AtomicInteger;
  7 | 
  8 | import org.apache.log4j.Logger;
  9 | 
 10 | /**
 11 |  * 把kafka回传数据从阿里云同步到新集群的kafka <br>
 12 |  * 1、读取线上kafka数据<br>
 13 |  * 2、写入到新集群kafka队列<br>
 14 |  * 3、启动新集群kafka消费MR<br>
 15 |  * 4、在HDFS生成json<br>
 16 |  * 5、启动新集群HDFS的json消费MR<br>
 17 |  * 6、在HDFS生成4张表。<br>
 18 |  * @author Europe
 19 |  */
 20 | public class Start {
 21 |     public static AtomicInteger SDK201 = new AtomicInteger(0);
 22 |     public static AtomicInteger APP202 = new AtomicInteger(0);
 23 |     public static AtomicInteger APP300 = new AtomicInteger(0);
 24 |     public static AtomicInteger ALLAPP = new AtomicInteger(0);
 25 |     public static AtomicInteger SDK168 = new AtomicInteger(0);
 26 |     public static AtomicInteger APP200 = new AtomicInteger(0);
 27 |     public static AtomicInteger H5SDK = new AtomicInteger(0);
 28 |     public static AtomicInteger SDK2012 = new AtomicInteger(0);
 29 |     public static AtomicInteger H5SDKRT = new AtomicInteger(0);
 30 |     public static AtomicInteger APPRT = new AtomicInteger(0);
 31 | 
 32 |     public static void main(String[] args) {
 33 |         Map<String, String> topicGroupMap = new HashMap<String, String>();
 34 |         topicGroupMap.put("sdk_2_0_1_hdfs_8089_kafka", "kafka2kafka_sdk_2_0_1_group02_new");
 35 |         topicGroupMap.put("app_2_0_2", "kafka2kafka_app_2_0_2_group02_new");
 36 |         topicGroupMap.put("app_3_0", "kafka2kafka_app_3_0_group02_new");
 37 |         topicGroupMap.put("app_all_CASSANDRA", "kafka2kafka_app_all_CASSANDRA_group02_new");
 38 |         topicGroupMap.put("sdk_HDFS_168", "kafka2kafka_sdk_168_group02_new");
 39 |         topicGroupMap.put("app_2_0_HDFS", "kafka2kafka_app_2_0_group02_new");
 40 |         topicGroupMap.put("h5jssdk", "kafka2kafka_h5jssdk_group02_new");
 41 |         topicGroupMap.put("sdk_2_0_1_hdfs_8089_kafka_2", "kafka2kafka_sdk_2_0_1_2_group02_new");
 42 |         topicGroupMap.put("h5jssdk_real_time", "kafka2kafka_h5jssdk_real_time_group02_new");
 43 |         topicGroupMap.put("app_real_time", "kafka2kafka_app_real_time_group02_new");
 44 |         boolean isConsumer = false;
 45 |         int c = 1;
 46 |         if (args.length >= 1) {
 47 |             isConsumer = "c".equalsIgnoreCase(args[0]);
 48 |         }
 49 |         if (isConsumer) {
 50 |             int processCnt = 1;
 51 |             if (args.length >= 2) {
 52 |                 processCnt = Integer.valueOf(args[1]);
 53 |             }
 54 |             for (int i = 0; i < 8; i++) {
 55 |                 try {
 56 |                     if (processCnt == 2) {
 57 |                         new Thread(new Consumer("sdk_HDFS_168", "kafka2kafka_sdk_168_group02_new", Logger.getLogger("sdk_HDFS_168"))).start();
 58 |                         new Thread(new Consumer("app_2_0_HDFS", "kafka2kafka_app_2_0_group02_new", Logger.getLogger("app_2_0_HDFS"))).start();
 59 |                         new Thread(new Consumer("h5jssdk", "kafka2kafka_h5jssdk_group02_new", Logger.getLogger("h5jssdk"))).start();
 60 |                     } else if (processCnt == 3) {
 61 |                         String topicName = String.valueOf(args[2]);
 62 |                         if (topicName.equals("app_2_0_2")) {
 63 |                             new Thread(new App202Consumer(topicName, topicGroupMap.get(topicName), Logger.getLogger(topicName))).start();
 64 |                         } else {
 65 |                             new Thread(new Consumer(topicName, topicGroupMap.get(topicName), Logger.getLogger(topicName))).start();
 66 |                         }
 67 |                     }
 68 |                 } catch (IOException e) {
 69 |                     e.printStackTrace();
 70 |                 } catch (Exception e) {
 71 |                     e.printStackTrace();
 72 |                 }
 73 |                 c++;
 74 |             }
 75 |         } else {
 76 |             int processCnt = 1;
 77 |             String index = "1";
 78 |             if (args.length >= 2) {
 79 |                 processCnt = Integer.valueOf(args[1]);
 80 |             }
 81 |             if (args.length >= 4) {
 82 |             	index = args[3];
 83 |             }
 84 |             if (processCnt == 1) {
 85 |                 try {
 86 |                     new Thread(new ProducerFromFile("app_2_0_HDFS", index)).start();
 87 |                     new Thread(new ProducerFromFile("app_2_0_2", index)).start();
 88 |                     new Thread(new ProducerFromFile("app_all_CASSANDRA", index)).start();
 89 |                     new Thread(new ProducerFromFile("sdk_HDFS_168", index)).start();
 90 |                 } catch (Exception e) {
 91 |                     e.printStackTrace();
 92 |                 }
 93 |             } else if (processCnt == 3) {
 94 |             	try {
 95 |             		new ProducerFromFile(String.valueOf(args[2]), index).run();
 96 | 				} catch (Exception e) {
 97 | 					e.printStackTrace();
 98 | 				}
 99 |             }
100 |         }
101 |     }
102 | }
103 | 


--------------------------------------------------------------------------------
/src/main/java/com/yiguan/kafka/service/TimeService.java:
--------------------------------------------------------------------------------
  1 | package com.yiguan.kafka.service;
  2 | 
  3 | import java.text.ParseException;
  4 | import java.text.SimpleDateFormat;
  5 | import java.util.Date;
  6 | 
  7 | public class TimeService {
  8 | 
  9 |     private long currentTimeHourLowerLimit = 0L;
 10 |     private long currentTimeHourUpperLimit = 0L;
 11 |     private long currentTimeDayLowerLimit = 0L;
 12 |     private long currentTimeDayUpperLimit = 0L;
 13 |     private String currentTimeYYYYMMDD = null;
 14 |     private Long endTime = 1462809600000L;// "2016-5-10 0:0:0"
 15 | 
 16 |     public Long getEndTime() {
 17 |         return endTime;
 18 |     }
 19 | 
 20 |     public long getCurrentTimeDayLowerLimit() {
 21 |         return currentTimeDayLowerLimit;
 22 |     }
 23 | 
 24 |     public long getCurrentTimeDayUpperLimit() {
 25 |         return currentTimeDayUpperLimit;
 26 |     }
 27 | 
 28 |     private static TimeService instance = new TimeService();
 29 | 
 30 |     public static TimeService getInstance() {
 31 |         return instance;
 32 |     }
 33 | 
 34 |     public long getCurrentTimeHourLowerLimit() {
 35 |         return currentTimeHourLowerLimit;
 36 |     }
 37 | 
 38 |     public long getCurrentTimeHourUpperLimit() {
 39 |         return currentTimeHourUpperLimit;
 40 |     }
 41 | 
 42 |     public String getCurrentTimeYYYYMMDDHHMMSS() {
 43 |         SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 44 |         return format.format(new Date());
 45 |     }
 46 | 
 47 |     private TimeService() {
 48 |         resetHourLimits();
 49 |         resetDayLimits();
 50 | 
 51 |     }
 52 | 
 53 |     public void resetHourLimits() {
 54 |         try {
 55 |             currentTimeHourLowerLimit = getDate(System.currentTimeMillis(), "yyyy-MM-dd HH ").getTime();
 56 |             currentTimeHourUpperLimit = getDate(addHour(System.currentTimeMillis(), 1).getTime(), "yyyy-MM-dd HH ").getTime();
 57 |         } catch (ParseException e) {
 58 |             throw new RuntimeException(e);
 59 |         }
 60 |     }
 61 | 
 62 |     public void resetDayLimits() {
 63 |         try {
 64 |             currentTimeDayLowerLimit = getDate(System.currentTimeMillis(), "yyyy-MM-dd").getTime();
 65 |             currentTimeDayUpperLimit = getDate(addHour(System.currentTimeMillis(), 24).getTime(), "yyyy-MM-dd").getTime();
 66 | 
 67 |             innerGetCurrentTimeYYYYMMDD();
 68 |         } catch (ParseException e) {
 69 |             throw new RuntimeException(e);
 70 |         }
 71 |     }
 72 | 
 73 |     private void innerGetCurrentTimeYYYYMMDD() {
 74 |         SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
 75 |         currentTimeYYYYMMDD = format.format(new Date(currentTimeDayLowerLimit));
 76 |     }
 77 | 
 78 |     public String getCurrentTimeYYYYMMDD() {
 79 |         if (System.currentTimeMillis() > currentTimeDayUpperLimit) {
 80 |             synchronized (instance) {
 81 |                 resetDayLimits();
 82 |             }
 83 |         }
 84 |         return currentTimeYYYYMMDD;
 85 |     }
 86 | 
 87 |     public Date getNow() {
 88 |         return new Date();
 89 |     }
 90 | 
 91 |     public Date getDate(long timestamp, String timeformat) throws ParseException {
 92 |         SimpleDateFormat format = new SimpleDateFormat(timeformat);
 93 |         String dateStr = format.format(new Date(timestamp));
 94 |         return format.parse(dateStr);
 95 |     }
 96 | 
 97 |     public Date addHour(long timestamp, int hour) {
 98 |         return new Date(timestamp + hour * 3600 * 1000);
 99 |     }
100 | }
101 | 


--------------------------------------------------------------------------------
/src/main/java/com/yiguan/kafka/util/Constants.java:
--------------------------------------------------------------------------------
 1 | package com.yiguan.kafka.util;
 2 | 
 3 | import com.analysys.common.util.PropertiesUtil;
 4 | 
 5 | /**
 6 |  * KafkaTool 配置项
 7 |  * @author GAO
 8 |  */
 9 | public class Constants {
10 | 
11 | 	public static final String KAFKA_BROKER = PropertiesUtil.getString("consumer.kafka.broker");// "knode001:9092,knode002:9092,knode003:9092";
12 | 	public static final String ZK_CONNECT = PropertiesUtil.getString("consumer.zk");// "10.9.42.55:2181,10.9.39.238:2181,10.9.49.65:2181/kafkachroot";
13 | 	public static final String WRITE_DATA_FILE_DIR = PropertiesUtil.getString("consumer.write.data.file.dir");// "/data/topicname2/log/";
14 | 	public static final String READ_DATA_FILE_DIR = PropertiesUtil.getString("producer.read.data.file.dir", "/data1_7T/fromucloud/log");
15 | 	public static final String DEAL_DATA_FILE_DIR = PropertiesUtil.getString("producer.deal.data.file.dir", "/data1_7T/fromucloud/fordeal");
16 | 	public static final String BACKUP_DATA_FILE_DIR = PropertiesUtil.getString("producer.backup.data.file.dir", "/data1_7T/fromucloud/baklog");
17 | 	public static final String BACKUP_DIR_DYNAMICS = PropertiesUtil.getString("producer.backup.dir.dynamics", "false");
18 | 	
19 | 	// 指定序列化处理类
20 | 	public static String serializerClass = "kafka.serializer.DefaultEncoder";
21 | 	// 指定分区处理类。默认kafka.producer.DefaultPartitioner，表通过key哈希到对应分区
22 | 	public static String partitionerClass = "kafka.producer.DefaultPartitioner";
23 | 	// 是否压缩，默认0表示不压缩，1表示用gzip压缩，2表示用snappy压缩
24 | 	public static String compressionCodec = "2";
25 | 	// 如果要压缩消息，这里指定哪些topic要压缩消息，默认empty，表示不压缩。
26 | 	// public static String compressedTopics = "kafka_test_ouzhou";
27 | 	public static String compressedTopics = "t08,test_sdk_gaochao";
28 | 	// 该属性表示你需要在消息被接收到的时候发送ack给发送者。以保证数据不丢失
29 | 	public static String acks = "1";
30 | 	// 默认“sync”表同步,"async"表异步.异步可以提高发送吞吐量
31 | 	public static String producerType = "async";
32 | 	// 在向producer发送ack之前,broker允许等待的最大时间 ,如果超时,broker将会向producer发送一个error
33 | 	// ACK.意味着上一次消息因为某种 # 原因未能成功(比如follower未能同步成功)
34 | 	public static String requestTimeoutMS = "10000";
35 | 	// 在async模式下,当message被缓存的时间超过此值后,将会批量发送给broker,默认为5000ms
36 | 	public static String queueBufferingMaxMS = "10000";
37 | 	public static String queueBufferingMaxMessages = "20000";
38 | 	// 异步 每次批次发送的数量 默认200
39 | 	public static String batchNumMessages = "500";
40 | 	public static String queueEnqueueTimeoutMS = "-1";
41 | 
42 | }
43 | 


--------------------------------------------------------------------------------
/src/main/java/com/yiguan/kafka/util/DistinctAST.java:
--------------------------------------------------------------------------------
 1 | package com.yiguan.kafka.util;
 2 | 
 3 | import java.io.BufferedWriter;
 4 | import java.io.File;
 5 | import java.io.FileOutputStream;
 6 | import java.io.IOException;
 7 | import java.io.OutputStreamWriter;
 8 | import java.util.ArrayList;
 9 | import java.util.List;
10 | import java.util.Map;
11 | 
12 | import net.sf.json.JSONArray;
13 | import net.sf.json.JSONObject;
14 | 
15 | public class DistinctAST {
16 | 
17 | 	@SuppressWarnings({ "rawtypes", "unused", "unchecked" })
18 | 	public static String distinctAll(String jsonStrDouble) throws IOException {
19 | 		JSONObject jsonObject = null;
20 | 		JSONObject jsonObjectNew = null;
21 | 		List listNew = new ArrayList();
22 | 		String timeStr = null;
23 | 		String AET = "";
24 | 		JSONObject retunJson = new JSONObject();
25 | 		// 处理重复数据
26 | 		String[] ss = jsonStrDouble.split("-----");
27 | 		jsonObject = JSONObject.fromObject(ss[0]);
28 | 		String asdString = jsonObject.getString("AInfo");
29 | 		JSONArray jsonarray = JSONArray.fromObject(asdString);
30 | 		List list = (List) JSONArray.toCollection(jsonarray, Map.class);
31 | 		if (list.size() > 0) {
32 | 			timeStr = JSONObject.fromObject(list.get(0)).getString("AST");
33 | 		}
34 | 		for (int i = 1, j = 1, count = list.size(); i < count; i++) {
35 | 			long time = Long.parseLong(timeStr);
36 | 			long timeNew = Long.parseLong(JSONObject.fromObject(list.get(i)).getString("AST"));
37 | 			jsonObjectNew = JSONObject.fromObject(list.get(i - j));
38 | 			if (time + 60000 > timeNew) {
39 | 				AET = JSONObject.fromObject(list.get(i)).getString("AST");
40 | 				j++;
41 | 				jsonObjectNew.put("AET", AET);
42 | 			} else {
43 | 				jsonObjectNew.put("AET", AET);
44 | 				timeStr = JSONObject.fromObject(list.get(i)).getString("AST");
45 | 				AET = "";
46 | 				j = 1;
47 | 				listNew.add(jsonObjectNew);
48 | 			}
49 | 			if (i + j == count + 1) {
50 | 				jsonObjectNew.put("AET", AET);
51 | 				listNew.add(jsonObjectNew);
52 | 			}
53 | 		}
54 | 		// 拼接字符串
55 | 		jsonObject.put("AInfo", listNew);
56 | 		String tmpNew = jsonObject + "-----" + ss[1];
57 | 		return tmpNew;
58 | 	}
59 | 
60 | 	public void writeToTxt(String data) throws IOException {
61 | 		FileOutputStream fos = new FileOutputStream(new File("E:\\1111\\new15"), true);
62 | 		OutputStreamWriter osw = new OutputStreamWriter(fos, "UTF-8");
63 | 		BufferedWriter bw = new BufferedWriter(osw);
64 | 		bw.write(data + "\t\n");
65 | 		bw.close();
66 | 		osw.close();
67 | 		fos.close();
68 | 	}
69 | }
70 | 


--------------------------------------------------------------------------------
/src/main/java/com/yiguan/kafka/util/OZBase64.java:
--------------------------------------------------------------------------------
  1 | package com.yiguan.kafka.util;
  2 | 
  3 | import java.io.UnsupportedEncodingException;
  4 | 
  5 | 
  6 | /**
  7 |  * @author apple
  8 |  * 用来进行字符串的加密
  9 |  */
 10 | public class OZBase64 {
 11 | 
 12 | 	/**
 13 | 	 * 将 base64 形式的字符串转为正常的字符串
 14 | 	 * @param base64str 传入的base64形式的字符串
 15 | 	 */
 16 | 	public static String ozBase64ToStr(String base64str){
 17 | 		String base64Codep;
 18 | 		try {
 19 | 			base64Codep = new String(OZBase64.decode(base64str));
 20 | 			System.out.println("Base64转为字符串完成，结果为：\r"+base64Codep+"\r");
 21 | 			return base64Codep;
 22 | 		} catch (UnsupportedEncodingException e) {
 23 | 			// TODO Auto-generated catch block
 24 | 			e.printStackTrace();
 25 | 			return null;
 26 | 		}
 27 | 	}
 28 | 	
 29 | 	/**
 30 | 	 * 将 字符串转为 base64 
 31 | 	 * @param str 字符串
 32 | 	 */
 33 | 	public static String ozStrToBase64(String str){
 34 | 		String basestr = OZBase64.encode(str.getBytes());
 35 | 		System.out.println("字符串转为Base64完成，结果为：\r"+basestr+"\r");
 36 | 		return basestr;
 37 | 	}
 38 | 	
 39 | 	private static char[] base64EncodeChars = new char[] {
 40 |         'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
 41 |         'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
 42 |         'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
 43 |         'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
 44 |         'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
 45 |         'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
 46 |         'w', 'x', 'y', 'z', '0', '1', '2', '3',
 47 |         '4', '5', '6', '7', '8', '9', '+', '/' };
 48 | 	
 49 | 	private static byte[] base64DecodeChars = new byte[] {
 50 | 	    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
 51 | 	    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
 52 | 	    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63,
 53 | 	    52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1,
 54 | 	    -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 55 | 	    15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1,
 56 | 	    -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
 57 | 	    41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1 };
 58 | 	
 59 | 	//编码
 60 |     public static String encode(byte[] data) {
 61 |         StringBuffer sb = new StringBuffer();
 62 |         int len = data.length;
 63 |         int i = 0;
 64 |         int b1, b2, b3;
 65 |         while (i < len) {
 66 |             b1 = data[i++] & 0xff;
 67 |             if (i == len)
 68 |             {
 69 |                 sb.append(base64EncodeChars[b1 >>> 2]);
 70 |                 sb.append(base64EncodeChars[(b1 & 0x3) << 4]);
 71 |                 sb.append("==");
 72 |                 break;
 73 |             }
 74 |             b2 = data[i++] & 0xff;
 75 |             if (i == len)
 76 |             {
 77 |                 sb.append(base64EncodeChars[b1 >>> 2]);
 78 |                 sb.append(base64EncodeChars[((b1 & 0x03) << 4) | ((b2 & 0xf0) >>> 4)]);
 79 |                 sb.append(base64EncodeChars[(b2 & 0x0f) << 2]);
 80 |                 sb.append("=");
 81 |                 break;
 82 |             }
 83 |             b3 = data[i++] & 0xff;
 84 |             sb.append(base64EncodeChars[b1 >>> 2]);
 85 |             sb.append(base64EncodeChars[((b1 & 0x03) << 4) | ((b2 & 0xf0) >>> 4)]);
 86 |             sb.append(base64EncodeChars[((b2 & 0x0f) << 2) | ((b3 & 0xc0) >>> 6)]);
 87 |             sb.append(base64EncodeChars[b3 & 0x3f]);
 88 |         }
 89 |         return sb.toString();
 90 |     }
 91 |     //解码
 92 |     public static byte[] decode(String str) throws UnsupportedEncodingException {
 93 |         StringBuffer sb = new StringBuffer();
 94 |         byte[] data = str.getBytes("US-ASCII");
 95 |         int len = data.length;
 96 |         int i = 0;
 97 |         int b1, b2, b3, b4;
 98 |         while (i < len) {
 99 |             /* b1 */
100 |             do {
101 |                 b1 = base64DecodeChars[data[i++]];
102 |             } while (i < len && b1 == -1);
103 |             if (b1 == -1) break;
104 |             /* b2 */
105 |             do {
106 |                 b2 = base64DecodeChars[data[i++]];
107 |             } while (i < len && b2 == -1);
108 |             if (b2 == -1) break;
109 |             sb.append((char)((b1 << 2) | ((b2 & 0x30) >>> 4)));
110 |             /* b3 */
111 |             do {
112 |                 b3 = data[i++];
113 |                 if (b3 == 61) return sb.toString().getBytes("iso8859-1");
114 |                 b3 = base64DecodeChars[b3];
115 |             } while (i < len && b3 == -1);
116 |             if (b3 == -1) break;
117 |             sb.append((char)(((b2 & 0x0f) << 4) | ((b3 & 0x3c) >>> 2)));
118 |             /* b4 */
119 |             do {
120 |                 b4 = data[i++];
121 |                 if (b4 == 61) return sb.toString().getBytes("iso8859-1");
122 |                 b4 = base64DecodeChars[b4];
123 |             } while (i < len && b4 == -1);
124 |             if (b4 == -1) break;
125 |             sb.append((char)(((b3 & 0x03) << 6) | b4));
126 |         }
127 |         return sb.toString().getBytes("iso8859-1");
128 |     }
129 | 	
130 | 	public static void main(String[] args) {
131 | 		ozBase64ToStr("YXNkZmFzZGZhc2Rm");
132 | 		ozStrToBase64("asdfasdfasdf");
133 | 	}
134 | 	
135 | }
136 | 


--------------------------------------------------------------------------------
/src/main/java/com/yiguan/kafka/util/ProducerConstants.java:
--------------------------------------------------------------------------------
 1 | package com.yiguan.kafka.util;
 2 | 
 3 | import com.analysys.common.util.PropertiesUtil;
 4 | 
 5 | /**
 6 |  * KafkaTool 配置项
 7 |  * @author GAO
 8 |  */
 9 | public class ProducerConstants {
10 | 	/*新集群*/
11 | 	public static final String KAFKA_BROKER = PropertiesUtil.getString("producer.kafka.broker");
12 | 	public static final String ZK_CONNECT = PropertiesUtil.getString("producer.zk");
13 | 	public static final String FILE_QUEUE_HOST = PropertiesUtil.getString("producer.read.file.queue.service");
14 | 
15 | 	// 指定序列化处理类
16 | 	public static String serializerClass = "kafka.serializer.DefaultEncoder";
17 | 	// 指定分区处理类。默认kafka.producer.DefaultPartitioner，表通过key哈希到对应分区
18 | 	public static String partitionerClass = "kafka.producer.DefaultPartitioner";
19 | 	// 是否压缩，默认0表示不压缩，1表示用gzip压缩，2表示用snappy压缩
20 | 	public static String compressionCodec = "2";
21 | 	// 如果要压缩消息，这里指定哪些topic要压缩消息，默认empty，表示不压缩。
22 | 	// public static String compressedTopics = "kafka_test_ouzhou";
23 | 	public static String compressedTopics = "t08,test_sdk_gaochao";
24 | 	// 该属性表示你需要在消息被接收到的时候发送ack给发送者。以保证数据不丢失
25 | 	public static String acks = "1";
26 | 	// 默认“sync”表同步,"async"表异步.异步可以提高发送吞吐量
27 | 	public static String producerType = "async";
28 | 	// 在向producer发送ack之前,broker允许等待的最大时间 ,如果超时,broker将会向producer发送一个error
29 | 	// ACK.意味着上一次消息因为某种 # 原因未能成功(比如follower未能同步成功)
30 | 	public static String requestTimeoutMS = "10000";
31 | 	// 在async模式下,当message被缓存的时间超过此值后,将会批量发送给broker,默认为5000ms
32 | 	public static String queueBufferingMaxMS = "10000";
33 | 	public static String queueBufferingMaxMessages = "20000";
34 | 	// 异步 每次批次发送的数量 默认200
35 | 	public static String batchNumMessages = "500";
36 | 	public static String queueEnqueueTimeoutMS = "-1";
37 | }
38 | 


--------------------------------------------------------------------------------
/src/main/main.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="JAVA_MODULE" version="4">
 3 |   <component name="NewModuleRootManager" inherit-compiler-output="true">
 4 |     <exclude-output />
 5 |     <content url="file://$MODULE_DIR$">
 6 |       <sourceFolder url="file://$MODULE_DIR$/java" isTestSource="false" />
 7 |     </content>
 8 |     <orderEntry type="inheritedJdk" />
 9 |     <orderEntry type="sourceFolder" forTests="false" />
10 |   </component>
11 | </module>


--------------------------------------------------------------------------------
/src/main/resources/beijingidc-resources/conf.properties:
--------------------------------------------------------------------------------
 1 | ###
 2 | consumer.kafka.broker=knode001:9092,knode002:9092,knode003:9092
 3 | consumer.zk=knode001:2181,knode002:2181,knode003:2181/kafkachroot
 4 | consumer.write.data.file.dir=/data/log
 5 | 
 6 | producer.kafka.broker=knode001:9092,knode002:9092,knode003:9092,knode004:9092,knode005:9092
 7 | producer.zk=knode001:2181,knode002:2181,knode003:2181/kafka
 8 | producer.read.file.queue.service=127.0.0.1:8092
 9 | producer.read.data.file.dir=/data/log
10 | producer.deal.data.file.dir=/data/fordeal
11 | producer.backup.data.file.dir=/data/baklog
12 | producer.backup.dir.dynamics=false
13 | 


--------------------------------------------------------------------------------
/src/main/resources/beijingidc-resources/log4j.properties:
--------------------------------------------------------------------------------
 1 | ####################### sdk facli 1.0 server 109 #####################
 2 | log4j.logger.ProducerLogs=INFO,ProducerLogs
 3 | log4j.additivity.ProducerLogs = false
 4 | log4j.appender.ProducerLogs=org.apache.log4j.RollingFileAppender
 5 | log4j.appender.ProducerLogs.MaxFileSize=500MB
 6 | log4j.appender.ProducerLogs.MaxBackupIndex=10000
 7 | log4j.appender.ProducerLogs.File=/data/tobeijingidc/plog/ProducerLogs
 8 | log4j.appender.ProducerLogs.layout=org.apache.log4j.PatternLayout
 9 | log4j.appender.ProducerLogs.filter.F1=org.apache.log4j.varia.LevelRangeFilter
10 | 
11 | log4j.logger.ErrorLogs=INFO,ErrorLogs
12 | log4j.additivity.ErrorLogs = false
13 | log4j.appender.ErrorLogs=org.apache.log4j.RollingFileAppender
14 | log4j.appender.ErrorLogs.MaxFileSize=500MB
15 | log4j.appender.ErrorLogs.MaxBackupIndex=10000
16 | log4j.appender.ErrorLogs.File=/data/toqingdaoKafkaRsync/monitor/logs/ErrorLogs
17 | log4j.appender.ErrorLogs.layout=org.apache.log4j.PatternLayout
18 | log4j.appender.ErrorLogs.filter.F1=org.apache.log4j.varia.LevelRangeFilter
19 | 


--------------------------------------------------------------------------------
/src/main/resources/ucloud-resources/conf.properties:
--------------------------------------------------------------------------------
1 | consumer.kafka.broker=knode001:9092,knode002:9092,knode003:9092
2 | consumer.zk=knode001:2181,knode002:2181,knode003:2181/kafka
3 | consumer.write.data.file.dir=/data/tobeijingidc/log
4 | 
5 | producer.kafka.broker=knode001:9092,knode002:9092,knode003:9092
6 | producer.zk=knode001:2181,knode002:2181,knode003:2181/kafka
7 | producer.read.file.queue.service=127.0.0.1:8091
8 | 


--------------------------------------------------------------------------------
/src/main/resources/ucloud-resources/log4j.properties:
--------------------------------------------------------------------------------
 1 | ###
 2 | log4j.logger.app_real_time=INFO,app_real_time
 3 | log4j.additivity.app_real_time = false
 4 | log4j.appender.app_real_time=org.apache.log4j.RollingFileAppender
 5 | log4j.appender.app_real_time.MaxFileSize=10MB
 6 | log4j.appender.app_real_time.File=/data/tobeijingidc/log/app_real_time${topic.day}
 7 | log4j.appender.app_real_time.layout=org.apache.log4j.PatternLayout
 8 | log4j.appender.app_real_time.filter.F1=org.apache.log4j.varia.LevelRangeFilter
 9 | log4j.appender.app_real_time.MaxBackupIndex=10000
10 | 
11 | ####################### sdk facli 1.0 server 109 #####################
12 | log4j.logger.app_2_0_2=INFO,app_2_0_2
13 | log4j.additivity.app_2_0_2 = false
14 | log4j.appender.app_2_0_2=org.apache.log4j.RollingFileAppender
15 | log4j.appender.app_2_0_2.MaxFileSize=500MB
16 | log4j.appender.app_2_0_2.File=/data/tobeijingidc/log/app_2_0_2${topic.day}
17 | log4j.appender.app_2_0_2.layout=org.apache.log4j.PatternLayout
18 | log4j.appender.app_2_0_2.filter.F1=org.apache.log4j.varia.LevelRangeFilter
19 | log4j.appender.app_2_0_2.MaxBackupIndex=10000
20 | 
21 | log4j.logger.app_3_0=INFO,app_3_0
22 | log4j.additivity.app_3_0 = false
23 | log4j.appender.app_3_0=org.apache.log4j.RollingFileAppender
24 | log4j.appender.app_3_0.MaxFileSize=500MB
25 | log4j.appender.app_3_0.File=/data/tobeijingidc/log/app_3_0${topic.day}
26 | log4j.appender.app_3_0.layout=org.apache.log4j.PatternLayout
27 | log4j.appender.app_3_0.filter.F1=org.apache.log4j.varia.LevelRangeFilter
28 | log4j.appender.app_3_0.MaxBackupIndex=10000
29 | 
30 | log4j.logger.sdk_HDFS_168=INFO,sdk_HDFS_168
31 | log4j.additivity.sdk_HDFS_168 = false
32 | log4j.appender.sdk_HDFS_168=org.apache.log4j.RollingFileAppender
33 | log4j.appender.sdk_HDFS_168.MaxFileSize=10MB
34 | log4j.appender.sdk_HDFS_168.File=/data/tobeijingidc/log/sdk_HDFS_168${topic.day}
35 | log4j.appender.sdk_HDFS_168.layout=org.apache.log4j.PatternLayout
36 | log4j.appender.sdk_HDFS_168.filter.F1=org.apache.log4j.varia.LevelRangeFilter
37 | log4j.appender.sdk_HDFS_168.MaxBackupIndex=10000
38 | 
39 | log4j.logger.app_2_0_HDFS=INFO,app_2_0_HDFS
40 | log4j.additivity.app_2_0_HDFS = false
41 | log4j.appender.app_2_0_HDFS=org.apache.log4j.RollingFileAppender
42 | log4j.appender.app_2_0_HDFS.MaxFileSize=10MB
43 | log4j.appender.app_2_0_HDFS.MaxBackupIndex=10000
44 | log4j.appender.app_2_0_HDFS.File=/data/tobeijingidc/log/app_2_0_HDFS${topic.day}
45 | log4j.appender.app_2_0_HDFS.layout=org.apache.log4j.PatternLayout
46 | log4j.appender.app_2_0_HDFS.filter.F1=org.apache.log4j.varia.LevelRangeFilter
47 | 
48 | log4j.logger.h5jssdk=INFO,h5jssdk
49 | log4j.additivity.h5jssdk = false
50 | log4j.appender.h5jssdk=org.apache.log4j.RollingFileAppender
51 | log4j.appender.h5jssdk.MaxFileSize=200KB
52 | log4j.appender.h5jssdk.MaxBackupIndex=10000
53 | log4j.appender.h5jssdk.File=/data/tobeijingidc/log/h5jssdk${topic.day}
54 | log4j.appender.h5jssdk.layout=org.apache.log4j.PatternLayout
55 | log4j.appender.h5jssdk.filter.F1=org.apache.log4j.varia.LevelRangeFilter
56 | 
57 | log4j.logger.sdk_2_0_1_hdfs_8089_kafka=INFO,sdk_2_0_1_hdfs_8089_kafka
58 | log4j.additivity.sdk_2_0_1_hdfs_8089_kafka = false
59 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka=org.apache.log4j.RollingFileAppender
60 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka.MaxFileSize=500MB
61 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka.File=/data/tobeijingidc/log/sdk_2_0_1_hdfs_8089_kafka${topic.day}
62 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka.layout=org.apache.log4j.PatternLayout
63 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka.filter.F1=org.apache.log4j.varia.LevelRangeFilter
64 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka.MaxBackupIndex=10000
65 | 
66 | log4j.logger.sdk_2_0_1_hdfs_8089_kafka_2=INFO,sdk_2_0_1_hdfs_8089_kafka_2
67 | log4j.additivity.sdk_2_0_1_hdfs_8089_kafka_2 = false
68 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka_2=org.apache.log4j.RollingFileAppender
69 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka_2.MaxFileSize=100MB
70 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka_2.MaxBackupIndex=10000
71 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka_2.File=/data/tobeijingidc/log/sdk_2_0_1_hdfs_8089_kafka_2${topic.day}
72 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka_2.layout=org.apache.log4j.PatternLayout
73 | log4j.appender.sdk_2_0_1_hdfs_8089_kafka_2.filter.F1=org.apache.log4j.varia.LevelRangeFilter
74 | 
75 | log4j.logger.ConsumeLogs=INFO,ConsumeLogs
76 | log4j.additivity.ConsumeLogs = false
77 | log4j.appender.ConsumeLogs=org.apache.log4j.RollingFileAppender
78 | log4j.appender.ConsumeLogs.MaxFileSize=500MB
79 | log4j.appender.ConsumeLogs.MaxBackupIndex=10000
80 | log4j.appender.ConsumeLogs.File=/data/tobeijingidc/plog/ConsumeLogs
81 | log4j.appender.ConsumeLogs.layout=org.apache.log4j.PatternLayout
82 | log4j.appender.ConsumeLogs.filter.F1=org.apache.log4j.varia.LevelRangeFilter
83 | 
84 | log4j.logger.ErrorLogs=INFO,ErrorLogs
85 | log4j.additivity.ErrorLogs = false
86 | log4j.appender.ErrorLogs=org.apache.log4j.RollingFileAppender
87 | log4j.appender.ErrorLogs.MaxFileSize=500MB
88 | log4j.appender.ErrorLogs.MaxBackupIndex=10000
89 | log4j.appender.ErrorLogs.File=/data/toqingdaoKafkaRsync/monitor/logs/ErrorLogs
90 | log4j.appender.ErrorLogs.layout=org.apache.log4j.PatternLayout
91 | log4j.appender.ErrorLogs.filter.F1=org.apache.log4j.varia.LevelRangeFilter
92 | 


--------------------------------------------------------------------------------
/src/test/test.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="JAVA_MODULE" version="4">
 3 |   <component name="NewModuleRootManager" inherit-compiler-output="true">
 4 |     <exclude-output />
 5 |     <content url="file://$MODULE_DIR$">
 6 |       <sourceFolder url="file://$MODULE_DIR$/java" isTestSource="true" />
 7 |     </content>
 8 |     <orderEntry type="inheritedJdk" />
 9 |     <orderEntry type="sourceFolder" forTests="false" />
10 |     <orderEntry type="module" module-name="main" />
11 |   </component>
12 | </module>


--------------------------------------------------------------------------------