├── .gitignore ├── flume-logging-monitor-handler ├── src │ └── main │ │ ├── java │ │ └── com │ │ │ └── shawn │ │ │ ├── logging │ │ │ └── monitor │ │ │ │ └── handler │ │ │ │ ├── EventSendingTestScript.java │ │ │ │ ├── LogWatchRunScript.java │ │ │ │ ├── EventSendingTest.java │ │ │ │ ├── JsonFormatMapConfigHelper.java │ │ │ │ ├── LogAttribute.java │ │ │ │ ├── LogMonitorClient.java │ │ │ │ ├── LogWatcher.java │ │ │ │ ├── LogWatcherBySchedule.java │ │ │ │ └── LogWatcherByWatchEvent.java │ │ │ └── App.java │ │ └── resources │ │ ├── applicationContext.xml │ │ └── configuration.properties └── pom.xml ├── flume-logging-monitor-client ├── src │ └── main │ │ ├── conf │ │ ├── applicationContext.xml │ │ ├── configuration.properties │ │ ├── logHeader.json.example │ │ └── developer-guide.md │ │ ├── assembly │ │ └── bin.xml │ │ └── bin │ │ └── monitor └── pom.xml ├── README.md └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | # kdiff3 ignore 2 | *.orig 3 | 4 | # maven ignore 5 | target/ 6 | 7 | # eclipse ignore 8 | .settings/ 9 | .project 10 | .classpath 11 | 12 | # idea ignore 13 | .idea/ 14 | *.ipr 15 | *.iml 16 | *.iws 17 | 18 | # temp ignore 19 | *.log 20 | *.cache 21 | *.diff 22 | *.patch 23 | *.tmp 24 | 25 | # system ignore 26 | .DS_Store 27 | Thumbs.db 28 | 29 | # configuration ignore 30 | *.json 31 | log4j.properties 32 | # package ignore (optional) 33 | # *.jar 34 | # *.war 35 | # *.zip 36 | # *.tar 37 | # *.tar.gz 38 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/java/com/shawn/logging/monitor/handler/EventSendingTestScript.java: -------------------------------------------------------------------------------- 1 | package com.shawn.logging.monitor.handler; 2 | 3 | import org.springframework.context.ApplicationContext; 4 | import org.springframework.context.support.ClassPathXmlApplicationContext; 5 | 6 | public class EventSendingTestScript { 7 | public static void main(String[] args) throws Exception{ 8 | ApplicationContext context = new ClassPathXmlApplicationContext("applicationContext.xml"); 9 | LogWatcher service = context.getBean(EventSendingTest.class); 10 | service.start(); 11 | service.process(); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/java/com/shawn/logging/monitor/handler/LogWatchRunScript.java: -------------------------------------------------------------------------------- 1 | package com.shawn.logging.monitor.handler; 2 | 3 | import org.springframework.context.ApplicationContext; 4 | import org.springframework.context.support.ClassPathXmlApplicationContext; 5 | 6 | 7 | public class LogWatchRunScript { 8 | public static void main(String[] args) throws Exception { 9 | ApplicationContext context = new ClassPathXmlApplicationContext("applicationContext.xml"); 10 | // LogWatcher service = context.getBean(LogWatcherByWatchEvent.class); 11 | LogWatcher service = context.getBean(LogWatcherBySchedule.class); 12 | service.start(); 13 | service.process(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /flume-logging-monitor-client/src/main/conf/applicationContext.xml: -------------------------------------------------------------------------------- 1 | 2 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/resources/applicationContext.xml: -------------------------------------------------------------------------------- 1 | 2 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/resources/configuration.properties: -------------------------------------------------------------------------------- 1 | #the agent host 2 | monitor.agent.host=127.0.0.1 3 | #the agent port 4 | monitor.agent.port=41414 5 | #timeout on connecting the agent port 6 | monitor.client.timeout=10000 7 | #retry times on connecting the agent 8 | monitor.client.maxAttempts = 2 9 | #time interval of the scheduling thread detects the log changes (in milliseconds) 10 | monitor.client.schedule.delay=1000 11 | #for multiple file ,separated by commas 12 | monitor.client.schedule.watchpaths=/opt/apache-tomcat-7.0.42/logs/test_logs/append-medium_0,/opt/apache-tomcat-7.0.42/logs/test_logs/append-medium_1,/opt/apache-tomcat-7.0.42/logs/test_logs/append-medium_2,/opt/apache-tomcat-7.0.42/logs/test_logs/append-medium_3 13 | 14 | #stash the log status to local file when client shutdown 15 | monitor.client.logStatusRecordPath=log_record 16 | 17 | #max size for each source event; theoretically, less than Integer.MAX_VALUE : 2147483647 18 | monitor.client.event.size = 5242880 19 | 20 | #event header configuration 21 | monitor.event.headerConfigPath=/home/shawncao/Desktop/logHeader.json 22 | 23 | # log suffix date format 24 | monitor.log.suffix.date=yy-MM-dd 25 | -------------------------------------------------------------------------------- /flume-logging-monitor-client/src/main/conf/configuration.properties: -------------------------------------------------------------------------------- 1 | #the agent host 2 | monitor.agent.host=10.40.5.3 3 | #the agent port 4 | monitor.agent.port=41410 5 | #timeout on connecting the agent port 6 | monitor.client.timeout=10000 7 | #retry times on connecting the agent 8 | monitor.client.maxAttempts = 2 9 | #time interval of the scheduling thread detects the log changes (in milliseconds) 10 | monitor.client.schedule.delay=2000 11 | #for multiple file ,separated by commas 12 | monitor.client.schedule.watchpaths=/var/log/nginx/www.shawn.com.access_log,/var/log/nginx/www.shawn.com.error_log,/var/log/nginx/login.shawn.com.access_log,/var/log/nginx/login.shawn.com.error_log,/var/log/nginx/job.shawn.com.access_log,/var/log/nginx/job.shawn.com.error_log,/var/log/nginx/image.shawn.com.access_log,/var/log/nginx/image.shawn.com.error_log,/var/log/nginx/rest.shawn.com.access_log,/var/log/nginx/rest.shawn.com.error_log,/var/log/haproxy.log, 13 | 14 | #max size for each source event; theoretically, less than Integer.MAX_VALUE : 2147483647 15 | monitor.client.event.size = 5242880 16 | 17 | #stash the log status to local file when client shutdown 18 | monitor.client.logStatusRecordPath=log_record 19 | 20 | #event header configuration 21 | monitor.event.headerConfigPath=./conf/logHeader_integ.json 22 | 23 | 24 | -------------------------------------------------------------------------------- /flume-logging-monitor-client/src/main/assembly/bin.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | bin 9 | 10 | tar.gz 11 | 12 | 13 | false 14 | 15 | 16 | 17 | flume-client/lib/ 18 | false 19 | 20 | 21 | 22 | 23 | 24 | src/main/README.md 25 | flume-client/ 26 | true 27 | 28 | 29 | 30 | 31 | 32 | src/main/conf 33 | flume-client/conf 34 | 35 | 36 | 37 | src/main/bin 38 | flume-client/bin 39 | 0775 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /flume-logging-monitor-client/src/main/conf/logHeader.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "/var/log/nginx/www.shawn.com.access_log": { 3 | "domain": "www", 4 | "log_type": "access", 5 | "server_type":"nginx" 6 | }, 7 | "/var/log/nginx/www.shawn.com.error_log": { 8 | "domain": "www", 9 | "log_type": "error", 10 | "server_type":"nginx" 11 | }, 12 | "/var/log/nginx/login.shawn.com.access_log": { 13 | "domain": "login", 14 | "log_type": "access", 15 | "server_type":"nginx" 16 | }, 17 | "/var/log/nginx/login.shawn.com.error_log": { 18 | "domain": "login", 19 | "log_type": "error", 20 | "server_type":"nginx" 21 | }, 22 | "/var/log/nginx/job.shawn.com.access_log": { 23 | "domain": "job", 24 | "log_type": "access", 25 | "server_type":"nginx" 26 | }, 27 | "/var/log/nginx/job.shawn.com.error_log": { 28 | "domain": "job", 29 | "log_type": "error", 30 | "server_type":"nginx" 31 | }, 32 | "/var/log/nginx/image.shawn.com.access_log": { 33 | "domain": "image", 34 | "log_type": "access", 35 | "server_type":"nginx" 36 | }, 37 | "/var/log/nginx/image.shawn.com.error_log": { 38 | "domain": "image", 39 | "log_type": "error", 40 | "server_type":"nginx" 41 | }, 42 | "/var/log/nginx/rest.shawn.com.access_log": { 43 | "domain": "rest", 44 | "log_type": "access", 45 | "server_type":"nginx" 46 | }, 47 | "/var/log/nginx/rest.shawn.com.error_log": { 48 | "domain": "rest", 49 | "log_type": "error", 50 | "server_type":"nginx" 51 | }, 52 | "/var/log/haproxy.log": { 53 | "domain": "default", 54 | "log_type": "default", 55 | "server_type":"haproxy" 56 | } 57 | } -------------------------------------------------------------------------------- /flume-logging-monitor-client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | com.shawn 6 | flume-logging-monitor 7 | 0.1-SNAPSHOT 8 | 9 | flume-logging-monitor-client 10 | pom 11 | flume-logging-monitor-client 12 | http://maven.apache.org 13 | 14 | UTF-8 15 | 16 | 17 | 18 | junit 19 | junit 20 | test 21 | 22 | 23 | ${project.groupId} 24 | flume-logging-monitor-handler 25 | ${project.version} 26 | 27 | 28 | 29 | 30 | 31 | org.apache.maven.plugins 32 | maven-assembly-plugin 33 | 34 | 35 | assembly 36 | package 37 | 38 | single 39 | 40 | 41 | 42 | src/main/assembly/bin.xml 43 | 44 | gnu 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | com.shawn 6 | flume-logging-monitor 7 | 0.1-SNAPSHOT 8 | 9 | flume-logging-monitor-handler 10 | jar 11 | flume-logging-monitor-handler 12 | http://maven.apache.org 13 | 14 | UTF-8 15 | 16 | 17 | 18 | junit 19 | junit 20 | test 21 | 22 | 23 | org.apache.flume 24 | flume-ng-core 25 | 26 | 27 | org.slf4j 28 | slf4j-api 29 | 30 | 31 | org.slf4j 32 | slf4j-log4j12 33 | 34 | 35 | org.springframework 36 | spring-core 37 | 38 | 39 | org.springframework 40 | spring-beans 41 | 42 | 43 | org.springframework 44 | spring-context 45 | 46 | 47 | com.google.code.gson 48 | gson 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /flume-logging-monitor-client/src/main/bin/monitor: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source /etc/profile 3 | export LANG=en_US.utf8 4 | PRG=$0 5 | PRGDIR=`dirname "$PRG"` 6 | CLASSPATH="$PRGDIR/../conf" 7 | FILES=$PRGDIR/../lib/* 8 | 9 | test() { 10 | ps aux |grep -v grep|grep EventSendingTestScript -q 11 | if [ $? -eq 0 ];then 12 | echo eventsTest is already running 13 | exit 1 14 | fi 15 | 16 | echo EventSendingTest is starting. 17 | for f in $FILES 18 | do 19 | CLASSPATH=${CLASSPATH}:$f 20 | done 21 | echo ${CLASSPATH} 22 | java -classpath ${CLASSPATH} com.shawn.logging.monitor.handler.EventSendingTestScript & 23 | 24 | } 25 | 26 | start(){ 27 | ps aux |grep -v grep|grep LogWatchRunScript -q 28 | if [ $? -eq 0 ];then 29 | echo log watcher is already running 30 | exit 1 31 | fi 32 | 33 | echo log watcher client is starting. 34 | for f in $FILES 35 | do 36 | CLASSPATH=${CLASSPATH}:$f 37 | done 38 | echo ${CLASSPATH} 39 | java -Xms1024m -Xmx2048m -classpath ${CLASSPATH} com.shawn.logging.monitor.handler.LogWatchRunScript & 40 | } 41 | 42 | stop(){ 43 | ps aux |grep -v grep|grep LogWatchRunScript -q 44 | if [ $? -ne 0 ];then 45 | echo LogWatchRunScript is not running 46 | else 47 | ps aux |grep -v grep|grep LogWatchRunScript|awk '{print $2}'|xargs kill -15 48 | sleep 3 49 | ps aux |grep -v grep|grep LogWatchRunScript -q 50 | [ $? -eq 0 ] && ps aux |grep LogWatchRunScript|grep -v grep|awk '{print $2}'|xargs kill -9 51 | fi 52 | 53 | 54 | } 55 | 56 | status(){ 57 | ps aux |grep -v grep|grep LogWatchRunScript -q 58 | if [ $? -eq 0 ];then 59 | echo LogWatchRunScript is running 60 | else 61 | echo LogWatchRunScript is not running 62 | fi 63 | } 64 | 65 | help(){ 66 | echo "Usage : [start|stop|restart|status]" 67 | } 68 | 69 | case $1 in 70 | help) 71 | help 72 | ;; 73 | test) 74 | test 75 | ;; 76 | start) 77 | start 78 | ;; 79 | stop) 80 | stop 81 | ;; 82 | restart) 83 | stop 84 | sleep 1 85 | start 86 | ;; 87 | status) 88 | status 89 | ;; 90 | *) 91 | echo "Usage : $0 [start|stop|restart|status|test]" 92 | ;; 93 | esac -------------------------------------------------------------------------------- /flume-logging-monitor-client/src/main/conf/developer-guide.md: -------------------------------------------------------------------------------- 1 | Log Monitor 2 | ========= 3 | 4 | logging client is used to capture the changed text and send them by avro client in the format of flume event. 5 | 6 | In the conf folder , there are configurations that you can work with . 7 | - *configuration.propreties* is the basic config , like the listening agent host and port ,the file that you want to watch the changes and send these to flume .etc . 8 | - *logHeader.json.sample* is the sample log header file that indicates the format of customized event header for certain watching file .The path of such header config can be specified by *monitor.event.headerConfigPath* in configuration.properties 9 | - everytime the client shuts down ,a hook method will automatically execute in order to stash the current status of the watching paths . When the client starts again ,it will first check the log status file ,if there is any , the status of the watching paths will be loaded and be further processed . 10 | 11 | To start the client: 12 | >./bin/logWatcherClient start 13 | 14 | Abstraction: 15 | The client consists of three phases . 16 | - Initialization phase 17 | 18 | Firstly, The client loads all static configuration from *configuration.properties*.If the the log status is pre stashed , it will load the log status file and map each log status to the current watching path .If the customized header file exists ,the client will also map the header to the current watching path. 19 | 20 | - Process phase 21 | 22 | During the process phase ,the client will firstly initialized a schedule thread to in order to do the job at fixed time interval and a thread pool with each thread assigning a task of checking the log changes on given path .When the changes occurs , the thread pool will return the result set of changes (List>) to the schedule thread ,eventually the schedule thread will initilize the avro client and send the event batch to the target agent . If the sending procedure is unsuccessful , the client will roll back the status of the watching path back to the status before the patches being sent. 23 | 24 | - Shutdown phase 25 | 26 | If the client is shut down by keyboard interruption or kill -5 . The status of current watching path will be logged to a file by a shut down hook method .Next time the client start , it will read and load the path status from that file . 27 | 28 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/java/com/shawn/logging/monitor/handler/EventSendingTest.java: -------------------------------------------------------------------------------- 1 | package com.shawn.logging.monitor.handler; 2 | 3 | import java.util.Date; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | import org.apache.flume.Event; 8 | import org.apache.flume.event.EventBuilder; 9 | import org.apache.http.protocol.HTTP; 10 | import org.springframework.stereotype.Service; 11 | 12 | import com.google.common.base.Charsets; 13 | import com.google.common.collect.ImmutableMap; 14 | import com.google.common.collect.Lists; 15 | import com.google.common.collect.Maps; 16 | import com.google.common.net.HttpHeaders; 17 | 18 | @Service 19 | public class EventSendingTest extends LogWatcher{ 20 | 21 | public EventSendingTest() throws Exception { 22 | } 23 | private String test = "TestTestTestTestTest"; 24 | @Override 25 | public void start() throws Exception { 26 | // logStatusMap = JsonFormatMapConfigHelper.loadJsonMap(logStatusMap, logStatusRecordPath,new TypeToken() {}); 27 | // headerConfigMap = JsonFormatMapConfigHelper.loadJsonMap(headerConfigMap, headerConfigPath,new TypeToken>() {}); 28 | logger.info("================================================"); 29 | logger.info("the event sending test started "); 30 | } 31 | 32 | @Override 33 | public void process() { 34 | List events = Lists.newLinkedList(); 35 | for (int i= 0 ; i < 10; i ++) { 36 | Map header = Maps.newHashMap(new ImmutableMap.Builder().put(HttpHeaders.LOCATION, IP) 37 | .put(FILE_NAME, "test").put(HTTP.CONTENT_TYPE, System.getProperty("file.encoding")) 38 | .put("domain","test").put("log_type","test").put("server_type","test").put("timestamp", new Date().getTime()+"") 39 | .build()); 40 | 41 | Event event = EventBuilder.withBody(test, Charsets.UTF_8, header); 42 | events.add(event); 43 | } 44 | 45 | logger.info("finished assemblng events: " + events.size()); 46 | client.init(); 47 | if(client.sendEventsToFlume(events)){ 48 | logger.info("Successfully findished sending events test"); 49 | }else{ 50 | logger.info("Events sending test failed...."); 51 | } 52 | logger.info("================================================"); 53 | client.cleanUp(); 54 | 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/java/com/shawn/logging/monitor/handler/JsonFormatMapConfigHelper.java: -------------------------------------------------------------------------------- 1 | package com.shawn.logging.monitor.handler; 2 | 3 | import java.io.File; 4 | import java.io.FileNotFoundException; 5 | import java.io.FileReader; 6 | import java.io.FileWriter; 7 | import java.io.IOException; 8 | import java.lang.reflect.Type; 9 | import java.util.Map; 10 | 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import com.google.common.base.Throwables; 15 | import com.google.common.collect.Maps; 16 | import com.google.common.reflect.TypeParameter; 17 | import com.google.common.reflect.TypeToken; 18 | import com.google.gson.Gson; 19 | import com.google.gson.GsonBuilder; 20 | import com.google.gson.JsonIOException; 21 | import com.google.gson.JsonSyntaxException; 22 | 23 | public final class JsonFormatMapConfigHelper { 24 | private JsonFormatMapConfigHelper(){} 25 | 26 | private static Gson gson = new GsonBuilder().enableComplexMapKeySerialization().create(); 27 | 28 | private static final Logger logger = LoggerFactory.getLogger(JsonFormatMapConfigHelper.class); 29 | 30 | public static void writeJsonMap(Map map,String path,TypeToken innerType){ 31 | FileWriter writer; 32 | try { 33 | @SuppressWarnings("serial") 34 | Type type = new TypeToken>() {}.where(new TypeParameter() {}, innerType).getType(); 35 | writer = new FileWriter(new File(path)); 36 | writer.append(gson.toJson(map,type)); 37 | writer.close(); 38 | } catch (IOException e) { 39 | logger.error("IO error when writing log status {}",Throwables.getStackTraceAsString(e)); 40 | } 41 | 42 | } 43 | 44 | public static Map loadJsonMap(Map map,String filepath,TypeToken innerType){ 45 | map = Maps.newConcurrentMap(); 46 | File file =new File(filepath); 47 | if(file.exists()){ 48 | try { 49 | @SuppressWarnings("serial") 50 | Type type = new TypeToken>() {}.where(new TypeParameter() {}, innerType).getType(); 51 | map = gson.fromJson(new FileReader(file), type); 52 | } catch (JsonIOException | JsonSyntaxException|FileNotFoundException e) { 53 | logger.error("json io error {}",Throwables.getStackTraceAsString(e)); 54 | } 55 | } 56 | return map; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/java/com/shawn/logging/monitor/handler/LogAttribute.java: -------------------------------------------------------------------------------- 1 | package com.shawn.logging.monitor.handler; 2 | 3 | import org.apache.commons.lang.builder.ToStringBuilder; 4 | 5 | public final class LogAttribute { 6 | 7 | private String logPath; 8 | private Long lastModifiedSize; 9 | private Long currentSize; 10 | private Long lastModifiedTime; 11 | private Long totalConsumed = 0L; 12 | 13 | 14 | public Long getTotalConsumed() { 15 | return totalConsumed; 16 | } 17 | 18 | public void setTotalConsumed(Long totalConsumed) { 19 | this.totalConsumed = totalConsumed; 20 | } 21 | 22 | public String getLogPath() { 23 | return logPath; 24 | } 25 | 26 | public void setLogPath(String logPath) { 27 | this.logPath = logPath; 28 | } 29 | 30 | public Long getLastModifiedSize() { 31 | return lastModifiedSize; 32 | } 33 | 34 | public void setLastModifiedSize(Long lastModifiedSize) { 35 | this.lastModifiedSize = lastModifiedSize; 36 | } 37 | 38 | public Long getCurrentSize() { 39 | return currentSize; 40 | } 41 | 42 | public void setCurrentSize(Long currentSize) { 43 | this.currentSize = currentSize; 44 | } 45 | 46 | public Long getLastModifiedTime() { 47 | return lastModifiedTime; 48 | } 49 | 50 | public void setLastModifiedTime(Long lastModifiedTime) { 51 | this.lastModifiedTime = lastModifiedTime; 52 | } 53 | 54 | public static class Builder { 55 | private LogAttribute instance = new LogAttribute(); 56 | 57 | public Builder logPath(String logPath) { 58 | instance.setLogPath(logPath); 59 | return this; 60 | } 61 | 62 | public Builder lastModifiedSize(Long lastModifiedSize) { 63 | instance.setLastModifiedSize(lastModifiedSize); 64 | return this; 65 | } 66 | 67 | public Builder currentSize(Long currentSize) { 68 | instance.setCurrentSize(currentSize); 69 | return this; 70 | } 71 | 72 | public Builder lastModifiedTime(Long lastModifiedTime) { 73 | instance.setLastModifiedTime(lastModifiedTime); 74 | return this; 75 | } 76 | 77 | 78 | public LogAttribute build() { 79 | return this.instance; 80 | } 81 | } 82 | 83 | @Override 84 | public String toString() { 85 | return ToStringBuilder.reflectionToString(this); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Log Monitor 2 | ========= 3 | 1. 准备 4 | - 拷贝flume-logging-monitor-client-*-bin.tar.gz 至每个需要监控的服务器，解压tar.gz压缩包。 5 | - 确保运行monitor的用户对监控文件有读的权限。 6 | - JRE1.7以上 7 | - 修改 ./bin/monitor 39行 8 | 9 | > java -classpath ${CLASSPATH} com.shawn.logging.monitor.handler.LogWatchRunScript & 10 | 11 | 由于传输时会有大量内存IO操作，请根据情况增大java运行时的heap size，buffer size等,如: 12 | 13 | > java -Xms512m -Xmx1024m -classpath ${CLASSPATH} com.shawn.logging.monitor.handler.LogWatchRunScript & 14 | 15 | 2. 配置 conf/configuration.properties 16 | - 配置flume agent的监听ip和端口。端口号参照flume-transfer中conf/collect-conf.properties。如,在flume-transfer中定义以下 17 | > collect_agent.sources.collect_source.type = avro 18 | > collect_agent.sources.collect_source.bind = 10.40.5.3 19 | > collect_agent.sources.collect_source.port = 41410 20 | 21 | 则在conf/configuration.properties 中配置 22 | > monitor.agent.host=10.40.5.3 23 | > monitor.agent.port=41410 24 | 25 | 26 | - 配置agent 连接参数，最大等待时间，最大尝试次数。每次检测到变化时，都会建立网络链接，将变化的log传输到flume-transfer，在此配置网络链接的超时时间和建立链接尝试次数。 27 | > monitor.client.timeout=10000(单位为毫秒) 28 | > monitor.client.maxAttempts=2 29 | 30 | - 配置检查文件变化的时间间隔和监控目录(检测多个log文件用 , 号间隔)，如: 31 | > monitor.client.schedule.delay=2000(单位为毫秒) 32 | > monitor.client.schedule.watchpaths=/var/log/nginx/www.shawn.com.access_log,/var/log/nginx/www.shawn.com.error_log 33 | 34 | 以上配置为每个两秒，查看www.shawn.com.access_log和www.shawn.com.error_log 是否变化。 35 | 36 | 37 | - 配置头文件位置和logStatus文件位置，如: 38 | > monitor.event.headerConfigPath=./conf/logHeader.json 39 | > monitor.client.logStatusRecordPath=./conf/log_record.json 40 | （具体logHeader.json和log_record.json文件格式请参照 *3.头文件和log status文件格式*） 41 | 42 | - 配置每次发送的eventBatch中单个event的最大大小。event为单个log改变部分的包。eventBatch由多个event组成，eventBatch的大小取决于有几个log变化，则eventBatch包含几个event。此参数限定每次发送的eventBatch中单个event的最大大小。 43 | > monitor.client.event.size = 5242880 (单位为byte) 44 | 45 | 46 | 3. 头文件和log status文件格式 47 | * 头文件 48 | - 用于对特定文件配置特殊属性，格式为以被监控文件全路径为key的map结构，value为方便flume-transfer做multiplexing以及在hdfs中根据log属性建立目录结构 49 | - 如: "/var/log/nginx/www.shawn.com.access_log": { 50 | "domain": "www", //domain可为www,job等 51 | "log_type": "access",//可为access,error等 52 | "server_type":"nginx" 53 | };"/var/log/nginx/www.shawn.com.access_log"为监控路径，domain,log_type,server_type为自定义的属性，可参照logHeader.json.sample。 54 | - 自定义的属性会以event header的方式发送给agent，以进行后续处理 55 | 56 | * log status 57 | - 在monitor正常停止时，会以json格式记录当时文件读取的大小 和其他状态 58 | 59 | 4. 测试 60 | * 运行网络测试，测试对应agent是否配置正常 61 | > ./bin/monitor test 62 | 63 | 查看logs/client.log,如出现*Successfully finishing sending events test*, 64 | 则网络以及配置正常 65 | 66 | 4. 启动monitor 67 | - 频繁的读取和传输都在内存中进行，请适量增大java运行时里的heap size 68 | >./bin/monitor start 69 | 70 | 运行jps，此时应该有LogWatchRunScript 的java进程在运行，启动成功 71 | 72 | 73 | 5. 停止monitor 74 | > ./bin/monitor stop 75 | 76 | 77 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/java/com/shawn/logging/monitor/handler/LogMonitorClient.java: -------------------------------------------------------------------------------- 1 | package com.shawn.logging.monitor.handler; 2 | 3 | import java.util.List; 4 | import java.util.Properties; 5 | 6 | import org.apache.flume.Event; 7 | import org.apache.flume.EventDeliveryException; 8 | import org.apache.flume.FlumeException; 9 | import org.apache.flume.api.RpcClient; 10 | import org.apache.flume.api.RpcClientConfigurationConstants; 11 | import org.apache.flume.api.RpcClientFactory; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | import org.springframework.beans.factory.annotation.Value; 15 | import org.springframework.stereotype.Service; 16 | 17 | import com.google.common.base.Throwables; 18 | 19 | @Service 20 | public class LogMonitorClient { 21 | 22 | private RpcClient client; 23 | 24 | @Value("#{config['monitor.agent.host']}") 25 | private String hostname; 26 | 27 | @Value("#{config['monitor.agent.port']}") 28 | private String port; 29 | 30 | @Value("#{config['monitor.client.timeout']}") 31 | private Integer timeout; 32 | 33 | @Value("#{config['monitor.client.maxAttempts']}") 34 | private Integer maxAttempts; 35 | 36 | final Logger logger = LoggerFactory.getLogger(LogMonitorClient.class); 37 | 38 | public void init() { 39 | Properties properties = new Properties(); 40 | properties.setProperty(RpcClientConfigurationConstants.CONFIG_CONNECT_TIMEOUT, String.valueOf(timeout)); 41 | properties.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS, "h1"); 42 | properties.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + "h1",hostname + ":" + port); 43 | properties.setProperty(RpcClientConfigurationConstants.CONFIG_MAX_ATTEMPTS, maxAttempts+""); 44 | try{ 45 | this.client = RpcClientFactory.getInstance(properties); 46 | 47 | }catch(Exception e){ 48 | //e.printStackTrace(); 49 | logger.error("can not init client instance: " + Throwables.getStackTraceAsString(e)); 50 | throw new FlumeException("connection refused "); 51 | } 52 | 53 | } 54 | 55 | public boolean sendEventsToFlume(List events) { 56 | if (events ==null || events.size() == 0) { 57 | return true; 58 | } 59 | try { 60 | client.appendBatch(events); 61 | logger.info("finished sending to flume agent, total events batch number is {}",events.size()); 62 | } catch (EventDeliveryException e) { 63 | logger.error("Event delivery exception: {}",Throwables.getStackTraceAsString(e)); 64 | cleanUp(); 65 | return false; 66 | } 67 | return true; 68 | } 69 | 70 | public void cleanUp() { 71 | try{ 72 | client.close(); 73 | }catch(FlumeException e){ 74 | logger.error("cannot shut down the clinet {}", Throwables.getStackTraceAsString(e)); 75 | } 76 | } 77 | 78 | 79 | public boolean isActive() { 80 | if (client!=null && client.isActive()) { 81 | return true; 82 | } 83 | return false; 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | com.shawn 6 | flume-logging-monitor 7 | 0.1-SNAPSHOT 8 | pom 9 | logging-monitor 10 | http://maven.apache.org 11 | 12 | UTF-8 13 | 1.8 14 | 1.8 15 | 1.8 16 | 4.3.3.RELEASE 17 | 1.7.5 18 | 1.7.0 19 | 20 | 21 | flume-logging-monitor-client 22 | flume-logging-monitor-handler 23 | 24 | 25 | 26 | 27 | 28 | junit 29 | junit 30 | 4.12 31 | 32 | 33 | org.apache.flume 34 | flume-ng-core 35 | ${org.apache.flume.version} 36 | 37 | 38 | org.slf4j 39 | slf4j-api 40 | ${org.slf4j.version} 41 | 42 | 43 | org.slf4j 44 | slf4j-log4j12 45 | ${org.slf4j.version} 46 | 47 | 48 | commons-logging 49 | commons-logging 50 | 1.2 51 | 52 | 53 | org.springframework 54 | spring-core 55 | ${org.springframework.version} 56 | 57 | 58 | commons-logging 59 | commons-logging 60 | 61 | 62 | org.springframework 63 | spring-beans 64 | ${org.springframework.version} 65 | 66 | 67 | org.springframework 68 | spring-context 69 | ${org.springframework.version} 70 | 71 | 72 | com.google.code.gson 73 | gson 74 | 2.7 75 | 76 | 77 | com.google.guava 78 | guava 79 | 19.0 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/java/com/shawn/App.java: -------------------------------------------------------------------------------- 1 | package com.shawn; 2 | 3 | import java.net.InetAddress; 4 | import java.net.UnknownHostException; 5 | import java.nio.charset.Charset; 6 | import java.nio.file.Files; 7 | import java.nio.file.Path; 8 | import java.nio.file.Paths; 9 | import java.util.Date; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | import org.apache.flume.Event; 14 | import org.apache.flume.EventDeliveryException; 15 | import org.apache.flume.api.RpcClient; 16 | import org.apache.flume.api.RpcClientFactory; 17 | import org.apache.flume.event.EventBuilder; 18 | 19 | import com.google.common.collect.Lists; 20 | import com.google.common.collect.Maps; 21 | 22 | public class App { 23 | public static void main(String[] args) throws Exception { 24 | MyRpcClientFacade client = new MyRpcClientFacade(); 25 | client.init("10.40.5.3", 41410); 26 | for(int k=0;k<50;k++){ 27 | client.sendDataToFlume(k); 28 | System.out.println("--------111231231231================="); 29 | } 30 | client.cleanUp(); 31 | } 32 | } 33 | 34 | class MyRpcClientFacade { 35 | 36 | private RpcClient client; 37 | 38 | private String hostname; 39 | 40 | private int port; 41 | 42 | public void init(String hostname, int port) { 43 | this.hostname = hostname; 44 | this.port = port; 45 | 46 | // Properties props = new Properties(); 47 | // props.put("client.type", "default_loadbalance"); 48 | // props.put("hosts", "h1 h2"); 49 | // String host1 = "localhost:41411"; 50 | // String host2 = "localhost:41410"; 51 | // props.put("hosts.h1", host1); 52 | // props.put("hosts.h2", host2); 53 | 54 | 55 | this.client = RpcClientFactory.getDefaultInstance(hostname, port); 56 | 57 | } 58 | 59 | public void sendDataToFlume(int k) throws UnknownHostException { 60 | 61 | List list = Lists.newArrayList(); 62 | StringBuilder builder = new StringBuilder(); 63 | for(int i=0;i<20000;i++){ 64 | String data = "10.40.5.253 [27/Feb/2014:15:43:49 +0800] \"GET /career/index?from=shawn HTTP/1.1\" 302 448 \"http://www.shawn.com/home\" \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36\""+i + "=="+k; 65 | builder.append(data); 66 | } 67 | 68 | try { 69 | Event event = EventBuilder.withBody(builder.toString(), Charset.forName("UTF-8")); 70 | Map headerMap = Maps.newHashMap(); 71 | headerMap.put("server_type","nginx"); 72 | headerMap.put("server_host","DEFAULT"); 73 | headerMap.put("filename","haproxy.log"); 74 | headerMap.put("timestamp",new Date().getTime()+""); 75 | event.setHeaders(headerMap); 76 | list.add(event); 77 | client.appendBatch(list); 78 | } catch (EventDeliveryException e) { 79 | client.close(); 80 | client = null; 81 | client = RpcClientFactory.getDefaultInstance(hostname, port); 82 | } 83 | } 84 | 85 | public void sendBigDataToFlume(String file,String header) throws Exception { 86 | Path path = Paths.get(file); 87 | Event event = EventBuilder.withBody(Files.readAllBytes(path)); 88 | Map headerMap = Maps.newHashMap(); 89 | headerMap.put("server_host",header); 90 | headerMap.put("timestamp",new Date().getTime()+""); 91 | headerMap.put("filename",path.getFileName().toString()); 92 | InetAddress addr = InetAddress.getLocalHost(); 93 | String ip=addr.getHostAddress(); 94 | headerMap.put("host",ip); 95 | 96 | event.setHeaders(headerMap); 97 | 98 | try { 99 | client.append(event); 100 | } catch (EventDeliveryException e) { 101 | client.close(); 102 | client = null; 103 | client = RpcClientFactory.getDefaultInstance(hostname, port); 104 | } 105 | } 106 | 107 | public void cleanUp() { 108 | // Close the RPC connection 109 | client.close(); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/java/com/shawn/logging/monitor/handler/LogWatcher.java: -------------------------------------------------------------------------------- 1 | package com.shawn.logging.monitor.handler; 2 | 3 | import java.net.InetAddress; 4 | import java.net.NetworkInterface; 5 | import java.net.UnknownHostException; 6 | import java.util.Enumeration; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.Set; 10 | import java.util.concurrent.ExecutorService; 11 | import java.util.concurrent.ScheduledExecutorService; 12 | 13 | import org.apache.http.protocol.HTTP; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | import org.springframework.beans.factory.annotation.Autowired; 17 | import org.springframework.beans.factory.annotation.Value; 18 | 19 | import com.google.common.collect.ImmutableMap; 20 | import com.google.common.collect.Maps; 21 | import com.google.common.net.HttpHeaders; 22 | 23 | public abstract class LogWatcher { 24 | 25 | public ExecutorService executorService; 26 | public ScheduledExecutorService scheduledExecutorService; 27 | 28 | 29 | 30 | @Value("#{config['monitor.client.timeout']}") 31 | public Integer timeout; 32 | 33 | @Value("#{config['monitor.client.schedule.delay']}") 34 | public int delay; 35 | 36 | @Value("#{config['monitor.client.schedule.watchpaths'].split(',')}") 37 | public List watchFilePaths; 38 | 39 | @Value("#{config['monitor.client.logStatusRecordPath']}") 40 | public String logStatusRecordPath; 41 | 42 | @Value("#{config['monitor.event.headerConfigPath']}") 43 | public String headerConfigPath; 44 | 45 | @Value("#{config['monitor.log.suffix.date']}") 46 | public String dateFormat; 47 | 48 | @Value("#{config['monitor.client.event.size']}") 49 | public int eventMaxSize; 50 | 51 | Map logStatusMap ; 52 | Map> headerConfigMap; 53 | 54 | 55 | @Autowired 56 | LogMonitorClient client; 57 | 58 | 59 | final Logger logger = LoggerFactory.getLogger(LogWatcher.class); 60 | static final String FILE_NAME = "file_name"; 61 | static final String CONTENT_TYPE = System.getProperty("file.encoding"); 62 | static final String FILE_SEPARATOR = System.getProperty("file.separator"); 63 | final String IP = getLocalHostLANAddress().getHostAddress(); 64 | public LogWatcher() throws Exception {} 65 | 66 | public Map buildHeader(String path) { 67 | Map header = Maps.newHashMap(new ImmutableMap.Builder().put(HttpHeaders.LOCATION, IP) 68 | .put(FILE_NAME, path.substring(path.lastIndexOf(FILE_SEPARATOR) + 1)).put(HTTP.CONTENT_TYPE, CONTENT_TYPE) 69 | .build()); 70 | if (headerConfigMap.containsKey(path)) { 71 | Map logSpecifiedAttr = headerConfigMap.get(path); 72 | Set keys = logSpecifiedAttr.keySet(); 73 | for (String key : keys) { 74 | header.put(key, logSpecifiedAttr.get(key)); 75 | } 76 | } 77 | headerConfigMap.put(path, header); 78 | return header; 79 | } 80 | 81 | public void rollBackLogStatus() { 82 | logger.warn("roll back file status ------"); 83 | Set logPaths = logStatusMap.keySet(); 84 | for (String log : logPaths) { 85 | LogAttribute logAttribute = logStatusMap.get(log); 86 | long current = logAttribute.getLastModifiedSize(); 87 | logAttribute.setCurrentSize(current); 88 | } 89 | } 90 | 91 | public abstract void start() throws Exception; 92 | 93 | public abstract void process(); 94 | 95 | public void shutdown() { 96 | executorService.shutdown(); 97 | scheduledExecutorService.shutdown(); 98 | } 99 | 100 | private InetAddress getLocalHostLANAddress() throws UnknownHostException { 101 | try { 102 | InetAddress candidateAddress = null; 103 | // Iterate all NICs (network interface cards)... 104 | for (Enumeration ifaces = NetworkInterface.getNetworkInterfaces(); ifaces.hasMoreElements();) { 105 | NetworkInterface iface = (NetworkInterface) ifaces.nextElement(); 106 | // Iterate all IP addresses assigned to each card... 107 | for (Enumeration inetAddrs = iface.getInetAddresses(); inetAddrs.hasMoreElements();) { 108 | InetAddress inetAddr = (InetAddress) inetAddrs.nextElement(); 109 | if (!inetAddr.isLoopbackAddress()) { 110 | 111 | if (inetAddr.isSiteLocalAddress()) { 112 | // Found non-loopback site-local address. Return it immediately... 113 | return inetAddr; 114 | } 115 | else if (candidateAddress == null) { 116 | // Found non-loopback address, but not necessarily site-local. 117 | // Store it as a candidate to be returned if site-local address is not subsequently found... 118 | candidateAddress = inetAddr; 119 | // Note that we don't repeatedly assign non-loopback non-site-local addresses as candidates, 120 | // only the first. For subsequent iterations, candidate will be non-null. 121 | } 122 | } 123 | } 124 | } 125 | if (candidateAddress != null) { 126 | // We did not find a site-local address, but we found some other non-loopback address. 127 | // Server might have a non-site-local address assigned to its NIC (or it might be running 128 | // IPv6 which deprecates the "site-local" concept). 129 | // Return this non-loopback candidate address... 130 | return candidateAddress; 131 | } 132 | // At this point, we did not find a non-loopback address. 133 | // Fall back to returning whatever InetAddress.getLocalHost() returns... 134 | InetAddress jdkSuppliedAddress = InetAddress.getLocalHost(); 135 | if (jdkSuppliedAddress == null) { 136 | throw new UnknownHostException("The JDK InetAddress.getLocalHost() method unexpectedly returned null."); 137 | } 138 | return jdkSuppliedAddress; 139 | } 140 | catch (Exception e) { 141 | UnknownHostException unknownHostException = new UnknownHostException("Failed to determine LAN address: " + e); 142 | unknownHostException.initCause(e); 143 | throw unknownHostException; 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/java/com/shawn/logging/monitor/handler/LogWatcherBySchedule.java: -------------------------------------------------------------------------------- 1 | package com.shawn.logging.monitor.handler; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.nio.ByteBuffer; 6 | import java.nio.channels.SeekableByteChannel; 7 | import java.nio.file.Files; 8 | import java.nio.file.Paths; 9 | import java.nio.file.StandardOpenOption; 10 | import java.util.Date; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.concurrent.Callable; 14 | import java.util.concurrent.ExecutionException; 15 | import java.util.concurrent.ExecutorService; 16 | import java.util.concurrent.Executors; 17 | import java.util.concurrent.Future; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import org.apache.flume.Event; 21 | import org.apache.flume.FlumeException; 22 | import org.apache.flume.event.EventBuilder; 23 | import org.springframework.stereotype.Service; 24 | 25 | import com.google.common.base.Throwables; 26 | import com.google.common.collect.Lists; 27 | import com.google.common.reflect.TypeToken; 28 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 29 | 30 | @Service 31 | public class LogWatcherBySchedule extends LogWatcher{ 32 | 33 | 34 | 35 | public LogWatcherBySchedule() throws Exception { 36 | super(); 37 | } 38 | 39 | public void start() throws Exception { 40 | logStatusMap = JsonFormatMapConfigHelper.loadJsonMap(logStatusMap, logStatusRecordPath,new TypeToken() {}); 41 | headerConfigMap = JsonFormatMapConfigHelper.loadJsonMap(headerConfigMap, headerConfigPath,new TypeToken>() {}); 42 | scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("log-watcher-sechdule").build()); 43 | executorService = Executors.newFixedThreadPool(watchFilePaths.size(), new ThreadFactoryBuilder().setNameFormat("file-change-thread").build()); 44 | logger.info("the log watcher started successfully "); 45 | } 46 | 47 | public void process() { 48 | List fileChangeTasks = Lists.newArrayListWithCapacity(watchFilePaths.size()); 49 | for (String path : watchFilePaths) { 50 | File file = new File(path); 51 | LogAttribute logAttribute = logStatusMap.containsKey(path)?logStatusMap.get(path):new LogAttribute.Builder() 52 | .logPath(path) 53 | .lastModifiedTime(file.lastModified()) 54 | .lastModifiedSize(0L) 55 | .currentSize(file.length()).build(); 56 | logStatusMap.put(logAttribute.getLogPath(), logAttribute); 57 | FileChangeCallable fileChangeRunnable = new FileChangeCallable(file,logAttribute, buildHeader(path)); 58 | fileChangeTasks.add(fileChangeRunnable); 59 | } 60 | scheduledExecutorService.scheduleAtFixedRate(new ScheduledFileChangeJob(fileChangeTasks, executorService, client), 0, delay, TimeUnit.MILLISECONDS); 61 | Runtime.getRuntime().addShutdownHook(new Thread(){ 62 | public void run() { 63 | logger.info("storing log processing to the temp file"); 64 | JsonFormatMapConfigHelper.writeJsonMap(logStatusMap, logStatusRecordPath,new TypeToken() {}); 65 | } 66 | }); 67 | } 68 | 69 | 70 | private final class ScheduledFileChangeJob implements Runnable { 71 | private List tasks; 72 | private ExecutorService executorService; 73 | private LogMonitorClient client; 74 | 75 | public ScheduledFileChangeJob(List tasks, ExecutorService executorService, LogMonitorClient client) { 76 | this.tasks = tasks; 77 | this.executorService = executorService; 78 | this.client = client; 79 | } 80 | 81 | public synchronized void run() { 82 | List events = null; 83 | try { 84 | List> results = executorService.invokeAll(tasks); 85 | events = Lists.newLinkedList(); 86 | for (Future e : results) { 87 | if (e.get() != null) 88 | events.add(e.get()); 89 | } 90 | if(events.size() <1 ) return ; 91 | } catch (InterruptedException | ExecutionException e) { 92 | logger.error(" thread exception : {}",Throwables.getStackTraceAsString(e )); 93 | } 94 | try { 95 | client.init(); 96 | } catch (FlumeException e) { 97 | rollBackLogStatus(); 98 | logger.error("cannot establish connection to flume agent , please check the network .{}",Throwables.getStackTraceAsString(e)); 99 | client.cleanUp(); 100 | return; 101 | } 102 | logger.info("established the connection , prepare to send the event batch"); 103 | if (!client.sendEventsToFlume(events)) { 104 | rollBackLogStatus(); 105 | } 106 | client.cleanUp(); 107 | //} 108 | } 109 | 110 | } 111 | 112 | 113 | private final class FileChangeCallable implements Callable { 114 | private File file; 115 | private LogAttribute logAttribute; 116 | private ByteBuffer byteBuffer; 117 | //private long total ; 118 | private Map headers; 119 | 120 | public FileChangeCallable(File file,LogAttribute logAttribute, Map headers) { 121 | this.file = file; 122 | this.headers = headers; 123 | this.logAttribute = logAttribute; 124 | } 125 | 126 | public Event call() { 127 | Event event = null; 128 | long fileCurrentSize; 129 | if (file.exists() && ((fileCurrentSize = file.length()) > 0) && (fileCurrentSize != logAttribute.getCurrentSize())) { 130 | logger.info(String.format("file %s changed from %s to %s ", file.getName(), logAttribute.getCurrentSize(), fileCurrentSize)); 131 | if (fileCurrentSize < logAttribute.getLastModifiedSize()) { 132 | logAttribute.setCurrentSize(0L); 133 | } 134 | long lastModifiedSize = logAttribute.getCurrentSize(); 135 | 136 | long rawReadLength = fileCurrentSize - logAttribute.getCurrentSize(); 137 | long actualReadLength = (rawReadLength > eventMaxSize)? eventMaxSize: rawReadLength; 138 | 139 | headers.put("timestamp", String.valueOf(new Date().getTime())); 140 | byte[] data = assemblingEvent(logAttribute.getCurrentSize(),actualReadLength); 141 | event = EventBuilder.withBody(data, headers); 142 | logAttribute.setLastModifiedTime(file.lastModified()); 143 | logAttribute.setLastModifiedSize(lastModifiedSize); 144 | logAttribute.setCurrentSize(lastModifiedSize+actualReadLength); 145 | long consumed = logAttribute.getTotalConsumed() + data.length; 146 | logAttribute.setTotalConsumed(consumed); 147 | } 148 | return event; 149 | } 150 | 151 | private byte[] assemblingEvent(long currentPosition,long readLength){ 152 | byte[] data = new byte[0]; 153 | 154 | if(readLength < eventMaxSize){ 155 | byteBuffer = ByteBuffer.allocate((int)readLength); 156 | }else{ 157 | byteBuffer = ByteBuffer.allocate(eventMaxSize); 158 | } 159 | try(SeekableByteChannel channel = Files.newByteChannel(Paths.get(file.getPath()), StandardOpenOption.READ);){ 160 | channel.position(currentPosition); 161 | channel.read(byteBuffer); 162 | data = byteBuffer.array(); 163 | byteBuffer.flip(); 164 | byteBuffer.rewind(); 165 | byteBuffer.clear(); 166 | channel.close(); 167 | }catch(IOException e){ 168 | logger.error("Io error: {}",Throwables.getStackTraceAsString(e)); 169 | } 170 | byteBuffer =null; 171 | return data; 172 | } 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /flume-logging-monitor-handler/src/main/java/com/shawn/logging/monitor/handler/LogWatcherByWatchEvent.java: -------------------------------------------------------------------------------- 1 | package com.shawn.logging.monitor.handler; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.nio.ByteBuffer; 6 | import java.nio.channels.SeekableByteChannel; 7 | import java.nio.file.ClosedWatchServiceException; 8 | import java.nio.file.FileSystem; 9 | import java.nio.file.FileSystems; 10 | import java.nio.file.Files; 11 | import java.nio.file.Path; 12 | import java.nio.file.Paths; 13 | import java.nio.file.StandardOpenOption; 14 | import java.nio.file.StandardWatchEventKinds; 15 | import java.nio.file.WatchEvent; 16 | import java.nio.file.WatchKey; 17 | import java.nio.file.WatchService; 18 | import java.util.List; 19 | import java.util.Map; 20 | import java.util.concurrent.ConcurrentHashMap; 21 | import java.util.concurrent.Executors; 22 | import java.util.concurrent.TimeUnit; 23 | 24 | import org.apache.flume.Event; 25 | import org.apache.flume.FlumeException; 26 | import org.apache.flume.event.EventBuilder; 27 | import org.springframework.stereotype.Service; 28 | 29 | import com.google.common.base.Throwables; 30 | import com.google.common.collect.Lists; 31 | import com.google.common.reflect.TypeToken; 32 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 33 | 34 | @Service 35 | public class LogWatcherByWatchEvent extends LogWatcher{ 36 | public LogWatcherByWatchEvent() throws Exception { 37 | } 38 | 39 | public void start() throws Exception { 40 | logStatusMap = JsonFormatMapConfigHelper.loadJsonMap(logStatusMap, logStatusRecordPath, new TypeToken() {}); 41 | headerConfigMap = JsonFormatMapConfigHelper.loadJsonMap(headerConfigMap, headerConfigPath, new TypeToken>() {}); 42 | executorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("file-change-thread").build()); 43 | logger.info("the log watcher started successfully "); 44 | } 45 | 46 | public void process() { 47 | for (String path : watchFilePaths) { 48 | File file = new File(path); 49 | LogAttribute logAttribute = logStatusMap.containsKey(path) ? logStatusMap.get(path) : new LogAttribute.Builder().logPath(path) 50 | .lastModifiedTime(file.lastModified()).lastModifiedSize(0L).currentSize(file.length()).build(); 51 | logStatusMap.put(logAttribute.getLogPath(), logAttribute); 52 | buildHeader(path); 53 | } 54 | executorService.submit(new WatcherRunnable(watchFilePaths,delay)); 55 | Runtime.getRuntime().addShutdownHook(new Thread() { 56 | public void run() { 57 | executorService.shutdown(); 58 | logger.info("storing log processing to the temp file"); 59 | JsonFormatMapConfigHelper.writeJsonMap(logStatusMap, logStatusRecordPath, new TypeToken() { 60 | }); 61 | ; 62 | } 63 | }); 64 | } 65 | 66 | 67 | 68 | 69 | 70 | private final class WatcherRunnable implements Runnable { 71 | 72 | final FileSystem fs = FileSystems.getDefault(); 73 | private WatchService ws; 74 | 75 | private List filenames; 76 | final Map keys = new ConcurrentHashMap<>(); 77 | int delay ; 78 | public WatcherRunnable(List paths,int delay) { 79 | try{ 80 | ws = fs.newWatchService(); 81 | }catch(IOException e){ 82 | logger.error("error on creating watch service,{} ",Throwables.getStackTraceAsString(e)); 83 | } 84 | filenames = Lists.newArrayListWithCapacity(paths.size()); 85 | for (String path : paths) { 86 | reg(fs.getPath(path.substring(0,path.lastIndexOf(System.getProperty("file.separator")))), keys, ws); 87 | filenames.add(path); 88 | } 89 | this.delay = delay; 90 | } 91 | 92 | public void run() { 93 | logger.info("watcher start..."); 94 | while (Thread.interrupted() == false) { 95 | WatchKey key; 96 | try { 97 | key = ws.poll(delay, TimeUnit.MILLISECONDS); 98 | logger.debug("wait millisec: " + delay); 99 | } catch (InterruptedException | ClosedWatchServiceException e) { 100 | logger.error(Throwables.getStackTraceAsString(e)); 101 | break; 102 | } 103 | if (key != null) { 104 | List events = Lists.newArrayList(); 105 | Path path = keys.get(key); 106 | for (WatchEvent i : key.pollEvents()) { 107 | WatchEvent event = cast(i); 108 | WatchEvent.Kind kind = event.kind(); 109 | Path name = event.context(); 110 | Path fullPath = path.resolve(name); 111 | logger.info(String.format("changed file : %s: %s%n", kind.name(), fullPath)); 112 | if(kind.equals(StandardWatchEventKinds.ENTRY_MODIFY) && filenames.contains(fullPath.toString())){ 113 | logger.info(String.format("event : %s %d %n", fullPath,fullPath.toFile().length())); 114 | events.add(transformChangedFileToEvent(fullPath)); 115 | } 116 | } if(!events.isEmpty()) 117 | processingEvent(events); 118 | if (key.reset() == false) { 119 | logger.error("{} is invalid ",key); 120 | keys.remove(key); 121 | if (keys.isEmpty()) { 122 | break; 123 | } 124 | } 125 | } 126 | } 127 | } 128 | 129 | 130 | void reg(Path path, Map keys, WatchService ws){ 131 | try{ 132 | WatchKey key = path.register(ws, StandardWatchEventKinds.ENTRY_MODIFY); 133 | keys.put(key, path); 134 | }catch(Exception e){ 135 | e.printStackTrace(); 136 | } 137 | } 138 | 139 | @SuppressWarnings("unchecked") 140 | WatchEvent cast(WatchEvent event) { 141 | return (WatchEvent) event; 142 | } 143 | 144 | Event transformChangedFileToEvent(Path path){ 145 | LogAttribute logAttribute = logStatusMap.get(path.toString()); 146 | File file = path.toFile(); 147 | ByteBuffer byteBuffer; 148 | Event event = null; 149 | if (file.length() < logAttribute.getLastModifiedSize()) { 150 | logAttribute.setCurrentSize(0L); 151 | } 152 | try { 153 | int readLength = (int) (file.length() - logAttribute.getCurrentSize()); 154 | 155 | SeekableByteChannel seekableByteChannel = Files.newByteChannel(Paths.get(file.getPath()), StandardOpenOption.READ); 156 | seekableByteChannel.position(logAttribute.getCurrentSize()); 157 | byteBuffer = ByteBuffer.allocate(readLength); 158 | while (seekableByteChannel.read(byteBuffer) > 0) { 159 | byteBuffer.flip(); 160 | event = EventBuilder.withBody(byteBuffer.array(), headerConfigMap.get(path.toString())); 161 | byteBuffer.clear(); 162 | } 163 | logAttribute.setLastModifiedTime(file.lastModified()); 164 | logAttribute.setLastModifiedSize(logAttribute.getCurrentSize()); 165 | logAttribute.setCurrentSize(file.length()); 166 | } catch (IOException e) { 167 | logger.error(Throwables.getStackTraceAsString(e)); 168 | } 169 | return event; 170 | } 171 | 172 | 173 | private void processingEvent(List events){ 174 | synchronized (LogMonitorClient.class) { 175 | try { 176 | client.init(); 177 | } catch (FlumeException e) { 178 | // shutdown(); 179 | rollBackLogStatus(); 180 | logger.error("cannot establish connection to flume agent , please check the network ."); 181 | return; 182 | } 183 | logger.info("established the connection , prepare to send the event batch"); 184 | if (!client.sendEventsToFlume(events)) { 185 | rollBackLogStatus(); 186 | } 187 | client.cleanUp(); 188 | } 189 | } 190 | } 191 | 192 | } 193 | --------------------------------------------------------------------------------