├── .classpath
├── .gitignore
├── .project
├── .settings
    ├── org.eclipse.core.resources.prefs
    ├── org.eclipse.jdt.core.prefs
    └── org.eclipse.m2e.core.prefs
├── README.md
├── pom.xml
└── src
    ├── main
        └── java
        │   └── com
        │       └── jinoos
        │           └── flume
        │               ├── DirPattern.java
        │               ├── DirectoryTailParserModulable.java
        │               ├── DirectoryTailSource.java
        │               ├── FileSet.java
        │               ├── MultiLineParserModule.java
        │               ├── SingleLineParserModule.java
        │               └── ThreadSafeSimpleDateFormat.java
    └── test
        └── java
            └── com
                └── jinoos
                    └── flume_ng_extends
                        └── AppTest.java


/.classpath:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <classpath>
 3 | 	<classpathentry kind="src" output="target/classes" path="src/main/java">
 4 | 		<attributes>
 5 | 			<attribute name="optional" value="true"/>
 6 | 			<attribute name="maven.pomderived" value="true"/>
 7 | 		</attributes>
 8 | 	</classpathentry>
 9 | 	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
10 | 		<attributes>
11 | 			<attribute name="optional" value="true"/>
12 | 			<attribute name="maven.pomderived" value="true"/>
13 | 		</attributes>
14 | 	</classpathentry>
15 | 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5">
16 | 		<attributes>
17 | 			<attribute name="maven.pomderived" value="true"/>
18 | 		</attributes>
19 | 	</classpathentry>
20 | 	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
21 | 		<attributes>
22 | 			<attribute name="maven.pomderived" value="true"/>
23 | 		</attributes>
24 | 	</classpathentry>
25 | 	<classpathentry kind="output" path="target/classes"/>
26 | </classpath>
27 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | 


--------------------------------------------------------------------------------
/.project:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <projectDescription>
 3 | 	<name>com.jinoos.flume-ng-extends-0.0.1-SNAPSHOT</name>
 4 | 	<comment></comment>
 5 | 	<projects>
 6 | 	</projects>
 7 | 	<buildSpec>
 8 | 		<buildCommand>
 9 | 			<name>org.eclipse.jdt.core.javabuilder</name>
10 | 			<arguments>
11 | 			</arguments>
12 | 		</buildCommand>
13 | 		<buildCommand>
14 | 			<name>org.eclipse.m2e.core.maven2Builder</name>
15 | 			<arguments>
16 | 			</arguments>
17 | 		</buildCommand>
18 | 	</buildSpec>
19 | 	<natures>
20 | 		<nature>org.eclipse.jdt.core.javanature</nature>
21 | 		<nature>org.eclipse.m2e.core.maven2Nature</nature>
22 | 	</natures>
23 | </projectDescription>
24 | 


--------------------------------------------------------------------------------
/.settings/org.eclipse.core.resources.prefs:
--------------------------------------------------------------------------------
1 | eclipse.preferences.version=1
2 | encoding//src/main/java=UTF-8
3 | encoding//src/test/java=UTF-8
4 | encoding/<project>=UTF-8
5 | 


--------------------------------------------------------------------------------
/.settings/org.eclipse.jdt.core.prefs:
--------------------------------------------------------------------------------
1 | eclipse.preferences.version=1
2 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
3 | org.eclipse.jdt.core.compiler.compliance=1.5
4 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
5 | org.eclipse.jdt.core.compiler.source=1.5
6 | 


--------------------------------------------------------------------------------
/.settings/org.eclipse.m2e.core.prefs:
--------------------------------------------------------------------------------
1 | activeProfiles=
2 | eclipse.preferences.version=1
3 | resolveWorkspaceProjects=true
4 | version=1
5 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | flume-ng-extends
 2 | ================
 3 | 
 4 | Source of Flume NG for tailing files in a directory
 5 | 
 6 | Configuration
 7 | =============
 8 | | Property Name | Default | Description |
 9 | | ------------- | :-----: | :---------- |
10 | | Channels | - |  |
11 | | Type | - | com.jinoos.flume.DirectoryTailSource |
12 | | Type | - | com.jinoos.flume.DirectoryTailSource |
13 | | dirs | - | Nick of directories, it's such as list of what directories are monitored |
14 | | parser | com.jinoos.flume.SingleLineParserModule | Implemented class path of DirectoryTailParserModulable. |
15 | | dirs.NICK.path | - | Directory path |
16 | | dirs.NICK.file-pattern | - | Pattern of target file(s), ex) '^(.*)$' is for all files |
17 | 
18 | For parser, there is two module classes what implemented DirectoryTailParserModulable
19 | * SingleLineParserModule
20 | * MultiLineParserModule 
21 | 
22 | * Example for SingleLineParserModule
23 | ```
24 | agent.sources = dirMon
25 | agent.sources.dirMon.type = com.jinoos.flume.DirectoryTailSource
26 | agent.sources.dirMon.parser = com.jinoos.flume.SingleLineParserModule
27 | agent.sources.dirMon.dirs = tmpDir varLogDir
28 | agent.sources.dirMon.dirs.tmpDir.path = /tmp
29 | agent.sources.dirMon.dirs.tmpDir.file-pattern = ^(message)$       # /var/log/message
30 | agent.sources.dirMon.dirs.varLogDir.path = /var/log
31 | agent.sources.dirMon.dirs.varLogDir.file-pattern = ^(.*)(\.log)$  # /tmp/*.log
32 | ```
33 | 
34 | * Example for MultiLineParserModule
35 | ```
36 | agent.sources = dirMon
37 | agent.sources.dirMon.type = com.jinoos.flume.DirectoryTailSource
38 | agent.sources.dirMon.parser = com.jinoos.flume.MultiLineParserModule
39 | # Regex pattern for first line, format "yyyy-MM-dd HH:mm:ss"
40 | agent.sources.dirMon.first-line-pattern = ^([\d]{4})-([\d]{2})-([\d]{2}) ([\d]{2}):([\d]{2}):([\d]{2})
41 | agent.sources.dirMon.dirs = tmpDir varLogDir
42 | agent.sources.dirMon.dirs.tmpDir.path = /tmp
43 | agent.sources.dirMon.dirs.tmpDir.file-pattern = ^(message)$       # /var/log/message
44 | agent.sources.dirMon.dirs.varLogDir.path = /var/log
45 | agent.sources.dirMon.dirs.varLogDir.file-pattern = ^(.*)(\.log)$  # /tmp/*.log
46 | ```
47 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
 1 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 2 |   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 3 |   <modelVersion>4.0.0</modelVersion>
 4 | 
 5 |   <groupId>com.jinoos</groupId>
 6 |   <artifactId>flume-ng-extends</artifactId>
 7 |   <version>0.0.1-SNAPSHOT</version>
 8 |   <packaging>jar</packaging>
 9 | 
10 |   <name>flume-ng-extends</name>
11 |   <url>http://jinoos.com</url>
12 | 
13 |   <properties>
14 |     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
15 |   </properties>
16 | 
17 |   <dependencies>
18 |     <dependency>
19 |       <groupId>junit</groupId>
20 |       <artifactId>junit</artifactId>
21 |       <version>3.8.1</version>
22 |       <scope>test</scope>
23 |     </dependency>
24 |     <dependency>
25 |     	<groupId>org.apache.commons</groupId>
26 |     	<artifactId>commons-vfs2</artifactId>
27 |     	<version>2.0</version>
28 |     </dependency>
29 |     <dependency>
30 |     	<groupId>org.apache.flume</groupId>
31 |     	<artifactId>flume-ng-core</artifactId>
32 |     	<version>1.4.0</version>
33 |     </dependency>
34 |   </dependencies>
35 | </project>
36 | 


--------------------------------------------------------------------------------
/src/main/java/com/jinoos/flume/DirPattern.java:
--------------------------------------------------------------------------------
 1 | package com.jinoos.flume;
 2 | 
 3 | import java.util.regex.Pattern;
 4 | 
 5 | public class DirPattern {
 6 |   private String path;
 7 |   private Pattern filePattern;
 8 | 
 9 |   public String getPath() {
10 |     return path;
11 |   }
12 | 
13 |   public void setPath(String path) {
14 |     this.path = path;
15 |   }
16 | 
17 |   public Pattern getFilePattern() {
18 |     return filePattern;
19 |   }
20 | 
21 |   public void setFilePattern(Pattern filePattern) {
22 |     this.filePattern = filePattern;
23 |   }
24 | }


--------------------------------------------------------------------------------
/src/main/java/com/jinoos/flume/DirectoryTailParserModulable.java:
--------------------------------------------------------------------------------
 1 | package com.jinoos.flume;
 2 | 
 3 | import java.util.List;
 4 | import java.util.Map;
 5 | 
 6 | import org.apache.flume.conf.Configurable;
 7 | 
 8 | /**
 9 |  * <PRE>
10 |  * 1. ClassName : 
11 |  * 2. FileName  : DirectoyTailModulable.java
12 |  * 3. Package  : kr.co.cplanet.logcollector.flumeng.source
13 |  * 4. Comment  : 
14 |  * 5. @author  : Jinoos Lee <jinoos@gmail.com>
15 |  * 6. 작성일   : 2013. 10. 25. 오전 12:51:16
16 |  * </PRE>
17 |  */
18 | public interface DirectoryTailParserModulable extends Configurable {
19 |   public void flush();
20 | 
21 |   public void parse(String line, FileSet header);
22 | 
23 |   public boolean isFirstLine(String line);
24 | 
25 |   public boolean isLastLine(String line);
26 | }
27 | 


--------------------------------------------------------------------------------
/src/main/java/com/jinoos/flume/DirectoryTailSource.java:
--------------------------------------------------------------------------------
  1 | package com.jinoos.flume;
  2 | 
  3 | import java.io.IOException;
  4 | import java.util.HashMap;
  5 | import java.util.Hashtable;
  6 | import java.util.Map;
  7 | import java.util.Map.Entry;
  8 | import java.util.concurrent.BlockingQueue;
  9 | import java.util.concurrent.ExecutorService;
 10 | import java.util.concurrent.Executors;
 11 | import java.util.concurrent.Future;
 12 | import java.util.concurrent.LinkedBlockingQueue;
 13 | import java.util.concurrent.TimeUnit;
 14 | import java.util.regex.Matcher;
 15 | import java.util.regex.Pattern;
 16 | import java.util.regex.PatternSyntaxException;
 17 | 
 18 | import org.apache.commons.vfs2.FileChangeEvent;
 19 | import org.apache.commons.vfs2.FileListener;
 20 | import org.apache.commons.vfs2.FileObject;
 21 | import org.apache.commons.vfs2.FileSystemException;
 22 | import org.apache.commons.vfs2.FileSystemManager;
 23 | import org.apache.commons.vfs2.FileType;
 24 | import org.apache.commons.vfs2.VFS;
 25 | import org.apache.commons.vfs2.impl.DefaultFileMonitor;
 26 | import org.apache.flume.Context;
 27 | import org.apache.flume.Event;
 28 | import org.apache.flume.EventDrivenSource;
 29 | import org.apache.flume.conf.Configurable;
 30 | import org.apache.flume.event.EventBuilder;
 31 | import org.apache.flume.instrumentation.SourceCounter;
 32 | import org.apache.flume.source.AbstractSource;
 33 | import org.slf4j.Logger;
 34 | import org.slf4j.LoggerFactory;
 35 | 
 36 | import com.google.common.base.Preconditions;
 37 | 
 38 | /**
 39 |  * <PRE>
 40 |  * 1. ClassName : 
 41 |  * 2. FileName  : WasAppLogSource.java
 42 |  * 3. Package  : kr.co.cplanet.flumeng.source
 43 |  * 4. Comment  :
 44 |  * 5. 작성자   : jinoos
 45 |  * 6. 작성일   : 2013. 10. 16. 오후 9:22:44
 46 |  * </PRE>
 47 |  */
 48 | public class DirectoryTailSource extends AbstractSource implements
 49 |     Configurable, EventDrivenSource {
 50 |   private static final String CONFIG_SEPERATOR = ".";
 51 |   private static final String CONFIG_DIRS = "dirs";
 52 |   private static final String CONFIG_PATH = "path";
 53 |   private static final String CONFIG_FILE_PATTERN = "file-pattern";
 54 |   private static final String CONFIG_PARSER = "parser";
 55 |   private static final String CONFIG_KEEP_BUFFER_INTERVAL = "keep-buffer-interval";
 56 |   private static final String CONFIG_FILE_MONITOR_SLEEP_TIME = "file-monitor-sleep-time";
 57 | 
 58 |   // group(0) : DateTime YYYY-MM-DD HH:ii:ss
 59 |   // group(1) : Severity
 60 |   // group(2) : Thread Number
 61 |   // group(3) : Class
 62 |   // group(5) : Method
 63 |   // group(6) : line number
 64 |   // group(7) : Message
 65 |   private static final String DEFAULT_FILE_PATTERN = "^(.*)$";
 66 | 
 67 |   private static final String DEFAULT_PARSER_MODULE_CLASS = "com.jinoos.flume.SingleLineParserModule";
 68 | 
 69 |   private static final long DEFAULT_KEEP_BUFFER_INTERVAL = 1000;
 70 |   private static final long DEFAULT_FILE_MONITOR_SLEEP_TIME = 500;
 71 | 
 72 |   private static final Logger logger = LoggerFactory
 73 |       .getLogger(DirectoryTailSource.class);
 74 | 
 75 |   private SourceCounter sourceCounter;
 76 | 
 77 |   private String confDirs;
 78 | 
 79 |   private Map<String, DirPattern> dirMap;
 80 |   private Map<String, DirPattern> pathMap;
 81 | 
 82 |   private ExecutorService executorService;
 83 |   private MonitorRunnable monitorRunnable;
 84 |   private Future<?> monitorFuture;
 85 | 
 86 |   private long keepBufferInterval;
 87 |   private long fileMonitorSleepTime;
 88 | 
 89 |   DirectoryTailParserModulable parserModule;
 90 | 
 91 |   private DefaultFileMonitor fileMonitor;
 92 | 
 93 |   // private static final long eventQueueWorkerTimeoutMiliSecond = 1000;
 94 |   private static final int eventQueueWorkerSize = 10;
 95 |   private static final int maxEventQueueSize = 1000 * 1000;
 96 |   private BlockingQueue<DirectoryTailEvent> eventQueue = new LinkedBlockingQueue<DirectoryTailEvent>(
 97 |       maxEventQueueSize);
 98 |   private Future<?>[] workerFuture = new Future<?>[eventQueueWorkerSize];
 99 | 
100 |   private FileSystemManager fsManager;
101 |   private Hashtable<String, FileSet> fileSetMap;
102 | 
103 |   /**
104 |    * <PRE>
105 |    * 1. MethodName : configure
106 |    * 2. ClassName  : DirectoryTailSource
107 |    * 3. Comment   :
108 |    * 4. 작성자    : Jinoos Lee <jinoos@gmail.com>
109 |    * 5. 작성일    : 2013. 11. 1. 오후 12:51:20
110 |    * </PRE>
111 |    * 
112 |    * @param context
113 |    */
114 |   public void configure(Context context) {
115 |     logger.info("Source Configuring..");
116 | 
117 |     dirMap = new HashMap<String, DirPattern>();
118 |     pathMap = new HashMap<String, DirPattern>();
119 | 
120 |     keepBufferInterval = context.getLong(CONFIG_KEEP_BUFFER_INTERVAL,
121 |         DEFAULT_KEEP_BUFFER_INTERVAL);
122 | 
123 |     fileMonitorSleepTime = context.getLong(CONFIG_FILE_MONITOR_SLEEP_TIME,
124 |         DEFAULT_FILE_MONITOR_SLEEP_TIME);
125 | 
126 |     confDirs = context.getString(CONFIG_DIRS).trim();
127 |     Preconditions.checkState(confDirs != null,
128 |         "Configuration must be specified directory(ies).");
129 | 
130 |     String[] confDirArr = confDirs.split(" ");
131 | 
132 |     Preconditions.checkState(confDirArr.length > 0, CONFIG_DIRS
133 |         + " must be specified at least one.");
134 | 
135 |     for (int i = 0; i < confDirArr.length; i++) {
136 | 
137 |       String path = context.getString(CONFIG_DIRS + CONFIG_SEPERATOR
138 |           + confDirArr[i] + CONFIG_SEPERATOR + CONFIG_PATH);
139 |       if (path == null) {
140 |         logger
141 |             .warn("Configuration is empty : " + CONFIG_DIRS + CONFIG_SEPERATOR
142 |                 + confDirArr[i] + CONFIG_SEPERATOR + CONFIG_PATH);
143 |         continue;
144 |       }
145 | 
146 |       String patternString = context.getString(CONFIG_DIRS + CONFIG_SEPERATOR
147 |           + confDirArr[i] + CONFIG_SEPERATOR + CONFIG_FILE_PATTERN);
148 |       if (patternString == null) {
149 |         patternString = DEFAULT_FILE_PATTERN;
150 |       }
151 | 
152 |       Pattern pattern = null;
153 |       try {
154 |         pattern = Pattern.compile(patternString);
155 |       } catch (PatternSyntaxException e) {
156 |         logger.warn("Configuration has wrong file pattern, " + CONFIG_DIRS
157 |             + "." + confDirArr[i] + "." + CONFIG_FILE_PATTERN + ":"
158 |             + patternString);
159 |         logger.warn("Directory will be set default file pattern, "
160 |             + DEFAULT_FILE_PATTERN);
161 | 
162 |         pattern = Pattern.compile(patternString);
163 |       }
164 | 
165 |       DirPattern dir = new DirPattern();
166 |       dir.setPath(path);
167 |       dir.setFilePattern(pattern);
168 | 
169 |       dirMap.put(confDirArr[i], dir);
170 |       logger.debug("parsed dirs configure dir : " + confDirArr[i] + ", path : "
171 |           + path);
172 |     }
173 | 
174 |     String confParserModule = DEFAULT_PARSER_MODULE_CLASS;
175 |     try {
176 |       confParserModule = context.getString(CONFIG_PARSER,
177 |           DEFAULT_PARSER_MODULE_CLASS);
178 |       parserModule = (DirectoryTailParserModulable) Class.forName(
179 |           confParserModule).newInstance();
180 |       logger.debug("parserClass : " + confParserModule);
181 |     } catch (ClassNotFoundException e) {
182 |       Preconditions.checkState(false, e.getMessage() + " " + confParserModule);
183 |       logger.error(e.getMessage(), e);
184 |     } catch (InstantiationException e) {
185 |       Preconditions.checkState(false, e.getMessage() + " " + confParserModule);
186 |       logger.error(e.getMessage(), e);
187 |     } catch (IllegalAccessException e) {
188 |       Preconditions.checkState(false, e.getMessage() + " " + confParserModule);
189 |       logger.error(e.getMessage(), e);
190 |     }
191 | 
192 |     parserModule.configure(context);
193 |   }
194 | 
195 |   /**
196 |    * <PRE>
197 |    * 1. MethodName : start
198 |    * 2. ClassName  : WasAppLogSource
199 |    * 3. Comment   :
200 |    * 4. 작성자    : jinoos
201 |    * 5. 작성일    : 2013. 10. 16. 오후 10:43:06
202 |    * </PRE>
203 |    */
204 |   @Override
205 |   public void start() {
206 |     logger.info("Source Starting..");
207 | 
208 |     if (sourceCounter == null) {
209 |       sourceCounter = new SourceCounter(getName());
210 |     }
211 | 
212 |     fileSetMap = new Hashtable<String, FileSet>();
213 | 
214 |     try {
215 |       fsManager = VFS.getManager();
216 |     } catch (FileSystemException e) {
217 |       logger.error(e.getMessage(), e);
218 |       return;
219 |     }
220 | 
221 |     monitorRunnable = new MonitorRunnable();
222 | 
223 |     fileMonitor = new DefaultFileMonitor(monitorRunnable);
224 |     fileMonitor.setRecursive(false);
225 | 
226 |     FileObject fileObject;
227 | 
228 |     logger.debug("Dirlist count " + dirMap.size());
229 |     for (Entry<String, DirPattern> entry : dirMap.entrySet()) {
230 |       logger.debug("Scan dir " + entry.getKey());
231 | 
232 |       DirPattern dirPattern = entry.getValue();
233 | 
234 |       try {
235 |         fileObject = fsManager.resolveFile(dirPattern.getPath());
236 |       } catch (FileSystemException e) {
237 |         logger.error(e.getMessage(), e);
238 |         continue;
239 |       }
240 | 
241 |       try {
242 |         if (!fileObject.isReadable()) {
243 |           logger.warn("No have readable permission, " + fileObject.getURL());
244 |           continue;
245 |         }
246 | 
247 |         if (FileType.FOLDER != fileObject.getType()) {
248 |           logger.warn("Not a directory, " + fileObject.getURL());
249 |           continue;
250 |         }
251 | 
252 |         // 폴더를 Monitoring 대상에 추가한다.
253 |         fileMonitor.addFile(fileObject);
254 |         logger.debug(fileObject.getName().getPath()
255 |             + " directory has been add in monitoring list");
256 |         pathMap.put(fileObject.getName().getPath(), entry.getValue());
257 |       } catch (FileSystemException e) {
258 |         logger.warn(e.getMessage(), e);
259 |         continue;
260 |       } catch (Exception e) {
261 |         logger.debug(e.getMessage(), e);
262 |       }
263 | 
264 |     }
265 | 
266 |     executorService = Executors.newFixedThreadPool(eventQueueWorkerSize + 1);
267 |     monitorFuture = executorService.submit(monitorRunnable);
268 | 
269 |     for (int i = 0; i < eventQueueWorkerSize; i++) {
270 |       workerFuture[i] = executorService.submit(new WorkerRunnable(this));
271 |     }
272 | 
273 |     sourceCounter.start();
274 |     super.start();
275 |   }
276 | 
277 |   /**
278 |    * <PRE>
279 |    * 1. MethodName : stop
280 |    * 2. ClassName  : WasAppLogSource
281 |    * 3. Comment   :
282 |    * 4. 작성자    : jinoos
283 |    * 5. 작성일    : 2013. 10. 16. 오후 10:43:03
284 |    * </PRE>
285 |    */
286 |   @Override
287 |   public void stop() {
288 |     logger.info("Source Stopping..");
289 |     fileMonitor.stop();
290 |     sourceCounter.stop();
291 | 
292 |   }
293 | 
294 |   private class WorkerRunnable implements Runnable {
295 |     private AbstractSource source;
296 | 
297 |     private WorkerRunnable(AbstractSource source) {
298 |       this.source = source;
299 |     }
300 | 
301 |     public void run() {
302 |       while (true) {
303 |         try {
304 |           // DirectoryTailEvent event = eventQueue.poll(
305 |           // eventQueueWorkerTimeoutMiliSecond,
306 |           // TimeUnit.MILLISECONDS);
307 |           DirectoryTailEvent event = eventQueue.take();
308 | 
309 |           if (event == null) {
310 |             continue;
311 |           }
312 | 
313 |           if (event.type == FileEventType.FILE_CHANGED) {
314 |             fileChanged(event.event);
315 |           } else if (event.type == FileEventType.FILE_CREATED) {
316 |             fileCreated(event.event);
317 |           } else if (event.type == FileEventType.FILE_DELETED) {
318 |             fileDeleted(event.event);
319 |           } else if (event.type == FileEventType.FLUSH) {
320 |             if (event.fileSet != null)
321 |               sendEvent(event.fileSet);
322 |           }
323 |         } catch (InterruptedException e) {
324 |           logger.debug(e.getMessage(), e);
325 |         } catch (FileSystemException e) {
326 |           logger.info(e.getMessage(), e);
327 |         }
328 |       }
329 |     }
330 | 
331 |     private void fileCreated(FileChangeEvent event) throws FileSystemException {
332 |       String path = event.getFile().getName().getPath();
333 |       String dirPath = event.getFile().getParent().getName().getPath();
334 | 
335 |       logger.debug(path + " has been created.");
336 | 
337 |       DirPattern dirPattern = null;
338 |       dirPattern = pathMap.get(dirPath);
339 | 
340 |       if (dirPattern == null) {
341 |         logger.warn("Occurred create event from un-indexed directory. "
342 |             + dirPath);
343 |         return;
344 |       }
345 | 
346 |       // 파일명이 대상인지 검사한다.
347 |       if (!isInFilePattern(event.getFile(), dirPattern.getFilePattern())) {
348 |         logger.debug(path + " is not in file pattern.");
349 |         return;
350 |       }
351 | 
352 |       FileSet fileSet;
353 | 
354 |       fileSet = fileSetMap.get(event.getFile().getName().getPath());
355 | 
356 |       if (fileSet == null) {
357 |         try {
358 |           logger.info(path
359 |               + " is not in monitoring list. It's going to be listed.");
360 |           fileSet = new FileSet(source, event.getFile());
361 |           //a little synchronized bug here.fixed by tqli,2014-08-07 ,E-mail:tiangang1126@126.com
362 |           synchronized(fileSetMap){
363 |             fileSetMap.put(path, fileSet);
364 |           }
365 |         } catch (IOException e) {
366 |           logger.error(e.getMessage(), e);
367 |           return;
368 |         }
369 |       }
370 |     }
371 | 
372 |     private void fileDeleted(FileChangeEvent event) throws FileSystemException {
373 |       String path = event.getFile().getName().getPath();
374 |       String dirPath = event.getFile().getParent().getName().getPath();
375 | 
376 |       logger.debug(path + " has been deleted.");
377 | 
378 |       DirPattern dirPattern = pathMap.get(dirPath);
379 |       if (dirPattern == null) {
380 |         logger.warn("Occurred delete event from un-indexed directory. "
381 |             + dirPath);
382 |         return;
383 |       }
384 | 
385 |       if (!isInFilePattern(event.getFile(), dirPattern.getFilePattern())) {
386 |         logger.debug(path + " is not in file pattern.");
387 |         return;
388 |       }
389 | 
390 |       FileSet fileSet = fileSetMap.get(path);
391 | 
392 |       if (fileSet != null) {
393 |           //a little synchronized bug here.fixed by tqli,2014-08-07,E-mail:tiangang1126@126.com
394 |           synchronized(fileSetMap){
395 |             fileSetMap.remove(path);
396 |           }
397 |         logger.debug("Removed monitoring fileSet.");
398 |       }
399 |     }
400 | 
401 |     private void fileChanged(FileChangeEvent event) throws FileSystemException {
402 |       String path = event.getFile().getName().getPath();
403 |       String dirPath = event.getFile().getParent().getName().getPath();
404 | 
405 |       logger.debug(path + " has been changed.");
406 | 
407 |       DirPattern dirPattern = pathMap.get(dirPath);
408 |       if (dirPattern == null) {
409 |         logger.warn("Occurred change event from un-indexed directory. "
410 |             + dirPath);
411 |         return;
412 |       }
413 | 
414 |       // 파일명이 대상인지 검사한다.
415 |       if (!isInFilePattern(event.getFile(), dirPattern.getFilePattern())) {
416 |         logger.debug("Not in file pattern, " + path);
417 |         return;
418 |       }
419 | 
420 |       FileSet fileSet = fileSetMap.get(event.getFile().getName().getPath());
421 | 
422 |       if (fileSet == null) {
423 |         logger.warn(path + "is not in monitoring list.");
424 |         try {
425 |           fileSet = new FileSet(source, event.getFile());
426 |           synchronized (fileSetMap) {
427 |             fileSetMap.put(path, fileSet);
428 |           }
429 |         } catch (IOException e) {
430 |           logger.error(e.getMessage(), e);
431 |           return;
432 |         }
433 |         return;
434 |       }
435 | 
436 |       readMessage(fileSet);
437 |     }
438 | 
439 |     // 파일이 대상 패턴에 존재하는지 검사한다.
440 |     private boolean isInFilePattern(FileObject file, Pattern pattern) {
441 |       String fileName = file.getName().getBaseName();
442 |       Matcher matcher = pattern.matcher(fileName);
443 |       if (matcher.find()) {
444 |         return true;
445 |       }
446 |       return false;
447 |     }
448 | 
449 |     // 파일을 읽고 Event를 생성한다.
450 |     private void readMessage(FileSet fileSet) {
451 |       try {
452 |         String buffer;
453 | 
454 |         synchronized (fileSet) {
455 | 
456 |           while ((buffer = fileSet.readLine()) != null) {
457 |             if (buffer.length() == 0) {
458 |               continue;
459 |             }
460 | 
461 |             boolean isFirstLine = parserModule.isFirstLine(buffer);
462 |             if (isFirstLine) {
463 |               sendEvent(fileSet);
464 |               fileSet.appendLine(buffer);
465 |               parserModule.parse(buffer, fileSet);
466 | 
467 |             } else {
468 |               if (fileSet.getLineSize() == 0) {
469 |                 logger.debug("Wrong log format, " + buffer);
470 |                 continue;
471 |               } else {
472 |                 fileSet.appendLine(buffer);
473 |                 parserModule.parse(buffer, fileSet);
474 |               }
475 |             }
476 | 
477 |             if (parserModule.isLastLine(buffer)) {
478 |               sendEvent(fileSet);
479 |             }
480 |           }
481 |         }
482 |       } catch (IOException e) {
483 |         logger.warn(e.getMessage(), e);
484 |       }
485 |     }
486 | 
487 |     private void sendEvent(FileSet fileSet) {
488 |       if (fileSet.getBufferList().isEmpty())
489 |         return;
490 | 
491 |       synchronized (fileSet) {
492 |         StringBuffer sb = fileSet.getAllLines();
493 |         Event event = EventBuilder.withBody(String.valueOf(sb).getBytes(),
494 |             fileSet.getHeaders());
495 |         source.getChannelProcessor().processEvent(event);
496 |         sourceCounter.incrementEventReceivedCount();
497 | 
498 |         fileSet.clear();
499 |       }
500 |     }
501 | 
502 |   }
503 | 
504 |   private class MonitorRunnable implements Runnable, FileListener {
505 |     public void run() {
506 | 
507 |       fileMonitor.setDelay(fileMonitorSleepTime);
508 |       fileMonitor.start();
509 | 
510 |       while (true) {
511 |         try {
512 |           Thread.sleep(keepBufferInterval);
513 |           fileMonitor.run();
514 |         } catch (InterruptedException e) {
515 |           logger.debug(e.getMessage(), e);
516 |         }
517 | 
518 |         flushFileSetBuffer();
519 |       }
520 |     }
521 | 
522 |     // 파일생성을 감지함.
523 |     public void fileCreated(FileChangeEvent event) throws Exception {
524 |       DirectoryTailEvent dtEvent = new DirectoryTailEvent(event,
525 |           FileEventType.FILE_CREATED);
526 |       eventQueue.put(dtEvent);
527 |     }
528 | 
529 |     // 파일 삭제를 감지함.
530 |     public void fileDeleted(FileChangeEvent event) throws Exception {
531 |       DirectoryTailEvent dtEvent = new DirectoryTailEvent(event,
532 |           FileEventType.FILE_DELETED);
533 |       eventQueue.put(dtEvent);
534 |     }
535 | 
536 |     // 파일 변경을 감지함.
537 |     public void fileChanged(FileChangeEvent event) throws Exception {
538 |       DirectoryTailEvent dtEvent = new DirectoryTailEvent(event,
539 |           FileEventType.FILE_CHANGED);
540 |       eventQueue.put(dtEvent);
541 |     }
542 | 
543 |     public void flush(FileSet fileSet) {
544 |       DirectoryTailEvent dtEvent = new DirectoryTailEvent(fileSet);
545 |       try {
546 |         eventQueue.put(dtEvent);
547 |       } catch (InterruptedException e) {
548 |         logger.warn(e.getMessage(), e);
549 |       }
550 |     }
551 | 
552 |     private void flushFileSetBuffer() {
553 |       synchronized (fileSetMap) {
554 |         long cutTime = System.currentTimeMillis() - keepBufferInterval;
555 | 
556 |         for (Map.Entry<String, FileSet> entry : fileSetMap.entrySet()) {
557 | 
558 |           // If lastAppendTime is over than keepBufferInterval,
559 |           // then, the message will be flushed even not be catched
560 |           // new first line or last line. It's to prevent last message
561 |           // delay delevery.
562 |           if (entry.getValue().getBufferList().size() > 0
563 |               && entry.getValue().getLastAppendTime() < cutTime) {
564 |             flush(entry.getValue());
565 |           }
566 |         }
567 |       }
568 |     }
569 |   }
570 | 
571 |   private enum FileEventType {
572 |     FILE_CREATED, FILE_CHANGED, FILE_DELETED, FLUSH
573 |   }
574 | 
575 |   private class DirectoryTailEvent {
576 |     FileChangeEvent event;
577 |     FileEventType type;
578 |     FileSet fileSet;
579 | 
580 |     public DirectoryTailEvent(FileChangeEvent event, FileEventType type) {
581 |       this.type = type;
582 |       this.event = event;
583 |       this.fileSet = null;
584 |     }
585 | 
586 |     public DirectoryTailEvent(FileSet fileSet) {
587 |       this.type = FileEventType.FLUSH;
588 |       this.fileSet = fileSet;
589 |       this.event = null;
590 |     }
591 |   }
592 | }
593 | 


--------------------------------------------------------------------------------
/src/main/java/com/jinoos/flume/FileSet.java:
--------------------------------------------------------------------------------
  1 | package com.jinoos.flume;
  2 | 
  3 | import java.io.BufferedReader;
  4 | import java.io.File;
  5 | import java.io.IOException;
  6 | import java.io.RandomAccessFile;
  7 | import java.util.ArrayList;
  8 | import java.util.HashMap;
  9 | import java.util.List;
 10 | import java.util.Map;
 11 | 
 12 | import org.apache.commons.vfs2.FileObject;
 13 | import org.apache.flume.Transaction;
 14 | import org.apache.flume.source.AbstractSource;
 15 | import org.slf4j.Logger;
 16 | import org.slf4j.LoggerFactory;
 17 | 
 18 | public class FileSet {
 19 |   private static final Logger logger = LoggerFactory.getLogger(FileSet.class);
 20 |   private AbstractSource source;
 21 |   private FileObject fileObject;
 22 |   private BufferedReader bufferedReader;
 23 |   private RandomAccessFile rReader;
 24 |   private Transaction transaction;
 25 |   private List<String> bufferList;
 26 |   private Map<String, String> headers;
 27 |   private long lastAppendTime;
 28 |   private Long seq;
 29 | 
 30 |   public FileSet(AbstractSource source, FileObject fileObject)
 31 |       throws IOException {
 32 |     this.source = source;
 33 |     this.fileObject = fileObject;
 34 | 
 35 |     this.bufferList = new ArrayList<String>();
 36 | 
 37 |     File f = new File(fileObject.getName().getPath());
 38 | 
 39 |     rReader = new RandomAccessFile(f, "r");
 40 |     rReader.seek(f.length());
 41 | 
 42 |     bufferList = new ArrayList<String>();
 43 |     headers = new HashMap<String, String>();
 44 |     logger.debug("FileSet has been created " + fileObject.getName().getPath());
 45 |     this.seq = 0L;
 46 |   }
 47 | 
 48 |   public String readLine() throws IOException {
 49 |     return rReader.readLine();
 50 |   }
 51 | 
 52 |   public long getLastAppendTime() {
 53 |     return lastAppendTime;
 54 |   }
 55 | 
 56 |   public void setLastAppendTime(long lastAppendTime) {
 57 |     this.lastAppendTime = lastAppendTime;
 58 |   }
 59 | 
 60 |   public boolean appendLine(String buffer) {
 61 |     boolean ret = bufferList.add(buffer);
 62 |     if (ret) {
 63 |       lastAppendTime = System.currentTimeMillis();
 64 |     }
 65 | 
 66 |     return ret;
 67 |   }
 68 | 
 69 |   public int getLineSize() {
 70 |     return bufferList.size();
 71 |   }
 72 | 
 73 |   public StringBuffer getAllLines() {
 74 | 
 75 |     StringBuffer sb = new StringBuffer();
 76 | 
 77 |     for (int i = 0; i < bufferList.size(); i++) {
 78 |       sb.append(bufferList.get(i));
 79 |     }
 80 |     return sb;
 81 |   }
 82 | 
 83 |   public void setHeader(String key, String value) {
 84 |     headers.put(key, value);
 85 |   }
 86 | 
 87 |   public String getHeader(String key) {
 88 |     headers.get(key);
 89 |     return null;
 90 |   }
 91 | 
 92 |   public void clear() {
 93 |     bufferList.clear();
 94 |     headers.clear();
 95 |   }
 96 | 
 97 |   public Map<String, String> getHeaders() {
 98 |     return headers;
 99 |   }
100 | 
101 |   public AbstractSource getSource() {
102 |     return source;
103 |   }
104 | 
105 |   public void setSource(AbstractSource source) {
106 |     this.source = source;
107 |   }
108 | 
109 |   public List<String> getBufferList() {
110 |     return bufferList;
111 |   }
112 | 
113 |   public void setBufferList(List<String> bufferList) {
114 |     this.bufferList = bufferList;
115 |   }
116 | 
117 |   public Transaction getTransaction() {
118 |     return transaction;
119 |   }
120 | 
121 |   public void setTransaction(Transaction transaction) {
122 |     this.transaction = transaction;
123 |   }
124 | 
125 |   public FileObject getFileObject() {
126 |     return fileObject;
127 |   }
128 | 
129 |   public void setFileObject(FileObject fileObject) {
130 |     this.fileObject = fileObject;
131 |   }
132 | 
133 |   public BufferedReader getBufferedReader() {
134 |     return bufferedReader;
135 |   }
136 | 
137 |   public void setBufferedReader(BufferedReader bufferedReader) {
138 |     this.bufferedReader = bufferedReader;
139 |   }
140 | 
141 |   public Long getSeq() {
142 |     return ++seq;
143 |   }
144 | 
145 |   /*
146 |    * public void setSeq(Long seq) { this.seq = seq; }
147 |    */
148 | 
149 | }
150 | 


--------------------------------------------------------------------------------
/src/main/java/com/jinoos/flume/MultiLineParserModule.java:
--------------------------------------------------------------------------------
 1 | package com.jinoos.flume;
 2 | 
 3 | import java.util.regex.Matcher;
 4 | import java.util.regex.Pattern;
 5 | import java.util.regex.PatternSyntaxException;
 6 | 
 7 | import org.apache.flume.Context;
 8 | import org.apache.flume.conf.Configurable;
 9 | import org.slf4j.Logger;
10 | import org.slf4j.LoggerFactory;
11 | 
12 | import com.google.common.base.Preconditions;
13 | 
14 | public class MultiLineParserModule implements Configurable,
15 |     DirectoryTailParserModulable {
16 |   private static final Logger logger = LoggerFactory
17 |       .getLogger(MultiLineParserModule.class);
18 | 
19 |   private static final String CONFIG_FIRST_LINE_PATTERN = "first-line-pattern";
20 |   private Pattern pattern;
21 |   private String patternString;
22 | 
23 |   public void flush() {
24 |   }
25 | 
26 |   public void parse(String line, FileSet header) {
27 |   }
28 | 
29 |   public boolean isFirstLine(String line) {
30 |     Matcher matcher = pattern.matcher(line);
31 | 
32 |     return matcher.find();
33 |   }
34 | 
35 |   public boolean isLastLine(String line) {
36 |     return false;
37 |   }
38 | 
39 |   public void configure(Context context) {
40 |     patternString = context.getString(CONFIG_FIRST_LINE_PATTERN);
41 |     Preconditions.checkState(patternString != null,
42 |         "Configuration must specify first-line-pattern.");
43 | 
44 |     try {
45 |       pattern = Pattern.compile(patternString);
46 |     } catch (PatternSyntaxException e) {
47 |       Preconditions.checkState(pattern != null, e.getMessage());
48 |       logger.error(e.getMessage(), e);
49 |       return;
50 |     }
51 |   }
52 | }
53 | 


--------------------------------------------------------------------------------
/src/main/java/com/jinoos/flume/SingleLineParserModule.java:
--------------------------------------------------------------------------------
 1 | package com.jinoos.flume;
 2 | 
 3 | import java.util.List;
 4 | import java.util.Map;
 5 | 
 6 | import org.apache.flume.Context;
 7 | import org.apache.flume.conf.Configurable;
 8 | 
 9 | public class SingleLineParserModule implements Configurable,
10 |     DirectoryTailParserModulable {
11 | 
12 |   public void flush() {
13 |   }
14 | 
15 |   public void parse(String line, FileSet header) {
16 |   }
17 | 
18 |   public boolean isFirstLine(String line) {
19 |     return false;
20 |   }
21 | 
22 |   public boolean isLastLine(String line) {
23 |     return true;
24 |   }
25 | 
26 |   public void configure(Context context) {
27 |     // TODO Auto-generated method stub
28 |   }
29 | }
30 | 


--------------------------------------------------------------------------------
/src/main/java/com/jinoos/flume/ThreadSafeSimpleDateFormat.java:
--------------------------------------------------------------------------------
 1 | package com.jinoos.flume;
 2 | 
 3 | import java.text.DateFormat;
 4 | import java.text.SimpleDateFormat;
 5 | import java.util.Date;
 6 | import java.text.ParseException;
 7 | 
 8 | public class ThreadSafeSimpleDateFormat {
 9 | 	private DateFormat df;
10 | 	
11 | 	public ThreadSafeSimpleDateFormat(String format)
12 | 	{
13 | 		this.df = new SimpleDateFormat(format);
14 | 	}
15 | 	
16 | 	public synchronized String format(Date date)
17 | 	{
18 | 		return df.format(date);
19 | 	}
20 | 	
21 | 	public synchronized Date parse(String string) throws ParseException {
22 | 		return df.parse(string);
23 | 	}
24 | }
25 | 


--------------------------------------------------------------------------------
/src/test/java/com/jinoos/flume_ng_extends/AppTest.java:
--------------------------------------------------------------------------------
 1 | package com.jinoos.flume_ng_extends;
 2 | 
 3 | import junit.framework.Test;
 4 | import junit.framework.TestCase;
 5 | import junit.framework.TestSuite;
 6 | 
 7 | /**
 8 |  * Unit test for simple App.
 9 |  */
10 | public class AppTest 
11 |     extends TestCase
12 | {
13 |     /**
14 |      * Create the test case
15 |      *
16 |      * @param testName name of the test case
17 |      */
18 |     public AppTest( String testName )
19 |     {
20 |         super( testName );
21 |     }
22 | 
23 |     /**
24 |      * @return the suite of tests being tested
25 |      */
26 |     public static Test suite()
27 |     {
28 |         return new TestSuite( AppTest.class );
29 |     }
30 | 
31 |     /**
32 |      * Rigourous Test :-)
33 |      */
34 |     public void testApp()
35 |     {
36 |         assertTrue( true );
37 |     }
38 | }
39 | 


--------------------------------------------------------------------------------