├── .classpath ├── .gitignore ├── .project ├── .settings ├── org.eclipse.core.resources.prefs ├── org.eclipse.jdt.core.prefs └── org.eclipse.m2e.core.prefs ├── HOWTO-KEYSTORE.md ├── LICENSE.md ├── README.md ├── pom.xml └── src ├── assembly └── tarball.xml ├── main └── java │ └── info │ └── fetter │ └── logstashforwarder │ ├── Event.java │ ├── FileModificationListener.java │ ├── FileReader.java │ ├── FileSigner.java │ ├── FileState.java │ ├── FileWatcher.java │ ├── Filter.java │ ├── Forwarder.java │ ├── InputReader.java │ ├── Multiline.java │ ├── ProtocolAdapter.java │ ├── Reader.java │ ├── Registrar.java │ ├── config │ ├── Configuration.java │ ├── ConfigurationManager.java │ ├── FilesSection.java │ └── NetworkSection.java │ ├── protocol │ └── LumberjackClient.java │ └── util │ ├── AdapterException.java │ ├── KMPMatch.java │ ├── LastModifiedFileFilter.java │ └── RandomAccessFile.java └── test ├── java └── info │ └── fetter │ └── logstashforwarder │ ├── FileReaderTest.java │ ├── FileWatcherTest.java │ ├── InputReaderTest.java │ ├── MockProtocolAdapter.java │ ├── RegistrarTest.java │ └── config │ └── ConfigurationManagerTest.java └── resources ├── config1.json └── state1.json /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /bin/ 3 | /testFileReader1.txt 4 | /state2.json 5 | .idea/ 6 | logstash-forwarder-java.iml -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | logstash-forwarder-java 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.m2e.core.maven2Nature 21 | org.eclipse.jdt.core.javanature 22 | 23 | 24 | -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//src/main/java=UTF-8 3 | encoding//src/main/resources=UTF-8 4 | encoding//src/test/java=UTF-8 5 | encoding//src/test/resources=UTF-8 6 | encoding/=UTF-8 7 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled 3 | org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate 4 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 5 | org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve 6 | org.eclipse.jdt.core.compiler.compliance=1.5 7 | org.eclipse.jdt.core.compiler.debug.lineNumber=generate 8 | org.eclipse.jdt.core.compiler.debug.localVariable=generate 9 | org.eclipse.jdt.core.compiler.debug.sourceFile=generate 10 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error 11 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error 12 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 13 | org.eclipse.jdt.core.compiler.source=1.5 14 | -------------------------------------------------------------------------------- /.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /HOWTO-KEYSTORE.md: -------------------------------------------------------------------------------- 1 | # How to create a java keystore ? 2 | 3 | ## From a .pem file 4 | 5 | If you have your CA certificate stored in a .pem text file, run the following command to create the keystore.jks file : 6 | 7 | keytool -importcert -trustcacerts -file cacert.pem -alias ca -keystore keystore.jks 8 | 9 | To list the contents of the keystore file, run this command : 10 | 11 | keytool -list -v -keystore keystore.jks 12 | 13 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2015 Didier Fetter 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | RandomAccessFile and KMPMatch classes by UCAR/Unidata. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # logstash-forwarder-java 2 | 3 | ## What is this ? 4 | 5 | Logstash-forwarder-java is a log shipper program written in java. This is in fact a java version of [logstash-forwarder](https://github.com/elasticsearch/logstash-forwarder) by jordansissel. 6 | Here are a few features of this program : 7 | - compatible with Java 5 runtime 8 | - lightweight : package size is ~2MB and memory footprint ~8MB 9 | - configuration compatible with logstash-forwarder 10 | - lumberjack output (including zlib compression) 11 | 12 | ## Why ? 13 | 14 | Logstash-forwarder is written in go. This programming language is not available on all platforms (for example AIX), that's why a java version is more portable. 15 | 16 | Logstash runs on java and provides a lumberjack output, but the file input doesn't run on all plaforms (for example AIX) and logstash requires a recent JVM. Moreover Logstash is heavier : it is a big package and uses more system resources. 17 | 18 | So logstash-forwarder-java is a solution for those who want a portable, lightweight log shipper for their ELK stack. 19 | 20 | ## How to install it ? 21 | 22 | Download one of the following archives : 23 | - [logstash-forwarder-java-0.2.4-bin.zip](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.4/logstash-forwarder-java-0.2.4-bin.zip) 24 | - [logstash-forwarder-java-0.2.4-bin.tar.gz](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.4/logstash-forwarder-java-0.2.4-bin.tar.gz) 25 | - [logstash-forwarder-java-0.2.4-bin.tar.bz2](https://github.com/didfet/logstash-forwarder-java/releases/download/0.2.4/logstash-forwarder-java-0.2.4-bin.tar.bz2) 26 | 27 | Or download the maven project and run maven package. Then you can install one of the archives located in the target directory. 28 | 29 | ## How to run it ? 30 | 31 | Just run this command : 32 | 33 | java -jar logstash-forwarder-java-X.Y.Z.jar -config /path/to/config/file.json 34 | 35 | For help run : 36 | 37 | java -jar logstash-forwarder-java-X.Y.Z.jar -help 38 | 39 | ## Differences with logstash-forwarder 40 | 41 | ### Configuration 42 | 43 | The configuration file is the same (json format), but there are a few differences : 44 | - the ssl ca parameter points to a java [keystore](https://github.com/didfet/logstash-forwarder-java/blob/master/HOWTO-KEYSTORE.md) containing the root certificate of the server, not a PEM file 45 | - comments are C-style comments 46 | - multiline support with attributes "pattern", "negate" (true/false) and "what" (previous/next) (version 0.2.5) 47 | - filtering support with attributes "pattern" and "negate" (true/false) (version 0.2.5) 48 | 49 | ### Command-line options 50 | 51 | Some options are the same : 52 | - config (but only for a file, not a directory) 53 | - quiet 54 | - idle-timeout (renamed idletimeout) 55 | - spool-size (renamed spoolsize) 56 | - tail 57 | - help 58 | 59 | There are a few more options : 60 | - debug : turn on debug logging level 61 | - trace : turn on trace logging level 62 | - signaturelength : size of the block used to compute the checksum 63 | - logfile : send logs to this file instead of stdout 64 | - logfilesize : maximum size of each log file (default 10M) 65 | - logfilenumber : number of rotated log files (default 5) 66 | 67 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | logstash-forwarder-java 5 | logstash-forwarder-java 6 | 0.2.5-SNAPSHOT 7 | logstash-forwarder-java 8 | Java version of logstash forwarder 9 | https://github.com/didfet/logstash-forwarder-java 10 | 11 | UTF-8 12 | 13 | 14 | 15 | The Apache Software License, Version 2.0 16 | http://www.apache.org/licenses/LICENSE-2.0.txt 17 | manual 18 | A business-friendly OSS license 19 | 20 | 21 | 22 | 23 | 24 | scm:git:https://github.com/didfet/logstash-forwarder-java 25 | scm:git:https://github.com/didfet/logstash-forwarder-java 26 | https://github.com/didfet/logstash-forwarder-java 27 | 28 | 29 | ${project.artifactId}-${project.version} 30 | 31 | 32 | org.apache.maven.plugins 33 | maven-surefire-plugin 34 | 2.18.1 35 | 36 | 0 37 | 38 | 39 | 40 | org.apache.maven.plugins 41 | maven-jar-plugin 42 | 2.4 43 | 44 | 45 | 46 | true 47 | lib/ 48 | info.fetter.logstashforwarder.Forwarder 49 | true 50 | 51 | 52 | ${prefix.revision} 53 | ${timestamp} 54 | 55 | 56 | 57 | 58 | 59 | org.apache.maven.plugins 60 | maven-dependency-plugin 61 | 2.8 62 | 63 | 64 | copy-dependencies 65 | package 66 | 67 | copy-dependencies 68 | 69 | 70 | ${project.build.directory}/lib 71 | false 72 | false 73 | true 74 | provided 75 | 76 | 77 | 78 | 79 | 80 | maven-assembly-plugin 81 | 2.5.3 82 | 83 | src/assembly/tarball.xml 84 | 85 | 86 | 87 | create-archive 88 | package 89 | 90 | single 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 100 | 101 | org.eclipse.m2e 102 | lifecycle-mapping 103 | 1.0.0 104 | 105 | 106 | 107 | 108 | 109 | 110 | org.apache.maven.plugins 111 | 112 | 113 | maven-dependency-plugin 114 | 115 | 116 | [2.8,) 117 | 118 | 119 | 120 | copy-dependencies 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | commons-io 138 | commons-io 139 | 2.2 140 | 141 | 142 | com.fasterxml.jackson.core 143 | jackson-databind 144 | 2.1.5 145 | 146 | 147 | junit 148 | junit 149 | 4.11 150 | test 151 | 152 | 153 | log4j 154 | log4j 155 | 1.2.17 156 | compile 157 | 158 | 159 | commons-lang 160 | commons-lang 161 | 2.6 162 | 163 | 164 | commons-cli 165 | commons-cli 166 | 1.2 167 | 168 | 169 | 170 | 171 | -------------------------------------------------------------------------------- /src/assembly/tarball.xml: -------------------------------------------------------------------------------- 1 | 5 | bin 6 | 7 | tar.gz 8 | tar.bz2 9 | zip 10 | 11 | 12 | 13 | ${project.basedir} 14 | / 15 | 16 | README* 17 | LICENSE* 18 | 19 | 20 | 21 | ${project.build.directory} 22 | / 23 | 24 | *.jar 25 | 26 | 27 | 28 | ${project.build.directory}/lib 29 | lib 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/Event.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import java.io.UnsupportedEncodingException; 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | 24 | public class Event { 25 | private Map keyValues = new HashMap(10); 26 | 27 | public Event() { 28 | } 29 | 30 | public Event(Event event) { 31 | if(event != null) { 32 | keyValues.putAll(event.keyValues); 33 | } 34 | } 35 | 36 | public Event(Map fields) throws UnsupportedEncodingException { 37 | for(String key : fields.keySet()) { 38 | addField(key, fields.get(key)); 39 | } 40 | } 41 | 42 | public Event addField(String key, byte[] value) { 43 | keyValues.put(key, value); 44 | return this; 45 | } 46 | 47 | public Event addField(String key, String value) throws UnsupportedEncodingException { 48 | keyValues.put(key, value.getBytes()); 49 | return this; 50 | } 51 | 52 | public Event addField(String key, long value) throws UnsupportedEncodingException { 53 | keyValues.put(key, String.valueOf(value).getBytes()); 54 | return this; 55 | } 56 | 57 | public Map getKeyValues() { 58 | return keyValues; 59 | } 60 | 61 | public byte[] getValue(String fieldName) { 62 | return keyValues.get(fieldName); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/FileModificationListener.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import java.io.File; 21 | 22 | import org.apache.commons.io.monitor.FileAlterationListener; 23 | import org.apache.commons.io.monitor.FileAlterationObserver; 24 | 25 | public class FileModificationListener implements FileAlterationListener { 26 | private Event fields; 27 | private FileWatcher watcher; 28 | private Multiline multiline; 29 | private Filter filter; 30 | 31 | public FileModificationListener(FileWatcher watcher, Event fields, Multiline multiline, Filter filter) { 32 | this.watcher = watcher; 33 | this.fields = fields; 34 | this.multiline = multiline; 35 | this.filter = filter; 36 | } 37 | 38 | public void onDirectoryChange(File file) { 39 | // Not implemented 40 | } 41 | 42 | public void onDirectoryCreate(File file) { 43 | // Not implemented 44 | } 45 | 46 | public void onDirectoryDelete(File file) { 47 | // Not implemented 48 | } 49 | 50 | public void onFileChange(File file) { 51 | watcher.onFileChange(file, fields, multiline, filter); 52 | } 53 | 54 | public void onFileCreate(File file) { 55 | watcher.onFileCreate(file, fields, multiline, filter); 56 | } 57 | 58 | public void onFileDelete(File file) { 59 | watcher.onFileDelete(file); 60 | } 61 | 62 | public void onStart(FileAlterationObserver file) { 63 | // Not implemented 64 | } 65 | 66 | public void onStop(FileAlterationObserver file) { 67 | // Not implemented 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/FileReader.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * Copyright 2017 Alberto González Palomo https://sentido-labs.com 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); 8 | * you may not use this file except in compliance with the License. 9 | * You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | * 19 | */ 20 | 21 | import info.fetter.logstashforwarder.util.AdapterException; 22 | import info.fetter.logstashforwarder.util.RandomAccessFile; 23 | 24 | import java.io.File; 25 | import java.io.IOException; 26 | //import java.io.RandomAccessFile; 27 | import java.nio.ByteBuffer; 28 | import java.util.Arrays; 29 | import java.util.Collection; 30 | import java.util.HashMap; 31 | import java.util.Map; 32 | import org.apache.log4j.Logger; 33 | 34 | 35 | public class FileReader extends Reader { 36 | private static Logger logger = Logger.getLogger(FileReader.class); 37 | private static final byte[] ZIP_MAGIC = new byte[] {(byte) 0x50, (byte) 0x4b, (byte) 0x03, (byte) 0x04}; 38 | private static final byte[] LZW_MAGIC = new byte[] {(byte) 0x1f, (byte) 0x9d}; 39 | private static final byte[] LZH_MAGIC = new byte[] {(byte) 0x1f, (byte) 0xa0}; 40 | private static final byte[] GZ_MAGIC = new byte[] {(byte) 0x1f, (byte) 0x8b, (byte) 0x08}; 41 | private static final byte[][] MAGICS = new byte[][] {ZIP_MAGIC, LZW_MAGIC, LZH_MAGIC, GZ_MAGIC}; 42 | private Map pointerMap; 43 | ByteBuffer bufferedLines = ByteBuffer.allocate(BYTEBUFFER_CAPACITY); 44 | 45 | public FileReader(int spoolSize) { 46 | super(spoolSize); 47 | } 48 | 49 | public int readFiles(Collection fileList) throws AdapterException { 50 | int eventCount = 0; 51 | if(logger.isTraceEnabled()) { 52 | logger.trace("Reading " + fileList.size() + " file(s)"); 53 | } 54 | pointerMap = new HashMap(fileList.size(),1); 55 | for(FileState state : fileList) { 56 | eventCount += readFile(state, spoolSize - eventCount); 57 | } 58 | if(eventCount > 0) { 59 | try { 60 | adapter.sendEvents(eventList); 61 | } catch(AdapterException e) { 62 | eventList.clear(); // Be sure no events will be sent twice after reconnect 63 | throw e; 64 | } 65 | } 66 | for(FileState state : fileList) { 67 | state.setPointer(pointerMap.get(state.getFile())); 68 | } 69 | eventList.clear(); 70 | return eventCount; // Return number of events sent to adapter 71 | } 72 | 73 | private int readFile(FileState state, int spaceLeftInSpool) { 74 | File file = state.getFile(); 75 | long pointer = state.getPointer(); 76 | int numberOfEvents = 0; 77 | try { 78 | if(state.isDeleted() || state.getRandomAccessFile() == null) { // Don't try to read this file 79 | if(logger.isTraceEnabled()) { 80 | logger.trace("File : " + file + " has been deleted"); 81 | } 82 | } else if(state.getRandomAccessFile().isEmpty()) { 83 | if(logger.isTraceEnabled()) { 84 | logger.trace("File : " + file + " is empty"); 85 | } 86 | } else { 87 | int eventListSizeBefore = eventList.size(); 88 | if(logger.isTraceEnabled()) { 89 | logger.trace("File : " + file + " pointer : " + pointer); 90 | logger.trace("Space left in spool : " + spaceLeftInSpool); 91 | } 92 | if(isCompressedFile(state)) { 93 | pointer = file.length(); 94 | } else { 95 | pointer = readLines(state, spaceLeftInSpool); 96 | } 97 | numberOfEvents = eventList.size() - eventListSizeBefore; 98 | } 99 | } catch(IOException e) { 100 | logger.warn("Exception raised while reading file : " + state.getFile(), e); 101 | } 102 | pointerMap.put(file, pointer); 103 | return numberOfEvents; // Return number of events read 104 | } 105 | 106 | private boolean isCompressedFile(FileState state) { 107 | RandomAccessFile reader = state.getRandomAccessFile(); 108 | 109 | try { 110 | for(byte[] magic : MAGICS) { 111 | byte[] fileBytes = new byte[magic.length]; 112 | reader.seek(0); 113 | int read = reader.read(fileBytes); 114 | if (read != magic.length) { 115 | continue; 116 | } 117 | if(Arrays.equals(magic, fileBytes)) { 118 | logger.debug("Compressed file detected : " + state.getFile()); 119 | return true; 120 | } 121 | } 122 | } catch(IOException e) { 123 | logger.warn("Exception raised while reading file : " + state.getFile(), e); 124 | } 125 | return false; 126 | } 127 | 128 | private static byte[] extractBytes(ByteBuffer byteBuffer) 129 | { 130 | byte[] bytes = new byte[byteBuffer.position()]; 131 | byteBuffer.rewind(); 132 | byteBuffer.get(bytes); 133 | byteBuffer.clear(); 134 | return bytes; 135 | } 136 | 137 | private static void copyLineToBuffer(byte[] line, ByteBuffer byteBuffer) { 138 | if(line.length <= byteBuffer.remaining()) { 139 | byteBuffer.put(line); 140 | } else { 141 | byteBuffer.put(line, 0, byteBuffer.remaining()); 142 | } 143 | } 144 | 145 | private long readLines(FileState state, int spaceLeftInSpool) { 146 | RandomAccessFile reader = state.getRandomAccessFile(); 147 | long pos = state.getPointer(); 148 | Multiline multiline = state.getMultiline(); 149 | try { 150 | reader.seek(pos); 151 | byte[] line = readLine(reader); 152 | bufferedLines.clear(); 153 | 154 | if(multiline != null && multiline.isPrevious()) { 155 | spaceLeftInSpool--; 156 | } 157 | while (line != null && spaceLeftInSpool > 0) { 158 | if(logger.isDebugEnabled()) { 159 | logger.debug("-- Read line : " + new String(line)); 160 | logger.debug("-- Space left in spool : " + spaceLeftInSpool); 161 | } 162 | pos = reader.getFilePointer(); 163 | if(multiline == null) { 164 | addEvent(state, pos, line); 165 | spaceLeftInSpool--; 166 | } 167 | else { 168 | if(logger.isDebugEnabled()) { 169 | logger.debug("-- Multiline : " + multiline + " matches " + multiline.isPatternFound(line)); 170 | } 171 | if(multiline.isPatternFound(line)) { 172 | // buffer the line 173 | if(bufferedLines.position() > 0 && bufferedLines.hasRemaining()) { 174 | bufferedLines.put(Multiline.JOINT); 175 | } 176 | copyLineToBuffer(line, bufferedLines); 177 | } 178 | else { 179 | if(multiline.isPrevious()) { 180 | // did not match, so new event started 181 | if (bufferedLines.position() > 0) { 182 | addEvent(state, pos, extractBytes(bufferedLines)); 183 | spaceLeftInSpool--; 184 | } 185 | copyLineToBuffer(line, bufferedLines); 186 | } 187 | else { 188 | // did not match, add the current line 189 | if(bufferedLines.position() > 0) { 190 | if(bufferedLines.hasRemaining()) { 191 | bufferedLines.put(Multiline.JOINT); 192 | } 193 | copyLineToBuffer(line, bufferedLines); 194 | addEvent(state, pos, extractBytes(bufferedLines)); 195 | spaceLeftInSpool--; 196 | } 197 | else { 198 | addEvent(state, pos, line); 199 | spaceLeftInSpool--; 200 | } 201 | } 202 | } 203 | } 204 | line = readLine(reader); 205 | } 206 | if(bufferedLines.position() > 0) { 207 | addEvent(state, pos, extractBytes(bufferedLines)); // send any buffered lines left 208 | } 209 | reader.seek(pos); // Ensure we can re-read if necessary 210 | } catch(IOException e) { 211 | logger.warn("Exception raised while reading file : " + state.getFile(), e); 212 | } 213 | return pos; 214 | } 215 | 216 | private byte[] readLine(RandomAccessFile reader) throws IOException { 217 | byteBuffer.clear(); 218 | int ch; 219 | boolean seenCR = false; 220 | while((ch=reader.read()) != -1) { 221 | switch(ch) { 222 | case '\n': 223 | return extractBytes(byteBuffer); 224 | case '\r': 225 | seenCR = true; 226 | break; 227 | default: 228 | if (seenCR) { 229 | if(byteBuffer.hasRemaining()) { 230 | byteBuffer.put((byte) '\r'); 231 | } 232 | seenCR = false; 233 | } 234 | if(byteBuffer.hasRemaining()) { 235 | byteBuffer.put((byte)ch); 236 | } 237 | } 238 | } 239 | return null; 240 | } 241 | 242 | } 243 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/FileSigner.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | import info.fetter.logstashforwarder.util.RandomAccessFile; 4 | 5 | import java.io.IOException; 6 | //import java.io.RandomAccessFile; 7 | import java.util.zip.Adler32; 8 | 9 | 10 | public class FileSigner { 11 | private static final Adler32 adler32 = new Adler32(); 12 | 13 | public static long computeSignature(RandomAccessFile file, int signatureLength) throws IOException { 14 | adler32.reset(); 15 | byte[] input = new byte[signatureLength]; 16 | file.seek(0); 17 | file.read(input); 18 | adler32.update(input); 19 | return adler32.getValue(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/FileState.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import info.fetter.logstashforwarder.util.RandomAccessFile; 21 | 22 | import java.io.File; 23 | import java.io.FileNotFoundException; 24 | import java.io.IOException; 25 | //import java.io.RandomAccessFile; 26 | 27 | 28 | 29 | import org.apache.commons.lang.builder.ToStringBuilder; 30 | 31 | import com.fasterxml.jackson.annotation.JsonIgnore; 32 | import java.util.Map; 33 | 34 | public class FileState { 35 | @JsonIgnore 36 | private File file; 37 | private String directory; 38 | private String fileName; 39 | @JsonIgnore 40 | private long lastModified; 41 | @JsonIgnore 42 | private long size; 43 | @JsonIgnore 44 | private boolean deleted = false; 45 | private long signature; 46 | private int signatureLength; 47 | @JsonIgnore 48 | private boolean changed = false; 49 | @JsonIgnore 50 | private RandomAccessFile randomAccessFile; 51 | private long pointer = 0; 52 | @JsonIgnore 53 | private FileState oldFileState; 54 | @JsonIgnore 55 | private Event fields; 56 | @JsonIgnore 57 | private Multiline multiline; 58 | @JsonIgnore 59 | private Filter filter; 60 | @JsonIgnore 61 | private boolean matchedToNewFile = false; 62 | 63 | public FileState() { 64 | } 65 | 66 | public FileState(File file) throws IOException { 67 | this.file = file; 68 | directory = file.getCanonicalFile().getParent(); 69 | fileName = file.getName(); 70 | randomAccessFile = new RandomAccessFile(file.getPath(), "r"); 71 | lastModified = file.lastModified(); 72 | size = file.length(); 73 | } 74 | 75 | private void setFileFromDirectoryAndName() throws FileNotFoundException { 76 | file = new File(directory + File.separator + fileName); 77 | if(file.exists()) { 78 | randomAccessFile = null; 79 | lastModified = file.lastModified(); 80 | size = file.length(); 81 | } else { 82 | deleted = true; 83 | } 84 | } 85 | 86 | public File getFile() { 87 | return file; 88 | } 89 | 90 | public long getLastModified() { 91 | return lastModified; 92 | } 93 | 94 | public long getSize() { 95 | return size; 96 | } 97 | 98 | public String getDirectory() { 99 | return directory; 100 | } 101 | 102 | public void setDirectory(String directory) throws FileNotFoundException { 103 | this.directory = directory; 104 | if(fileName != null && directory != null) { 105 | setFileFromDirectoryAndName(); 106 | } 107 | } 108 | 109 | public String getFileName() { 110 | return fileName; 111 | } 112 | 113 | public void setFileName(String fileName) throws FileNotFoundException { 114 | this.fileName = fileName; 115 | if(fileName != null && directory != null) { 116 | setFileFromDirectoryAndName(); 117 | } 118 | } 119 | 120 | public boolean isDeleted() { 121 | return deleted; 122 | } 123 | 124 | public void setDeleted() { 125 | deleted = true; 126 | } 127 | 128 | public boolean hasChanged() { 129 | return changed; 130 | } 131 | 132 | public void setChanged(boolean changed) { 133 | this.changed = changed; 134 | } 135 | 136 | public long getSignature() { 137 | return signature; 138 | } 139 | 140 | public void setSignature(long signature) { 141 | this.signature = signature; 142 | } 143 | 144 | public RandomAccessFile getRandomAccessFile() { 145 | return randomAccessFile; 146 | } 147 | 148 | public long getPointer() { 149 | return pointer; 150 | } 151 | 152 | public void setPointer(long pointer) { 153 | this.pointer = pointer; 154 | } 155 | 156 | public int getSignatureLength() { 157 | return signatureLength; 158 | } 159 | 160 | public void setSignatureLength(int signatureLength) { 161 | this.signatureLength = signatureLength; 162 | } 163 | 164 | public FileState getOldFileState() { 165 | return oldFileState; 166 | } 167 | 168 | public void setOldFileState(FileState oldFileState) { 169 | this.oldFileState = oldFileState; 170 | oldFileState.setMatchedToNewFile(true); 171 | } 172 | 173 | public void deleteOldFileState() { 174 | try { 175 | oldFileState.getRandomAccessFile().close(); 176 | oldFileState = null; 177 | } catch(Exception e) {} 178 | } 179 | 180 | public Event getFields() { 181 | return fields; 182 | } 183 | 184 | public void setFields(Event fields) { 185 | this.fields = fields; 186 | } 187 | 188 | public Multiline getMultiline() { 189 | return multiline; 190 | } 191 | 192 | public void setMultiline(Multiline multiline) { 193 | this.multiline = multiline; 194 | } 195 | 196 | public Filter getFilter() { 197 | return filter; 198 | } 199 | 200 | public void setFilter(Filter filter) { 201 | this.filter = filter; 202 | } 203 | 204 | public boolean isMatchedToNewFile() { 205 | return matchedToNewFile; 206 | } 207 | 208 | public void setMatchedToNewFile(boolean matchedToNewFile) { 209 | this.matchedToNewFile = matchedToNewFile; 210 | } 211 | 212 | @Override 213 | public String toString() { 214 | return new ToStringBuilder(this). 215 | append("fileName", fileName). 216 | append("directory", directory). 217 | append("pointer", pointer). 218 | append("signature", signature). 219 | append("signatureLength", signatureLength). 220 | toString(); 221 | } 222 | 223 | } 224 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/FileWatcher.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import info.fetter.logstashforwarder.util.AdapterException; 21 | import info.fetter.logstashforwarder.util.LastModifiedFileFilter; 22 | 23 | import java.io.File; 24 | import java.io.IOException; 25 | import java.util.ArrayList; 26 | import java.util.HashMap; 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | import org.apache.commons.io.FileUtils; 31 | import org.apache.commons.io.FilenameUtils; 32 | import org.apache.commons.io.filefilter.FileFilterUtils; 33 | import org.apache.commons.io.filefilter.IOFileFilter; 34 | import org.apache.commons.io.filefilter.WildcardFileFilter; 35 | import org.apache.commons.io.monitor.FileAlterationObserver; 36 | import org.apache.log4j.Logger; 37 | 38 | public class FileWatcher { 39 | private static final Logger logger = Logger.getLogger(FileWatcher.class); 40 | private List observerList = new ArrayList(); 41 | public static final int ONE_DAY = 24 * 3600 * 1000; 42 | private Map oldWatchMap = new HashMap(); 43 | private Map newWatchMap = new HashMap(); 44 | private FileState[] savedStates; 45 | private int maxSignatureLength; 46 | private boolean tail = false; 47 | private Event stdinFields; 48 | private boolean stdinConfigured = false; 49 | private String sincedbFile = null; 50 | 51 | public FileWatcher() { 52 | } 53 | 54 | public void initialize() throws IOException { 55 | logger.debug("Initializing FileWatcher"); 56 | if(savedStates != null) { 57 | for(FileState state : savedStates) { 58 | logger.info("Loading file state: " + state.getFile() + ":" + state.getPointer()); 59 | oldWatchMap.put(state.getFile(), state); 60 | } 61 | } 62 | processModifications(); 63 | if(tail) { 64 | for(FileState state : oldWatchMap.values()) { 65 | if(state.getPointer() == 0) { 66 | state.setPointer(state.getSize()); 67 | } 68 | } 69 | } 70 | printWatchMap(); 71 | } 72 | 73 | public void addFilesToWatch(String fileToWatch, Event fields, long deadTime, Multiline multiline, Filter filter) { 74 | try { 75 | if(fileToWatch.equals("-")) { 76 | addStdIn(fields); 77 | } else if(fileToWatch.contains("*")) { 78 | addWildCardFiles(fileToWatch, fields, deadTime, multiline, filter); 79 | } else { 80 | addSingleFile(fileToWatch, fields, deadTime, multiline, filter); 81 | } 82 | } catch(Exception e) { 83 | throw new RuntimeException(e); 84 | } 85 | } 86 | 87 | public void checkFiles() throws IOException { 88 | logger.trace("Checking files"); 89 | logger.trace("=============="); 90 | for(FileAlterationObserver observer : observerList) { 91 | observer.checkAndNotify(); 92 | } 93 | processModifications(); 94 | printWatchMap(); 95 | } 96 | 97 | public int readFiles(FileReader reader) throws IOException, AdapterException { 98 | logger.trace("Reading files"); 99 | logger.trace("=============="); 100 | int numberOfLinesRead = reader.readFiles(oldWatchMap.values()); 101 | Registrar.writeStateToJson(sincedbFile,oldWatchMap.values()); 102 | return numberOfLinesRead; 103 | } 104 | 105 | public int readStdin(InputReader reader) throws AdapterException, IOException { 106 | if(stdinConfigured) { 107 | logger.debug("Reading stdin"); 108 | reader.setFields(stdinFields); 109 | int numberOfLinesRead = reader.readInput(); 110 | return numberOfLinesRead; 111 | } else { 112 | return 0; 113 | } 114 | } 115 | 116 | private void processModifications() throws IOException { 117 | 118 | for(File file : newWatchMap.keySet()) { 119 | FileState state = newWatchMap.get(file); 120 | if(logger.isTraceEnabled()) { 121 | logger.trace("Checking file : " + file.getCanonicalPath()); 122 | logger.trace("-- Last modified : " + state.getLastModified()); 123 | logger.trace("-- Size : " + state.getSize()); 124 | logger.trace("-- Directory : " + state.getDirectory()); 125 | logger.trace("-- Filename : " + state.getFileName()); 126 | } 127 | 128 | logger.trace("Determine if file has just been written to"); 129 | FileState oldState = oldWatchMap.get(file); 130 | if(oldState != null) { 131 | if(oldState.getSize() > state.getSize()) { 132 | logger.trace("File shorter : file can't be the same"); 133 | } else { 134 | if(oldState.getSignatureLength() == state.getSignatureLength() && oldState.getSignature() == state.getSignature()) { 135 | state.setOldFileState(oldState); 136 | logger.trace("Same signature size and value : file is the same"); 137 | continue; 138 | } else if(oldState.getSignatureLength() < state.getSignatureLength()){ 139 | long signature = FileSigner.computeSignature(state.getRandomAccessFile(), oldState.getSignatureLength()); 140 | if(signature == oldState.getSignature()) { 141 | state.setOldFileState(oldState); 142 | logger.trace("Same signature : file is the same"); 143 | continue; 144 | } else { 145 | logger.trace("Signature different : file can't be the same"); 146 | } 147 | } else if(oldState.getSignatureLength() > state.getSignatureLength()){ 148 | logger.trace("Signature shorter : file can't be the same"); 149 | } 150 | } 151 | } 152 | 153 | if(state.getOldFileState() == null) { 154 | logger.trace("Determine if file has been renamed and/or written to"); 155 | for(File otherFile : oldWatchMap.keySet()) { 156 | FileState otherState = oldWatchMap.get(otherFile); 157 | if(otherState != null && state.getSize() >= otherState.getSize() && state.getDirectory().equals(otherState.getDirectory())) { 158 | if(logger.isTraceEnabled()) { 159 | logger.trace("Comparing to : " + otherFile.getCanonicalPath()); 160 | } 161 | if(otherState.getSignatureLength() == state.getSignatureLength() && otherState.getSignature() == state.getSignature()) { 162 | state.setOldFileState(otherState); 163 | logger.trace("Same signature size and value : file is the same"); 164 | break; 165 | } else if(otherState.getSignatureLength() < state.getSignatureLength()){ 166 | long signature = FileSigner.computeSignature(state.getRandomAccessFile(), otherState.getSignatureLength()); 167 | if(signature == otherState.getSignature()) { 168 | state.setOldFileState(otherState); 169 | logger.trace("Same signature : file is the same"); 170 | break; 171 | } else { 172 | logger.trace("Signature different : file can't be the same"); 173 | } 174 | } else if(otherState.getSignatureLength() > state.getSignatureLength()){ 175 | logger.trace("Signature shorter : file can't be the same"); 176 | } 177 | } 178 | } 179 | } 180 | } 181 | 182 | for(FileState state : newWatchMap.values()) { 183 | if(logger.isTraceEnabled()) { 184 | logger.trace("Refreshing file state: " + state.getFile()); 185 | } 186 | FileState oldState = state.getOldFileState(); 187 | if(oldState == null) { 188 | if(logger.isDebugEnabled()) { 189 | logger.debug("File " + state.getFile() + " has been truncated or created, not retrieving pointer"); 190 | } 191 | oldState = oldWatchMap.get(state.getFile()); 192 | if(oldState != null && ! oldState.isMatchedToNewFile()) { 193 | if(logger.isDebugEnabled()) { 194 | logger.debug("File " + state.getFile() + " has been replaced and not renamed, removing from watchMap"); 195 | } 196 | try { 197 | oldState.getRandomAccessFile().close(); 198 | } catch(Exception e) {} 199 | oldWatchMap.remove(state.getFile()); 200 | } 201 | } else { 202 | if(logger.isInfoEnabled() && ! state.getFileName().equals(oldState.getFileName())) 203 | { 204 | logger.info("File rename was detected: " + oldState.getFile() + " -> " + state.getFile()); 205 | } 206 | if(logger.isDebugEnabled()) { 207 | logger.debug("File " + state.getFile() + " has not been truncated or created, retrieving pointer: " + oldState.getPointer()); 208 | } 209 | state.setPointer(oldState.getPointer()); 210 | state.deleteOldFileState(); 211 | } 212 | } 213 | 214 | logger.trace("Replacing old state"); 215 | for(File file : newWatchMap.keySet()) { 216 | FileState state = newWatchMap.get(file); 217 | oldWatchMap.put(file, state); 218 | } 219 | 220 | // Truncating changedWatchMap 221 | newWatchMap.clear(); 222 | 223 | removeMarkedFilesFromWatchMap(); 224 | } 225 | 226 | private void addSingleFile(String fileToWatch, Event fields, long deadTime, Multiline multiline, Filter filter) throws Exception { 227 | logger.info("Watching file : " + new File(fileToWatch).getCanonicalPath()); 228 | String directory = FilenameUtils.getFullPath(fileToWatch); 229 | String fileName = FilenameUtils.getName(fileToWatch); 230 | IOFileFilter fileFilter = FileFilterUtils.and( 231 | FileFilterUtils.fileFileFilter(), 232 | FileFilterUtils.nameFileFilter(fileName), 233 | new LastModifiedFileFilter(deadTime)); 234 | initializeWatchMap(new File(directory), fileFilter, fields, multiline, filter); 235 | } 236 | 237 | private void addWildCardFiles(String filesToWatch, Event fields, long deadTime, Multiline multiline, Filter filter) throws Exception { 238 | logger.info("Watching wildcard files : " + filesToWatch); 239 | String directory = FilenameUtils.getFullPath(filesToWatch); 240 | String wildcard = FilenameUtils.getName(filesToWatch); 241 | logger.trace("Directory : " + new File(directory).getCanonicalPath() + ", wildcard : " + wildcard); 242 | IOFileFilter fileFilter = FileFilterUtils.and( 243 | FileFilterUtils.fileFileFilter(), 244 | new WildcardFileFilter(wildcard), 245 | new LastModifiedFileFilter(deadTime)); 246 | initializeWatchMap(new File(directory), fileFilter, fields, multiline, filter); 247 | } 248 | 249 | private void addStdIn(Event fields) { 250 | logger.error("Watching stdin"); 251 | stdinFields = fields; 252 | stdinConfigured = true; 253 | } 254 | 255 | private void initializeWatchMap(File directory, IOFileFilter fileFilter, Event fields, Multiline multiline, Filter filter) throws Exception { 256 | if(!directory.isDirectory()) { 257 | logger.warn("Directory " + directory + " does not exist"); 258 | return; 259 | } 260 | FileAlterationObserver observer = new FileAlterationObserver(directory, fileFilter); 261 | FileModificationListener listener = new FileModificationListener(this, fields, multiline, filter); 262 | observer.addListener(listener); 263 | observerList.add(observer); 264 | observer.initialize(); 265 | for(File file : FileUtils.listFiles(directory, fileFilter, null)) { 266 | addFileToWatchMap(newWatchMap, file, fields, multiline, filter); 267 | } 268 | } 269 | 270 | private void addFileToWatchMap(Map map, File file, Event fields, Multiline multiline, Filter filter) { 271 | try { 272 | FileState state = new FileState(file); 273 | state.setFields(fields); 274 | int signatureLength = (int) (state.getSize() > maxSignatureLength ? maxSignatureLength : state.getSize()); 275 | state.setSignatureLength(signatureLength); 276 | long signature = FileSigner.computeSignature(state.getRandomAccessFile(), signatureLength); 277 | state.setSignature(signature); 278 | logger.trace("Setting signature of size : " + signatureLength + " on file : " + file + " : " + signature); 279 | state.setMultiline(multiline); 280 | state.setFilter(filter); 281 | map.put(file, state); 282 | } catch(IOException e) { 283 | logger.error("Caught IOException in addFileToWatchMap : " + 284 | e.getMessage()); 285 | } 286 | } 287 | 288 | public void onFileChange(File file, Event fields, Multiline multiline, Filter filter) { 289 | try { 290 | logger.debug("Change detected on file : " + file.getCanonicalPath()); 291 | addFileToWatchMap(newWatchMap, file, fields, multiline, filter); 292 | } catch (IOException e) { 293 | logger.error("Caught IOException in onFileChange : " + 294 | e.getMessage()); 295 | } 296 | } 297 | 298 | public void onFileCreate(File file, Event fields, Multiline multiline, Filter filter) { 299 | try { 300 | logger.debug("Create detected on file : " + file.getCanonicalPath()); 301 | addFileToWatchMap(newWatchMap, file, fields, multiline, filter); 302 | } catch (IOException e) { 303 | logger.error("Caught IOException in onFileCreate : " + 304 | e.getMessage()); 305 | } 306 | } 307 | 308 | public void onFileDelete(File file) { 309 | try { 310 | logger.debug("Delete detected on file : " + file.getCanonicalPath()); 311 | FileState state = oldWatchMap.get(file); 312 | if (state != null) state.setDeleted(); 313 | } catch (IOException e) { 314 | logger.error("Caught IOException in onFileDelete: " + 315 | e.getMessage()); 316 | } 317 | } 318 | 319 | private void printWatchMap() throws IOException { 320 | if(logger.isTraceEnabled()) { 321 | logger.trace("WatchMap contents : "); 322 | for(File file : oldWatchMap.keySet()) { 323 | FileState state = oldWatchMap.get(file); 324 | logger.trace("\tFile : " + file.getCanonicalPath() + " marked for deletion : " + state.isDeleted()); 325 | } 326 | } 327 | } 328 | 329 | private void removeMarkedFilesFromWatchMap() throws IOException { 330 | logger.trace("Removing deleted files from watchMap"); 331 | List markedList = null; 332 | for(File file : oldWatchMap.keySet()) { 333 | FileState state = oldWatchMap.get(file); 334 | if(state.getRandomAccessFile() == null) { 335 | state.setDeleted(); 336 | } 337 | if(state.isDeleted()) { 338 | if(! file.exists()) { 339 | if(markedList == null) { 340 | markedList = new ArrayList(); 341 | } 342 | markedList.add(file); 343 | } 344 | try { 345 | state.getRandomAccessFile().close(); 346 | } catch(Exception e) {} 347 | } 348 | } 349 | if(markedList != null) { 350 | for(File file : markedList) { 351 | oldWatchMap.remove(file); 352 | logger.debug("File " + file + " removed from watchMap"); 353 | } 354 | } 355 | } 356 | 357 | public void close() throws IOException { 358 | logger.debug("Closing all files"); 359 | for(File file : oldWatchMap.keySet()) { 360 | FileState state = oldWatchMap.get(file); 361 | state.getRandomAccessFile().close(); 362 | } 363 | } 364 | 365 | public int getMaxSignatureLength() { 366 | return maxSignatureLength; 367 | } 368 | 369 | public void setMaxSignatureLength(int maxSignatureLength) { 370 | this.maxSignatureLength = maxSignatureLength; 371 | } 372 | 373 | public void setTail(boolean tail) { 374 | this.tail = tail; 375 | } 376 | 377 | public void setSincedb(String sincedbFile) { 378 | this.sincedbFile = sincedbFile; 379 | try { 380 | logger.debug("Loading saved states"); 381 | savedStates = Registrar.readStateFromJson(sincedbFile); 382 | } catch(Exception e) { 383 | logger.warn("Could not load saved states : " + e.getMessage(), e); 384 | } 385 | } 386 | 387 | } 388 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/Filter.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2017 Alberto González Palomo http://sentido-labs.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import java.io.UnsupportedEncodingException; 21 | import java.util.Map; 22 | import java.util.regex.Pattern; 23 | import org.apache.commons.lang.builder.ToStringBuilder; 24 | 25 | public class Filter { 26 | private Pattern pattern = null; 27 | private boolean negate = false; 28 | private String charset = "UTF-8"; 29 | 30 | public Filter() { 31 | } 32 | 33 | public Filter(Filter event) { 34 | if(event != null) { 35 | this.negate = event.negate; 36 | this.pattern = event.pattern; 37 | } 38 | } 39 | 40 | public Filter(Map fields) throws UnsupportedEncodingException { 41 | String strPattern = ""; 42 | for(String key : fields.keySet()) { 43 | if ("pattern".equals(key)) 44 | strPattern = fields.get(key); 45 | else if ("negate".equals(key)) 46 | negate = Boolean.parseBoolean(fields.get(key)); 47 | else 48 | throw new UnsupportedEncodingException(key + " not supported"); 49 | } 50 | pattern = Pattern.compile(strPattern); 51 | 52 | } 53 | 54 | public Pattern getPattern() { 55 | return pattern; 56 | } 57 | 58 | public boolean isNegate() { 59 | return negate; 60 | } 61 | 62 | public boolean accept (byte[] line) { 63 | try { 64 | return accept(new String(line, charset)); 65 | } catch (UnsupportedEncodingException e) { 66 | System.err.println("ERROR: unsupported encoding " + charset); 67 | System.err.flush();// In case we crash at new String(line), 68 | // because the behaviour if the bytes are not decodable with 69 | // the platform's default encoding is undefined. 70 | // Last ditch effort, decode with the platform's encoding: 71 | return accept(new String(line)); 72 | } 73 | } 74 | 75 | public boolean accept (String line) { 76 | boolean result = pattern.matcher(line).find(); 77 | if (negate) return !result; 78 | return result; 79 | } 80 | 81 | @Override 82 | public String toString() { 83 | return new ToStringBuilder(this). 84 | append("pattern", pattern). 85 | append("negate", negate). 86 | toString(); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/Forwarder.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import static org.apache.log4j.Level.*; 21 | 22 | import java.io.IOException; 23 | import java.util.List; 24 | import java.util.Random; 25 | 26 | import info.fetter.logstashforwarder.config.ConfigurationManager; 27 | import info.fetter.logstashforwarder.config.FilesSection; 28 | import info.fetter.logstashforwarder.protocol.LumberjackClient; 29 | import info.fetter.logstashforwarder.util.AdapterException; 30 | 31 | import org.apache.commons.cli.CommandLine; 32 | import org.apache.commons.cli.CommandLineParser; 33 | import org.apache.commons.cli.GnuParser; 34 | import org.apache.commons.cli.HelpFormatter; 35 | import org.apache.commons.cli.Option; 36 | import org.apache.commons.cli.OptionBuilder; 37 | import org.apache.commons.cli.Options; 38 | import org.apache.commons.cli.ParseException; 39 | import org.apache.log4j.Appender; 40 | import org.apache.log4j.BasicConfigurator; 41 | import org.apache.log4j.ConsoleAppender; 42 | import org.apache.log4j.Layout; 43 | import org.apache.log4j.Level; 44 | import org.apache.log4j.Logger; 45 | import org.apache.log4j.PatternLayout; 46 | import org.apache.log4j.RollingFileAppender; 47 | import org.apache.log4j.spi.RootLogger; 48 | 49 | public class Forwarder { 50 | private static final String SINCEDB = ".logstash-forwarder-java"; 51 | private static Logger logger = Logger.getLogger(Forwarder.class); 52 | private static int spoolSize = 1024; 53 | private static int idleTimeout = 5000; 54 | private static int networkTimeout = 15000; 55 | private static String config; 56 | private static ConfigurationManager configManager; 57 | private static FileWatcher watcher; 58 | private static FileReader fileReader; 59 | private static InputReader inputReader; 60 | private static Level logLevel = INFO; 61 | private static boolean debugWatcherSelected = false; 62 | private static ProtocolAdapter adapter; 63 | private static Random random = new Random(); 64 | private static int signatureLength = 4096; 65 | private static boolean tailSelected = false; 66 | private static String logfile = null; 67 | private static String logfileSize = "10MB"; 68 | private static int logfileNumber = 5; 69 | private static String sincedbFile = SINCEDB; 70 | 71 | public static void main(String[] args) { 72 | try { 73 | parseOptions(args); 74 | setupLogging(); 75 | watcher = new FileWatcher(); 76 | watcher.setMaxSignatureLength(signatureLength); 77 | watcher.setTail(tailSelected); 78 | watcher.setSincedb(sincedbFile); 79 | configManager = new ConfigurationManager(config); 80 | configManager.readConfiguration(); 81 | for(FilesSection files : configManager.getConfig().getFiles()) { 82 | for(String path : files.getPaths()) { 83 | watcher.addFilesToWatch(path, new Event(files.getFields()), files.getDeadTimeInSeconds() * 1000, files.getMultiline(), files.getFilter()); 84 | } 85 | } 86 | watcher.initialize(); 87 | fileReader = new FileReader(spoolSize); 88 | inputReader = new InputReader(spoolSize, System.in); 89 | connectToServer(); 90 | infiniteLoop(); 91 | } catch(Exception e) { 92 | e.printStackTrace(); 93 | System.exit(3); 94 | } 95 | } 96 | 97 | private static void infiniteLoop() throws IOException, InterruptedException { 98 | while(true) { 99 | try { 100 | watcher.checkFiles(); 101 | while(watcher.readFiles(fileReader) == spoolSize); 102 | while(watcher.readStdin(inputReader) == spoolSize); 103 | Thread.sleep(idleTimeout); 104 | } catch(AdapterException e) { 105 | logger.error("Lost server connection"); 106 | Thread.sleep(networkTimeout); 107 | connectToServer(); 108 | } 109 | } 110 | } 111 | 112 | private static void connectToServer() { 113 | int randomServerIndex = 0; 114 | List serverList = configManager.getConfig().getNetwork().getServers(); 115 | networkTimeout = configManager.getConfig().getNetwork().getTimeout() * 1000; 116 | if(adapter != null) { 117 | try { 118 | adapter.close(); 119 | } catch(AdapterException e) { 120 | logger.error("Error while closing connection to " + adapter.getServer() + ":" + adapter.getPort()); 121 | } finally { 122 | adapter = null; 123 | } 124 | } 125 | while(adapter == null) { 126 | try { 127 | randomServerIndex = random.nextInt(serverList.size()); 128 | String[] serverAndPort = serverList.get(randomServerIndex).split(":"); 129 | logger.info("Trying to connect to " + serverList.get(randomServerIndex)); 130 | adapter = new LumberjackClient(configManager.getConfig().getNetwork().getSslCA(),serverAndPort[0],Integer.parseInt(serverAndPort[1]), networkTimeout); 131 | fileReader.setAdapter(adapter); 132 | inputReader.setAdapter(adapter); 133 | } catch(Exception ex) { 134 | if(logger.isDebugEnabled()) { 135 | logger.error("Failed to connect to server " + serverList.get(randomServerIndex) + " : ", ex); 136 | } else { 137 | logger.error("Failed to connect to server " + serverList.get(randomServerIndex) + " : " + ex.getMessage()); 138 | } 139 | try { 140 | Thread.sleep(networkTimeout); 141 | } catch (InterruptedException e) { 142 | logger.error(e.getMessage()); 143 | } 144 | } 145 | } 146 | } 147 | 148 | @SuppressWarnings("static-access") 149 | static void parseOptions(String[] args) { 150 | Options options = new Options(); 151 | Option helpOption = new Option("help", "print this message"); 152 | Option quietOption = new Option("quiet", "operate in quiet mode - only emit errors to log"); 153 | Option debugOption = new Option("debug", "operate in debug mode"); 154 | Option debugWatcherOption = new Option("debugwatcher", "operate watcher in debug mode"); 155 | Option traceOption = new Option("trace", "operate in trace mode"); 156 | Option tailOption = new Option("tail", "read new files from the end"); 157 | 158 | Option spoolSizeOption = OptionBuilder.withArgName("number of events") 159 | .hasArg() 160 | .withDescription("event count spool threshold - forces network flush") 161 | .create("spoolsize"); 162 | Option idleTimeoutOption = OptionBuilder.withArgName("") 163 | .hasArg() 164 | .withDescription("time between file reads in seconds") 165 | .create("idletimeout"); 166 | Option configOption = OptionBuilder.withArgName("config file") 167 | .hasArg() 168 | .isRequired() 169 | .withDescription("path to logstash-forwarder configuration file") 170 | .create("config"); 171 | Option signatureLengthOption = OptionBuilder.withArgName("signature length") 172 | .hasArg() 173 | .withDescription("Maximum length of file signature") 174 | .create("signaturelength"); 175 | Option logfileOption = OptionBuilder.withArgName("logfile name") 176 | .hasArg() 177 | .withDescription("Logfile name") 178 | .create("logfile"); 179 | Option logfileSizeOption = OptionBuilder.withArgName("logfile size") 180 | .hasArg() 181 | .withDescription("Logfile size (default 10M)") 182 | .create("logfilesize"); 183 | Option logfileNumberOption = OptionBuilder.withArgName("number of logfiles") 184 | .hasArg() 185 | .withDescription("Number of logfiles (default 5)") 186 | .create("logfilenumber"); 187 | Option sincedbOption = OptionBuilder.withArgName("sincedb file") 188 | .hasArg() 189 | .withDescription("Sincedb file name") 190 | .create("sincedb"); 191 | 192 | options.addOption(helpOption) 193 | .addOption(idleTimeoutOption) 194 | .addOption(spoolSizeOption) 195 | .addOption(quietOption) 196 | .addOption(debugOption) 197 | .addOption(debugWatcherOption) 198 | .addOption(traceOption) 199 | .addOption(tailOption) 200 | .addOption(signatureLengthOption) 201 | .addOption(configOption) 202 | .addOption(logfileOption) 203 | .addOption(logfileNumberOption) 204 | .addOption(logfileSizeOption) 205 | .addOption(sincedbOption); 206 | 207 | CommandLineParser parser = new GnuParser(); 208 | try { 209 | CommandLine line = parser.parse(options, args); 210 | if(line.hasOption("spoolsize")) { 211 | spoolSize = Integer.parseInt(line.getOptionValue("spoolsize")); 212 | } 213 | if(line.hasOption("idletimeout")) { 214 | idleTimeout = Integer.parseInt(line.getOptionValue("idletimeout")); 215 | } 216 | if(line.hasOption("config")) { 217 | config = line.getOptionValue("config"); 218 | } 219 | if(line.hasOption("signaturelength")) { 220 | signatureLength = Integer.parseInt(line.getOptionValue("signaturelength")); 221 | } 222 | if(line.hasOption("quiet")) { 223 | logLevel = ERROR; 224 | } 225 | if(line.hasOption("debug")) { 226 | logLevel = DEBUG; 227 | } 228 | if(line.hasOption("trace")) { 229 | logLevel = TRACE; 230 | } 231 | if(line.hasOption("debugwatcher")) { 232 | debugWatcherSelected = true; 233 | } 234 | if(line.hasOption("tail")) { 235 | tailSelected = true; 236 | } 237 | if(line.hasOption("logfile")) { 238 | logfile = line.getOptionValue("logfile"); 239 | } 240 | if(line.hasOption("logfilesize")) { 241 | logfileSize = line.getOptionValue("logfilesize"); 242 | } 243 | if(line.hasOption("logfilenumber")) { 244 | logfileNumber = Integer.parseInt(line.getOptionValue("logfilenumber")); 245 | } 246 | if(line.hasOption("sincedb")) { 247 | sincedbFile = line.getOptionValue("sincedb"); 248 | } 249 | } catch(ParseException e) { 250 | printHelp(options); 251 | System.exit(1);; 252 | } catch(NumberFormatException e) { 253 | System.err.println("Value must be an integer"); 254 | printHelp(options); 255 | System.exit(2);; 256 | } 257 | } 258 | 259 | private static void printHelp(Options options) { 260 | HelpFormatter formatter = new HelpFormatter(); 261 | formatter.printHelp("logstash-forwarder", options); 262 | } 263 | 264 | private static void setupLogging() throws IOException { 265 | Appender appender; 266 | Layout layout = new PatternLayout("%d %p %c{1} - %m%n"); 267 | if(logfile == null) { 268 | appender = new ConsoleAppender(layout); 269 | } else { 270 | RollingFileAppender rolling = new RollingFileAppender(layout, logfile, true); 271 | rolling.setMaxFileSize(logfileSize); 272 | rolling.setMaxBackupIndex(logfileNumber); 273 | appender = rolling; 274 | } 275 | BasicConfigurator.configure(appender); 276 | RootLogger.getRootLogger().setLevel(logLevel); 277 | if(debugWatcherSelected) { 278 | Logger.getLogger(FileWatcher.class).addAppender(appender); 279 | Logger.getLogger(FileWatcher.class).setLevel(DEBUG); 280 | Logger.getLogger(FileWatcher.class).setAdditivity(false); 281 | } 282 | // Logger.getLogger(FileReader.class).addAppender((Appender)RootLogger.getRootLogger().getAllAppenders().nextElement()); 283 | // Logger.getLogger(FileReader.class).setLevel(TRACE); 284 | // Logger.getLogger(FileReader.class).setAdditivity(false); 285 | } 286 | 287 | } 288 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/InputReader.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import info.fetter.logstashforwarder.util.AdapterException; 21 | 22 | import java.io.BufferedReader; 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | import java.io.InputStreamReader; 26 | 27 | import org.apache.log4j.Logger; 28 | 29 | public class InputReader extends Reader { 30 | private static Logger logger = Logger.getLogger(InputReader.class); 31 | private BufferedReader reader; 32 | private long position = 0; 33 | private Event fields; 34 | 35 | public InputReader(int spoolSize, InputStream in) { 36 | super(spoolSize); 37 | reader = new BufferedReader(new InputStreamReader(in)); 38 | } 39 | 40 | public int readInput() throws AdapterException, IOException { 41 | int eventCount = 0; 42 | logger.trace("Reading stdin"); 43 | 44 | eventCount += readLines(); 45 | 46 | if(eventCount > 0) { 47 | adapter.sendEvents(eventList); 48 | } 49 | 50 | eventList.clear(); 51 | return eventCount; 52 | } 53 | 54 | private int readLines() throws IOException { 55 | int lineCount = 0; 56 | byte[] line; 57 | while(lineCount < spoolSize && (line = readLine()) != null) { 58 | position += line.length; 59 | lineCount++; 60 | addEvent("stdin", fields, position, line); 61 | } 62 | return lineCount; 63 | } 64 | 65 | private byte[] readLine() throws IOException { 66 | int ch; 67 | boolean seenCR = false; 68 | while(reader.ready()) { 69 | ch=reader.read(); 70 | switch(ch) { 71 | case '\n': 72 | byte[] line = new byte[byteBuffer.position()]; 73 | byteBuffer.rewind(); 74 | byteBuffer.get(line); 75 | byteBuffer.clear(); 76 | return line; 77 | case '\r': 78 | seenCR = true; 79 | break; 80 | default: 81 | if (seenCR) { 82 | byteBuffer.put((byte) '\r'); 83 | seenCR = false; 84 | } 85 | byteBuffer.put((byte)ch); 86 | } 87 | } 88 | return null; 89 | } 90 | 91 | public void setFields(Event fields) { 92 | this.fields = fields; 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/Multiline.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import java.io.UnsupportedEncodingException; 21 | import java.util.Map; 22 | import java.util.regex.Pattern; 23 | import org.apache.commons.lang.builder.ToStringBuilder; 24 | 25 | public class Multiline { 26 | public enum WhatType { previous, next }; 27 | public static byte JOINT = (byte) '\n'; 28 | 29 | private Pattern pattern = null; 30 | private boolean negate = false; 31 | private WhatType what = WhatType.previous; 32 | 33 | public Multiline() { 34 | } 35 | 36 | public Multiline(Multiline event) { 37 | if(event != null) { 38 | this.negate = event.negate; 39 | this.pattern = event.pattern; 40 | this.what = event.what; 41 | } 42 | } 43 | 44 | public Multiline(Map fields) throws UnsupportedEncodingException { 45 | String strPattern = ""; 46 | for(String key : fields.keySet()) { 47 | if ("pattern".equals(key)) 48 | strPattern = fields.get(key); 49 | else if ("negate".equals(key)) 50 | negate = Boolean.parseBoolean(fields.get(key)); 51 | else if ("what".equals(key)) 52 | what = WhatType.valueOf(fields.get(key)); 53 | else 54 | throw new UnsupportedEncodingException(key + " not supported"); 55 | } 56 | pattern = Pattern.compile(strPattern); 57 | } 58 | 59 | public Pattern getPattern() { 60 | return pattern; 61 | } 62 | 63 | public boolean isNegate() { 64 | return negate; 65 | } 66 | 67 | public WhatType getWhat() { 68 | return what; 69 | } 70 | 71 | public boolean isPrevious() { 72 | return what == WhatType.previous; 73 | } 74 | 75 | public boolean isPatternFound (byte[] line) { 76 | boolean result = pattern.matcher(new String(line)).find(); 77 | if (negate) return !result; 78 | return result; 79 | } 80 | 81 | @Override 82 | public String toString() { 83 | return new StringBuilder() 84 | .append("[pattern=") 85 | .append(pattern) 86 | .append(",negate=") 87 | .append(negate) 88 | .append(",what=") 89 | .append(what) 90 | .append("]") 91 | .toString(); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/ProtocolAdapter.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import info.fetter.logstashforwarder.util.AdapterException; 21 | 22 | import java.util.List; 23 | 24 | public interface ProtocolAdapter { 25 | public int sendEvents(List eventList) throws AdapterException; 26 | public void close() throws AdapterException; 27 | public String getServer(); 28 | public int getPort(); 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/Reader.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | 4 | /* 5 | * Copyright 2015 Didier Fetter 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); 8 | * you may not use this file except in compliance with the License. 9 | * You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | * 19 | */ 20 | 21 | import java.io.IOException; 22 | import java.net.InetAddress; 23 | import java.net.UnknownHostException; 24 | import java.nio.ByteBuffer; 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | 28 | 29 | public abstract class Reader { 30 | protected ProtocolAdapter adapter; 31 | protected int spoolSize = 0; 32 | protected List eventList; 33 | protected final int BYTEBUFFER_CAPACITY = 1024 * 1024; 34 | protected ByteBuffer byteBuffer = ByteBuffer.allocate(BYTEBUFFER_CAPACITY); 35 | private String hostname; 36 | { 37 | try { 38 | hostname = InetAddress.getLocalHost().getHostName(); 39 | } catch (UnknownHostException e) { 40 | throw new RuntimeException(e); 41 | } 42 | } 43 | 44 | protected Reader(int spoolSize) { 45 | this.spoolSize = spoolSize; 46 | eventList = new ArrayList(spoolSize); 47 | } 48 | 49 | protected void addEvent(FileState state, long pos, String line) throws IOException { 50 | Filter filter = state.getFilter(); 51 | if (filter == null || filter.accept(line)) { 52 | addEvent(state.getFile().getCanonicalPath(), 53 | state.getFields(), pos, line); 54 | } 55 | } 56 | 57 | protected void addEvent(FileState state, long pos, byte[] line) throws IOException { 58 | Filter filter = state.getFilter(); 59 | if (filter == null || filter.accept(line)) { 60 | addEvent(state.getFile().getCanonicalPath(), 61 | state.getFields(), pos, line); 62 | } 63 | } 64 | 65 | protected void addEvent(String fileName, Event fields, long pos, byte[] line) throws IOException { 66 | Event event = new Event(fields); 67 | event.addField("file", fileName) 68 | .addField("offset", pos) 69 | .addField("line", line) 70 | .addField("host", hostname); 71 | eventList.add(event); 72 | } 73 | 74 | protected void addEvent(String fileName, Event fields, long pos, String line) throws IOException { 75 | Event event = new Event(fields); 76 | event.addField("file", fileName) 77 | .addField("offset", pos) 78 | .addField("line", line) 79 | .addField("host", hostname); 80 | eventList.add(event); 81 | } 82 | 83 | public ProtocolAdapter getAdapter() { 84 | return adapter; 85 | } 86 | 87 | public void setAdapter(ProtocolAdapter adapter) { 88 | this.adapter = adapter; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/Registrar.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import java.io.File; 21 | import java.io.IOException; 22 | import java.util.Collection; 23 | 24 | import com.fasterxml.jackson.core.JsonGenerationException; 25 | import com.fasterxml.jackson.core.JsonParseException; 26 | import com.fasterxml.jackson.databind.JsonMappingException; 27 | import com.fasterxml.jackson.databind.ObjectMapper; 28 | 29 | public class Registrar { 30 | 31 | 32 | private static ObjectMapper mapper = new ObjectMapper(); 33 | 34 | public static FileState[] readStateFromJson(File file) throws JsonParseException, JsonMappingException, IOException { 35 | FileState[] stateArray = mapper.readValue(file, FileState[].class); 36 | return stateArray; 37 | } 38 | 39 | public static FileState[] readStateFromJson(String file) throws JsonParseException, JsonMappingException, IOException { 40 | return readStateFromJson(new File(file)); 41 | } 42 | 43 | public static void writeStateToJson(File file, Collection stateList) throws JsonGenerationException, JsonMappingException, IOException { 44 | mapper.writeValue(file, stateList); 45 | } 46 | 47 | public static void writeStateToJson(String file, Collection stateList) throws JsonGenerationException, JsonMappingException, IOException { 48 | writeStateToJson(new File(file), stateList); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/config/Configuration.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder.config; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import java.util.List; 21 | 22 | import org.apache.commons.lang.builder.ToStringBuilder; 23 | 24 | public class Configuration { 25 | private NetworkSection network; 26 | private List files; 27 | 28 | public NetworkSection getNetwork() { 29 | return network; 30 | } 31 | 32 | public void setNetwork(NetworkSection network) { 33 | this.network = network; 34 | } 35 | 36 | public List getFiles() { 37 | return files; 38 | } 39 | 40 | public void setFiles(List files) { 41 | this.files = files; 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return new ToStringBuilder(this). 47 | append("network", network). 48 | append("files", files). 49 | toString(); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/config/ConfigurationManager.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder.config; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import java.io.File; 21 | import java.io.IOException; 22 | 23 | import com.fasterxml.jackson.core.JsonParseException; 24 | import com.fasterxml.jackson.core.JsonParser; 25 | import com.fasterxml.jackson.databind.JsonMappingException; 26 | import com.fasterxml.jackson.databind.ObjectMapper; 27 | 28 | public class ConfigurationManager { 29 | private File configFile; 30 | private Configuration config; 31 | private ObjectMapper mapper; 32 | 33 | public ConfigurationManager(String configFilePath) { 34 | this(new File(configFilePath)); 35 | } 36 | 37 | public ConfigurationManager(File file) { 38 | configFile = file; 39 | mapper = new ObjectMapper(); 40 | mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); 41 | mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); 42 | } 43 | 44 | public void readConfiguration() throws JsonParseException, JsonMappingException, IOException { 45 | config = mapper.readValue(configFile, Configuration.class); 46 | } 47 | 48 | public void writeConfiguration() { 49 | } 50 | 51 | /** 52 | * @return the config 53 | */ 54 | public Configuration getConfig() { 55 | return config; 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/config/FilesSection.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder.config; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import java.util.List; 21 | import java.util.Map; 22 | import org.apache.commons.lang.builder.ToStringBuilder; 23 | 24 | import com.fasterxml.jackson.annotation.JsonProperty; 25 | import info.fetter.logstashforwarder.Multiline; 26 | import info.fetter.logstashforwarder.Filter; 27 | import java.io.UnsupportedEncodingException; 28 | 29 | public class FilesSection { 30 | private List paths; 31 | private Map fields; 32 | @JsonProperty("dead time") 33 | private String deadTime = "24h"; 34 | private Multiline multiline; 35 | private Filter filter; 36 | 37 | public List getPaths() { 38 | return paths; 39 | } 40 | 41 | public void setPaths(List paths) { 42 | this.paths = paths; 43 | } 44 | 45 | public Map getFields() { 46 | return fields; 47 | } 48 | 49 | public void setFields(Map fields) { 50 | this.fields = fields; 51 | } 52 | 53 | public String getDeadTime() { 54 | return deadTime; 55 | } 56 | 57 | public long getDeadTimeInSeconds() { 58 | long deadTimeInSeconds = 0; 59 | String remaining = deadTime; 60 | 61 | if(deadTime.contains("h")) { 62 | String[] splitByHour = deadTime.split("h",2); 63 | if(splitByHour.length > 1) { 64 | remaining = splitByHour[1]; 65 | } 66 | deadTimeInSeconds += Integer.parseInt(splitByHour[0]) * 3600; 67 | } 68 | if(remaining.contains("m")) { 69 | String[] splitByMinute = remaining.split("m",2); 70 | if(splitByMinute.length > 1) { 71 | remaining = splitByMinute[1]; 72 | } 73 | deadTimeInSeconds += Integer.parseInt(splitByMinute[0]) * 60; 74 | } 75 | if(remaining.contains("s")) { 76 | String[] splitBySecond = remaining.split("s",2); 77 | deadTimeInSeconds += Integer.parseInt(splitBySecond[0]); 78 | } 79 | return deadTimeInSeconds; 80 | } 81 | 82 | public void setDeadTime(String deadTime) { 83 | this.deadTime = deadTime; 84 | } 85 | 86 | public Multiline getMultiline() { 87 | return multiline; 88 | } 89 | 90 | public void setMultiline(Map multilineMap) throws UnsupportedEncodingException { 91 | this.multiline = new Multiline(multilineMap); 92 | } 93 | 94 | public Filter getFilter() { 95 | return filter; 96 | } 97 | 98 | public void setFilter(Map filterMap) throws UnsupportedEncodingException { 99 | this.filter = new Filter(filterMap); 100 | } 101 | 102 | @Override 103 | public String toString() { 104 | return new ToStringBuilder(this). 105 | append("paths", paths). 106 | append("fields", fields). 107 | append("dead time", deadTime). 108 | append("multiline", multiline). 109 | append("filter", filter). 110 | toString(); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/config/NetworkSection.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder.config; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import java.util.List; 21 | 22 | import org.apache.commons.lang.builder.ToStringBuilder; 23 | 24 | import com.fasterxml.jackson.annotation.JsonProperty; 25 | 26 | public class NetworkSection { 27 | private List servers; 28 | @JsonProperty("ssl certificate") 29 | private String sslCertificate; 30 | @JsonProperty("ssl ca") 31 | private String sslCA; 32 | @JsonProperty("ssl key") 33 | private String sslKey; 34 | private int timeout; 35 | 36 | public List getServers() { 37 | return servers; 38 | } 39 | 40 | public void setServers(List servers) { 41 | this.servers = servers; 42 | } 43 | 44 | public String getSslCertificate() { 45 | return sslCertificate; 46 | } 47 | 48 | public void setSslCertificate(String sslCertificate) { 49 | this.sslCertificate = sslCertificate; 50 | } 51 | 52 | public String getSslCA() { 53 | return sslCA; 54 | } 55 | 56 | public void setSslCA(String sslCA) { 57 | this.sslCA = sslCA; 58 | } 59 | 60 | public int getTimeout() { 61 | return timeout; 62 | } 63 | 64 | public void setTimeout(int timeout) { 65 | this.timeout = timeout; 66 | } 67 | 68 | public String getSslKey() { 69 | return sslKey; 70 | } 71 | 72 | public void setSslKey(String sslKey) { 73 | this.sslKey = sslKey; 74 | } 75 | 76 | @Override 77 | public String toString() { 78 | return new ToStringBuilder(this). 79 | append("servers", servers). 80 | append("sslCertificate", sslCertificate). 81 | append("sslCA", sslCA). 82 | append("timeout", timeout). 83 | toString(); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/protocol/LumberjackClient.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder.protocol; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import info.fetter.logstashforwarder.Event; 21 | import info.fetter.logstashforwarder.ProtocolAdapter; 22 | import info.fetter.logstashforwarder.util.AdapterException; 23 | 24 | import java.io.BufferedOutputStream; 25 | import java.io.ByteArrayOutputStream; 26 | import java.io.DataInputStream; 27 | import java.io.DataOutputStream; 28 | import java.io.FileInputStream; 29 | import java.io.IOException; 30 | import java.net.InetAddress; 31 | import java.net.InetSocketAddress; 32 | import java.net.ProtocolException; 33 | import java.net.Socket; 34 | import java.security.KeyStore; 35 | import java.util.ArrayList; 36 | import java.util.List; 37 | import java.util.Map; 38 | import java.util.zip.Deflater; 39 | 40 | import javax.net.ssl.SSLContext; 41 | import javax.net.ssl.SSLSocket; 42 | import javax.net.ssl.SSLSocketFactory; 43 | import javax.net.ssl.TrustManagerFactory; 44 | 45 | import org.apache.commons.io.HexDump; 46 | import org.apache.log4j.Logger; 47 | 48 | public class LumberjackClient implements ProtocolAdapter { 49 | private final static Logger logger = Logger.getLogger(LumberjackClient.class); 50 | private final static byte PROTOCOL_VERSION = 0x31; 51 | private final static byte FRAME_ACK = 0x41; 52 | private final static byte FRAME_WINDOW_SIZE = 0x57; 53 | private final static byte FRAME_DATA = 0x44; 54 | private final static byte FRAME_COMPRESSED = 0x43; 55 | 56 | private Socket socket; 57 | private SSLSocket sslSocket; 58 | private KeyStore keyStore; 59 | private String server; 60 | private int port; 61 | private DataOutputStream output; 62 | private DataInputStream input; 63 | private int sequence = 1; 64 | 65 | public LumberjackClient(String keyStorePath, String server, int port, int timeout) throws IOException { 66 | this.server = server; 67 | this.port = port; 68 | 69 | try { 70 | if(keyStorePath == null) { 71 | throw new IOException("Key store not configured"); 72 | } 73 | if(server == null) { 74 | throw new IOException("Server address not configured"); 75 | } 76 | 77 | keyStore = KeyStore.getInstance("JKS"); 78 | keyStore.load(new FileInputStream(keyStorePath), null); 79 | 80 | TrustManagerFactory tmf = TrustManagerFactory.getInstance("PKIX"); 81 | tmf.init(keyStore); 82 | 83 | SSLContext context = SSLContext.getInstance("TLS"); 84 | context.init(null, tmf.getTrustManagers(), null); 85 | 86 | SSLSocketFactory socketFactory = context.getSocketFactory(); 87 | socket = new Socket(); 88 | socket.connect(new InetSocketAddress(InetAddress.getByName(server), port), timeout); 89 | socket.setSoTimeout(timeout); 90 | sslSocket = (SSLSocket)socketFactory.createSocket(socket, server, port, true); 91 | sslSocket.setUseClientMode(true); 92 | sslSocket.startHandshake(); 93 | 94 | output = new DataOutputStream(new BufferedOutputStream(sslSocket.getOutputStream())); 95 | input = new DataInputStream(sslSocket.getInputStream()); 96 | 97 | logger.info("Connected to " + server + ":" + port); 98 | } catch(IOException e) { 99 | throw e; 100 | } catch(Exception e) { 101 | throw new RuntimeException(e); 102 | } 103 | } 104 | 105 | public int sendWindowSizeFrame(int size) throws IOException { 106 | output.writeByte(PROTOCOL_VERSION); 107 | output.writeByte(FRAME_WINDOW_SIZE); 108 | output.writeInt(size); 109 | output.flush(); 110 | if(logger.isDebugEnabled()) { 111 | logger.debug("Sending window size frame : " + size + " frames"); 112 | } 113 | return 6; 114 | } 115 | 116 | private int sendDataFrame(DataOutputStream output, Map keyValues) throws IOException { 117 | output.writeByte(PROTOCOL_VERSION); 118 | output.writeByte(FRAME_DATA); 119 | output.writeInt(sequence++); 120 | output.writeInt(keyValues.size()); 121 | int bytesSent = 10; 122 | for(String key : keyValues.keySet()) { 123 | int keyLength = key.length(); 124 | output.writeInt(keyLength); 125 | bytesSent += 4; 126 | output.write(key.getBytes()); 127 | bytesSent += keyLength; 128 | byte[] value = keyValues.get(key); 129 | output.writeInt(value.length); 130 | bytesSent += 4; 131 | output.write(value); 132 | bytesSent += value.length; 133 | } 134 | output.flush(); 135 | return bytesSent; 136 | } 137 | 138 | public int sendDataFrameInSocket(Map keyValues) throws IOException { 139 | return sendDataFrame(output, keyValues); 140 | } 141 | 142 | public int sendCompressedFrame(List> keyValuesList) throws IOException { 143 | output.writeByte(PROTOCOL_VERSION); 144 | output.writeByte(FRAME_COMPRESSED); 145 | 146 | ByteArrayOutputStream uncompressedBytes = new ByteArrayOutputStream(); 147 | DataOutputStream uncompressedOutput = new DataOutputStream(uncompressedBytes); 148 | for(Map keyValues : keyValuesList) { 149 | logger.trace("Adding data frame"); 150 | sendDataFrame(uncompressedOutput, keyValues); 151 | } 152 | uncompressedOutput.close(); 153 | Deflater compressor = new Deflater(); 154 | byte[] uncompressedData = uncompressedBytes.toByteArray(); 155 | if(logger.isDebugEnabled()) { 156 | logger.debug("Deflating data : " + uncompressedData.length + " bytes"); 157 | } 158 | if(logger.isTraceEnabled()) { 159 | HexDump.dump(uncompressedData, 0, System.out, 0); 160 | } 161 | compressor.setInput(uncompressedData); 162 | compressor.finish(); 163 | 164 | ByteArrayOutputStream compressedBytes = new ByteArrayOutputStream(); 165 | byte[] buffer = new byte[1024]; 166 | while(!compressor.finished()) { 167 | int count = compressor.deflate(buffer); 168 | compressedBytes.write(buffer, 0, count); 169 | } 170 | compressedBytes.close(); 171 | byte[] compressedData = compressedBytes.toByteArray(); 172 | if(logger.isDebugEnabled()) { 173 | logger.debug("Deflated data : " + compressor.getTotalOut() + " bytes"); 174 | } 175 | if(logger.isTraceEnabled()) { 176 | HexDump.dump(compressedData, 0, System.out, 0); 177 | } 178 | 179 | output.writeInt(compressor.getTotalOut()); 180 | output.write(compressedData); 181 | output.flush(); 182 | 183 | if(logger.isDebugEnabled()) { 184 | logger.debug("Sending compressed frame : " + keyValuesList.size() + " frames"); 185 | } 186 | return 6 + compressor.getTotalOut(); 187 | } 188 | 189 | public int readAckFrame() throws ProtocolException, IOException { 190 | byte protocolVersion = input.readByte(); 191 | if(protocolVersion != PROTOCOL_VERSION) { 192 | throw new ProtocolException("Protocol version should be 1, received " + protocolVersion); 193 | } 194 | byte frameType = input.readByte(); 195 | if(frameType != FRAME_ACK) { 196 | throw new ProtocolException("Frame type should be Ack, received " + frameType); 197 | } 198 | int sequenceNumber = input.readInt(); 199 | if(logger.isDebugEnabled()) { 200 | logger.debug("Received ack sequence : " + sequenceNumber); 201 | } 202 | return sequenceNumber; 203 | } 204 | 205 | public int sendEvents(List eventList) throws AdapterException { 206 | try { 207 | int beginSequence = sequence; 208 | int numberOfEvents = eventList.size(); 209 | if(logger.isInfoEnabled()) { 210 | logger.info("Sending " + numberOfEvents + " events"); 211 | } 212 | sendWindowSizeFrame(numberOfEvents); 213 | List> keyValuesList = new ArrayList>(numberOfEvents); 214 | for(Event event : eventList) { 215 | keyValuesList.add(event.getKeyValues()); 216 | } 217 | sendCompressedFrame(keyValuesList); 218 | while(readAckFrame() < (sequence - 1) ) {} 219 | return sequence - beginSequence; 220 | } catch(Exception e) { 221 | throw new AdapterException(e); 222 | } 223 | } 224 | 225 | public void close() throws AdapterException { 226 | try { 227 | sslSocket.close(); 228 | } catch(Exception e) { 229 | throw new AdapterException(e); 230 | } 231 | logger.info("Connection to " + server + ":" + port + " closed"); 232 | } 233 | 234 | public String getServer() { 235 | return server; 236 | } 237 | 238 | public int getPort() { 239 | return port; 240 | } 241 | 242 | } 243 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/util/AdapterException.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder.util; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | public class AdapterException extends Exception { 21 | private static final long serialVersionUID = -5236526584082990877L; 22 | 23 | public AdapterException(String message) { 24 | super(message); 25 | } 26 | 27 | public AdapterException(Throwable cause) { 28 | super(cause); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/util/KMPMatch.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 1998-2009 University Corporation for Atmospheric Research/Unidata 3 | * 4 | * Portions of this software were developed by the Unidata Program at the 5 | * University Corporation for Atmospheric Research. 6 | * 7 | * Access and use of this software shall impose the following obligations 8 | * and understandings on the user. The user is granted the right, without 9 | * any fee or cost, to use, copy, modify, alter, enhance and distribute 10 | * this software, and any derivative works thereof, and its supporting 11 | * documentation for any purpose whatsoever, provided that this entire 12 | * notice appears in all copies of the software, derivative works and 13 | * supporting documentation. Further, UCAR requests that the user credit 14 | * UCAR/Unidata in any publications that result from the use of this 15 | * software or in any product that includes this software. The names UCAR 16 | * and/or Unidata, however, may not be used in any advertising or publicity 17 | * to endorse or promote any products or commercial entity unless specific 18 | * written permission is obtained from UCAR/Unidata. The user also 19 | * understands that UCAR/Unidata is not obligated to provide the user with 20 | * any support, consulting, training or assistance of any kind with regard 21 | * to the use, operation and performance of this software nor to provide 22 | * the user with any updates, revisions, new versions or "bug fixes." 23 | * 24 | * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR 25 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 26 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 27 | * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL, 28 | * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING 29 | * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, 30 | * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION 31 | * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE. 32 | */ 33 | package info.fetter.logstashforwarder.util; 34 | 35 | /** 36 | * Knuth-Morris-Pratt Algorithm for Pattern Matching. 37 | * Immutable 38 | * 39 | * @author caron 40 | * @see http://www.fmi.uni-sofia.bg/fmi/logic/vboutchkova/sources/KMPMatch_java.html 41 | * @since May 9, 2008 42 | */ 43 | public class KMPMatch { 44 | 45 | private final byte[] match; 46 | private final int[] failure; 47 | 48 | /** 49 | * Constructor 50 | * @param match search for this byte pattern 51 | */ 52 | public KMPMatch(byte[] match) { 53 | this.match = match; 54 | failure = computeFailure(match); 55 | } 56 | 57 | public int getMatchLength() { return match.length; } 58 | 59 | /** 60 | * Finds the first occurrence of match in data. 61 | * @param data search in this byte block 62 | * @param start start at data[start] 63 | * @param max end at data[start+max] 64 | * @return index into data[] of first match, else -1 if not found. 65 | */ 66 | public int indexOf(byte[] data, int start, int max) { 67 | int j = 0; 68 | if (data.length == 0) return -1; 69 | 70 | for (int i = start; i < start + max; i++) { 71 | while (j > 0 && match[j] != data[i]) 72 | j = failure[j - 1]; 73 | 74 | if (match[j] == data[i]) 75 | j++; 76 | 77 | if (j == match.length) 78 | return i - match.length + 1; 79 | 80 | } 81 | return -1; 82 | } 83 | 84 | /* 85 | * Finds the first occurrence of match in data. 86 | * @param data search in this byte block 87 | * @param start start at data[start] 88 | * @param max end at data[start+max] 89 | * @return index into block of first match, else -1 if not found. 90 | * 91 | public int scan(InputStream is, int start, int max) { 92 | int j = 0; 93 | if (data.length == 0) return -1; 94 | 95 | for (int i = start; i < start + max; i++) { 96 | while (j > 0 && match[j] != data[i]) 97 | j = failure[j - 1]; 98 | 99 | if (match[j] == data[i]) 100 | j++; 101 | 102 | if (j == match.length) 103 | return i - match.length + 1; 104 | 105 | } 106 | return -1; 107 | } // */ 108 | 109 | 110 | private int[] computeFailure(byte[] match) { 111 | int[] result = new int[match.length]; 112 | 113 | int j = 0; 114 | for (int i = 1; i < match.length; i++) { 115 | while (j > 0 && match[j] != match[i]) 116 | j = result[j - 1]; 117 | 118 | if (match[i] == match[i]) 119 | j++; 120 | 121 | result[i] = j; 122 | } 123 | 124 | return result; 125 | } 126 | } 127 | 128 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/util/LastModifiedFileFilter.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder.util; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import java.io.File; 21 | 22 | import org.apache.commons.io.FileUtils; 23 | import org.apache.commons.io.filefilter.AbstractFileFilter; 24 | 25 | public class LastModifiedFileFilter extends AbstractFileFilter { 26 | private boolean after; 27 | private long cutoff; 28 | 29 | public LastModifiedFileFilter(long cutoff, boolean after) { 30 | this.after = after; 31 | this.cutoff = cutoff; 32 | } 33 | 34 | public LastModifiedFileFilter(long cutoff) { 35 | this(cutoff, true); 36 | } 37 | 38 | @Override 39 | public boolean accept(File file) { 40 | long timeMillis = System.currentTimeMillis() - cutoff; 41 | if(after) { 42 | return FileUtils.isFileNewer(file, timeMillis); 43 | } else { 44 | return FileUtils.isFileOlder(file, timeMillis); 45 | } 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/info/fetter/logstashforwarder/util/RandomAccessFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 1998-2009 University Corporation for Atmospheric Research/Unidata 3 | * 4 | * Portions of this software were developed by the Unidata Program at the 5 | * University Corporation for Atmospheric Research. 6 | * 7 | * Access and use of this software shall impose the following obligations 8 | * and understandings on the user. The user is granted the right, without 9 | * any fee or cost, to use, copy, modify, alter, enhance and distribute 10 | * this software, and any derivative works thereof, and its supporting 11 | * documentation for any purpose whatsoever, provided that this entire 12 | * notice appears in all copies of the software, derivative works and 13 | * supporting documentation. Further, UCAR requests that the user credit 14 | * UCAR/Unidata in any publications that result from the use of this 15 | * software or in any product that includes this software. The names UCAR 16 | * and/or Unidata, however, may not be used in any advertising or publicity 17 | * to endorse or promote any products or commercial entity unless specific 18 | * written permission is obtained from UCAR/Unidata. The user also 19 | * understands that UCAR/Unidata is not obligated to provide the user with 20 | * any support, consulting, training or assistance of any kind with regard 21 | * to the use, operation and performance of this software nor to provide 22 | * the user with any updates, revisions, new versions or "bug fixes." 23 | * 24 | * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR 25 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 26 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 27 | * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL, 28 | * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING 29 | * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, 30 | * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION 31 | * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE. 32 | */ 33 | 34 | package info.fetter.logstashforwarder.util; 35 | 36 | import java.io.*; 37 | 38 | import java.util.*; 39 | import java.util.concurrent.atomic.AtomicInteger; 40 | import java.util.concurrent.atomic.AtomicLong; 41 | import java.nio.channels.WritableByteChannel; 42 | 43 | 44 | /** 45 | * A buffered drop-in replacement for java.io.RandomAccessFile. 46 | * Instances of this class realise substantial speed increases over 47 | * java.io.RandomAccessFile through the use of buffering. This is a 48 | * subclass of Object, as it was not possible to subclass 49 | * java.io.RandomAccessFile because many of the methods are 50 | * final. However, if it is necessary to use RandomAccessFile and 51 | * java.io.RandomAccessFile interchangeably, both classes implement the 52 | * DataInput and DataOutput interfaces. 53 | * 54 | * By Russ Rew, based on 55 | * BufferedRandomAccessFile by Alex McManus, based on Sun's source code 56 | * for java.io.RandomAccessFile. For Alex McManus version from which 57 | * this derives, see his 58 | * Freeware Java Classes. 59 | * 60 | * 61 | * @author Alex McManus 62 | * @author Russ Rew 63 | * @author john caron 64 | * @see DataInput 65 | * @see DataOutput 66 | * @see java.io.RandomAccessFile 67 | */ 68 | 69 | public class RandomAccessFile implements DataInput, DataOutput { 70 | 71 | static public final int BIG_ENDIAN = 0; 72 | static public final int LITTLE_ENDIAN = 1; 73 | 74 | // debug leaks - keep track of open files 75 | 76 | 77 | static protected boolean debugLeaks = false; 78 | static protected boolean debugAccess = false; 79 | static protected Set allFiles = new HashSet(); 80 | static protected List openFiles = Collections.synchronizedList(new ArrayList()); 81 | static private AtomicInteger debug_nseeks = new AtomicInteger(); 82 | static private AtomicLong debug_nbytes = new AtomicLong(); 83 | 84 | /** 85 | * Debugging, do not use. 86 | * 87 | * @return true if debugLeaks is on 88 | */ 89 | static public boolean getDebugLeaks() { 90 | return debugLeaks; 91 | } 92 | 93 | /** 94 | * Debugging, do not use. 95 | * 96 | * @param b set true to track java.io.RandomAccessFile 97 | */ 98 | static public void setDebugLeaks(boolean b) { 99 | debugLeaks = b; 100 | } 101 | 102 | /** 103 | * Debugging, do not use. 104 | * 105 | * @return list of open files. 106 | */ 107 | static public List getOpenFiles() { 108 | return openFiles; 109 | } 110 | 111 | static public List getAllFiles() { 112 | List result = new ArrayList(); 113 | if (null == allFiles) return null; 114 | Iterator iter = allFiles.iterator(); 115 | while (iter.hasNext()) { 116 | result.add( iter.next()); 117 | } 118 | Collections.sort(result); 119 | return result; 120 | } 121 | 122 | /** 123 | * Debugging, do not use. 124 | * 125 | * @param b to debug file reading 126 | */ 127 | static public void setDebugAccess(boolean b) { 128 | debugAccess = b; 129 | if (b) { 130 | debug_nseeks = new AtomicInteger(); 131 | debug_nbytes = new AtomicLong(); 132 | } 133 | } 134 | 135 | static public int getDebugNseeks() { 136 | return (debug_nseeks == null) ? 0 : debug_nseeks.intValue(); 137 | } 138 | 139 | static public long getDebugNbytes() { 140 | return (debug_nbytes == null) ? 0 : debug_nbytes.longValue(); 141 | } 142 | 143 | static protected boolean showOpen = false; 144 | static protected boolean showRead = false; 145 | 146 | /** 147 | * The default buffer size, in bytes. 148 | */ 149 | protected static final int defaultBufferSize = 8092; 150 | 151 | ///////////////////////////////////////////////////////////////////////////////////////////// 152 | 153 | /** 154 | * File location 155 | */ 156 | protected String location; 157 | 158 | /** 159 | * The underlying java.io.RandomAccessFile. 160 | */ 161 | protected java.io.RandomAccessFile file; 162 | protected java.nio.channels.FileChannel fileChannel; 163 | 164 | /** 165 | * The offset in bytes from the file start, of the next read or 166 | * write operation. 167 | */ 168 | protected long filePosition; 169 | 170 | /** 171 | * The buffer used for reading the data. 172 | */ 173 | protected byte buffer[]; 174 | 175 | /** 176 | * The offset in bytes of the start of the buffer, from the start of the file. 177 | */ 178 | protected long bufferStart; 179 | 180 | /** 181 | * The offset in bytes of the end of the data in the buffer, from 182 | * the start of the file. This can be calculated from 183 | * bufferStart + dataSize, but it is cached to speed 184 | * up the read( ) method. 185 | */ 186 | protected long dataEnd; 187 | 188 | /** 189 | * The size of the data stored in the buffer, in bytes. This may be 190 | * less than the size of the buffer. 191 | */ 192 | protected int dataSize; 193 | 194 | /** 195 | * True if we are at the end of the file. 196 | */ 197 | protected boolean endOfFile; 198 | 199 | /** 200 | * The access mode of the file. 201 | */ 202 | protected boolean readonly; 203 | 204 | /** 205 | * The current endian (big or little) mode of the file. 206 | */ 207 | protected boolean bigEndian; 208 | 209 | /** 210 | * True if the data in the buffer has been modified. 211 | */ 212 | boolean bufferModified = false; 213 | 214 | /** 215 | * make sure file is this long when closed 216 | */ 217 | private long minLength = 0; 218 | 219 | /** 220 | * stupid extendMode for truncated, yet valid files - old code allowed NOFILL to do this 221 | */ 222 | boolean extendMode = false; 223 | 224 | /** 225 | * Constructor, for subclasses 226 | * 227 | * @param bufferSize size of read buffer 228 | */ 229 | protected RandomAccessFile(int bufferSize) { 230 | file = null; 231 | readonly = true; 232 | init(bufferSize); 233 | } 234 | 235 | /** 236 | * Constructor, default buffer size. 237 | * 238 | * @param location location of the file 239 | * @param mode same as for java.io.RandomAccessFile 240 | * @throws IOException on open error 241 | */ 242 | public RandomAccessFile(String location, String mode) throws IOException { 243 | this(location, mode, defaultBufferSize); 244 | this.location = location; 245 | } 246 | 247 | /** 248 | * Constructor. 249 | * 250 | * @param location location of the file 251 | * @param mode same as for java.io.RandomAccessFile 252 | * @param bufferSize size of buffer to use. 253 | * @throws IOException on open error 254 | */ 255 | public RandomAccessFile(String location, String mode, int bufferSize) throws IOException { 256 | this.location = location; 257 | if (debugLeaks) { 258 | allFiles.add(location); 259 | if ((location.indexOf("01janN") >= 0) || (location.indexOf("02febN") >= 0)) 260 | System.out.printf("HEY!%n"); 261 | } 262 | 263 | this.file = new java.io.RandomAccessFile(location, mode); 264 | this.readonly = mode.equals("r"); 265 | init(bufferSize); 266 | 267 | if (debugLeaks) { 268 | openFiles.add(location); 269 | if (showOpen) System.out.println(" open " + location); 270 | } 271 | } 272 | 273 | /** 274 | * Allow access to the underlying java.io.RandomAccessFile. 275 | * WARNING! BROKEN ENCAPSOLATION, DO NOT USE. May change implementation in the future. 276 | * 277 | * @return the underlying java.io.RandomAccessFile. 278 | */ 279 | public java.io.RandomAccessFile getRandomAccessFile() { 280 | return this.file; 281 | } 282 | 283 | private void init(int bufferSize) { 284 | // Initialise the buffer 285 | bufferStart = 0; 286 | dataEnd = 0; 287 | dataSize = 0; 288 | filePosition = 0; 289 | buffer = new byte[bufferSize]; 290 | endOfFile = false; 291 | } 292 | 293 | /** 294 | * Set the buffer size. 295 | * If writing, call flush() first. 296 | * 297 | * @param bufferSize length in bytes 298 | */ 299 | public void setBufferSize(int bufferSize) { 300 | init(bufferSize); 301 | } 302 | 303 | /** 304 | * Get the buffer size 305 | * 306 | * @return bufferSize length in bytes 307 | */ 308 | public int getBufferSize() { 309 | return buffer.length; 310 | } 311 | 312 | /** 313 | * Close the file, and release any associated system resources. 314 | * 315 | * @throws IOException if an I/O error occurrs. 316 | */ 317 | public void close() throws IOException { 318 | if (debugLeaks) { 319 | openFiles.remove(location); 320 | if (showOpen) System.out.println(" close " + location); 321 | } 322 | 323 | if (file == null) 324 | return; 325 | 326 | // If we are writing and the buffer has been modified, flush the contents 327 | // of the buffer. 328 | flush(); 329 | 330 | /* 331 | if (!readonly && bufferModified) { 332 | file.seek(bufferStart); 333 | file.write(buffer, 0, dataSize); 334 | } */ 335 | 336 | // may need to extend file, in case no fill is being used 337 | // may need to truncate file in case overwriting a longer file 338 | // use only if minLength is set (by N3iosp) 339 | long fileSize = file.length(); 340 | if (!readonly && (minLength != 0) && (minLength != fileSize)) { 341 | file.setLength(minLength); 342 | // System.out.println("TRUNCATE!!! minlength="+minLength); 343 | } 344 | 345 | // Close the underlying file object. 346 | file.close(); 347 | //file = null; // help the gc => commented because of problems with nullpointerexceptions when trying to read after close 348 | } 349 | 350 | /** 351 | * Return true if file pointer is at end of file. 352 | * 353 | * @return true if file pointer is at end of file 354 | */ 355 | public boolean isAtEndOfFile() { 356 | return endOfFile; 357 | } 358 | 359 | /** 360 | * Set the position in the file for the next read or write. 361 | * 362 | * @param pos the offset (in bytes) from the start of the file. 363 | * @throws IOException if an I/O error occurrs. 364 | */ 365 | public void seek(long pos) throws IOException { 366 | 367 | // If the seek is into the buffer, just update the file pointer. 368 | if ((pos >= bufferStart) && (pos < dataEnd)) { 369 | filePosition = pos; 370 | return; 371 | } 372 | 373 | // need new buffer, starting at pos 374 | readBuffer(pos); 375 | } 376 | 377 | protected void readBuffer(long pos) throws IOException { 378 | // If the current buffer is modified, write it to disk. 379 | if (bufferModified) { 380 | flush(); 381 | } 382 | 383 | bufferStart = pos; 384 | filePosition = pos; 385 | 386 | dataSize = read_(pos, buffer, 0, buffer.length); 387 | 388 | if (dataSize <= 0) { 389 | dataSize = 0; 390 | endOfFile = true; 391 | } else { 392 | endOfFile = false; 393 | } 394 | 395 | // Cache the position of the buffer end. 396 | dataEnd = bufferStart + dataSize; 397 | } 398 | 399 | /** 400 | * Returns the current position in the file, where the next read or 401 | * write will occur. 402 | * 403 | * @return the offset from the start of the file in bytes. 404 | * @throws IOException if an I/O error occurrs. 405 | */ 406 | public long getFilePointer() throws IOException { 407 | return filePosition; 408 | } 409 | 410 | /** 411 | * Get the file location, or name. 412 | * 413 | * @return file location 414 | */ 415 | public String getLocation() { 416 | return location; 417 | } 418 | 419 | /** 420 | * Get the length of the file. The data in the buffer (which may not 421 | * have been written the disk yet) is taken into account. 422 | * 423 | * @return the length of the file in bytes. 424 | * @throws IOException if an I/O error occurrs. 425 | */ 426 | public long length() throws IOException { 427 | long fileLength = file.length(); 428 | if (fileLength < dataEnd) { 429 | return dataEnd; 430 | } else { 431 | return fileLength; 432 | } 433 | } 434 | 435 | /** 436 | * Change the current endian mode. Subsequent reads of short, int, float, double, long, char will 437 | * use this. Does not currently affect writes. 438 | * Default values is BIG_ENDIAN. 439 | * 440 | * @param endian RandomAccessFile.BIG_ENDIAN or RandomAccessFile.LITTLE_ENDIAN 441 | */ 442 | public void order(int endian) { 443 | if (endian < 0) return; 444 | this.bigEndian = (endian == BIG_ENDIAN); 445 | } 446 | 447 | /** 448 | * Returns the opaque file descriptor object associated with this file. 449 | * 450 | * @return the file descriptor object associated with this file. 451 | * @throws IOException if an I/O error occurs. 452 | */ 453 | public FileDescriptor getFD() throws IOException { 454 | return (file == null) ? null : file.getFD(); 455 | } 456 | 457 | public boolean isEmpty() throws IOException { 458 | return length() == 0; 459 | } 460 | 461 | /** 462 | * Copy the contents of the buffer to the disk. 463 | * 464 | * @throws IOException if an I/O error occurs. 465 | */ 466 | public void flush() throws IOException { 467 | if (bufferModified) { 468 | file.seek(bufferStart); 469 | file.write(buffer, 0, dataSize); 470 | //System.out.println("--flush at "+bufferStart+" dataSize= "+dataSize+ " filePosition= "+filePosition); 471 | bufferModified = false; 472 | } 473 | 474 | /* check min length 475 | if (!readonly && (minLength != 0) && (minLength != file.length())) { 476 | file.setLength(minLength); 477 | } */ 478 | } 479 | 480 | /** 481 | * Make sure file is at least this long when its closed. 482 | * needed when not using fill mode, and not all data is written. 483 | * 484 | * @param minLength minimum length of the file. 485 | */ 486 | public void setMinLength(long minLength) { 487 | this.minLength = minLength; 488 | } 489 | 490 | /** 491 | * Set extendMode for truncated, yet valid files - old NetCDF code allowed this 492 | * when NOFILL on, and user doesnt write all variables. 493 | */ 494 | public void setExtendMode() { 495 | this.extendMode = true; 496 | } 497 | 498 | ////////////////////////////////////////////////////////////////////////////////////////////// 499 | // Read primitives. 500 | // 501 | 502 | /** 503 | * Read a byte of data from the file, blocking until data is 504 | * available. 505 | * 506 | * @return the next byte of data, or -1 if the end of the file is 507 | * reached. 508 | * @throws IOException if an I/O error occurrs. 509 | */ 510 | public int read() throws IOException { 511 | 512 | // If the file position is within the data, return the byte... 513 | if (filePosition < dataEnd) { 514 | int pos = (int) (filePosition - bufferStart); 515 | filePosition++; 516 | return (buffer[pos] & 0xff); 517 | 518 | // ...or should we indicate EOF... 519 | } else if (endOfFile) { 520 | return -1; 521 | 522 | // ...or seek to fill the buffer, and try again. 523 | } else { 524 | seek(filePosition); 525 | return read(); 526 | } 527 | } 528 | 529 | /** 530 | * Read up to len bytes into an array, at a specified 531 | * offset. This will block until at least one byte has been read. 532 | * 533 | * @param b the byte array to receive the bytes. 534 | * @param off the offset in the array where copying will start. 535 | * @param len the number of bytes to copy. 536 | * @return the actual number of bytes read, or -1 if there is not 537 | * more data due to the end of the file being reached. 538 | * @throws IOException if an I/O error occurrs. 539 | */ 540 | protected int readBytes(byte b[], int off, int len) throws IOException { 541 | 542 | // Check for end of file. 543 | if (endOfFile) { 544 | return -1; 545 | } 546 | 547 | // See how many bytes are available in the buffer - if none, 548 | // seek to the file position to update the buffer and try again. 549 | int bytesAvailable = (int) (dataEnd - filePosition); 550 | if (bytesAvailable < 1) { 551 | seek(filePosition); 552 | return readBytes(b, off, len); 553 | } 554 | 555 | // Copy as much as we can. 556 | int copyLength = (bytesAvailable >= len) 557 | ? len 558 | : bytesAvailable; 559 | System.arraycopy(buffer, (int) (filePosition - bufferStart), b, off, copyLength); 560 | filePosition += copyLength; 561 | 562 | // If there is more to copy... 563 | if (copyLength < len) { 564 | int extraCopy = len - copyLength; 565 | 566 | // If the amount remaining is more than a buffer's length, read it 567 | // directly from the file. 568 | if (extraCopy > buffer.length) { 569 | extraCopy = read_(filePosition, b, off + copyLength, len - copyLength); 570 | 571 | // ...or read a new buffer full, and copy as much as possible... 572 | } else { 573 | seek(filePosition); 574 | if (!endOfFile) { 575 | extraCopy = (extraCopy > dataSize) 576 | ? dataSize 577 | : extraCopy; 578 | System.arraycopy(buffer, 0, b, off + copyLength, extraCopy); 579 | } else { 580 | extraCopy = -1; 581 | } 582 | } 583 | 584 | // If we did manage to copy any more, update the file position and 585 | // return the amount copied. 586 | if (extraCopy > 0) { 587 | filePosition += extraCopy; 588 | return copyLength + extraCopy; 589 | } 590 | } 591 | 592 | // Return the amount copied. 593 | return copyLength; 594 | } 595 | 596 | /** 597 | * Read nbytes bytes, at the specified file offset, send to a WritableByteChannel. 598 | * This will block until all bytes are read. 599 | * This uses the underlying file channel directly, bypassing all user buffers. 600 | * 601 | * @param dest write to this WritableByteChannel. 602 | * @param offset the offset in the file where copying will start. 603 | * @param nbytes the number of bytes to read. 604 | * @return the actual number of bytes read and transfered 605 | * @throws IOException if an I/O error occurs. 606 | */ 607 | public long readToByteChannel(WritableByteChannel dest, long offset, long nbytes) throws IOException { 608 | 609 | if (fileChannel == null) 610 | fileChannel = file.getChannel(); 611 | 612 | long need = nbytes; 613 | while (need > 0) { 614 | long count = fileChannel.transferTo(offset, need, dest); 615 | //if (count == 0) break; // LOOK not sure what the EOF condition is 616 | need -= count; 617 | offset += count; 618 | } 619 | return nbytes - need; 620 | } 621 | 622 | 623 | /** 624 | * Read directly from file, without going through the buffer. 625 | * All reading goes through here or readToByteChannel; 626 | * 627 | * @param pos start here in the file 628 | * @param b put data into this buffer 629 | * @param offset buffer offset 630 | * @param len this number of bytes 631 | * @return actual number of bytes read 632 | * @throws IOException on io error 633 | */ 634 | protected int read_(long pos, byte[] b, int offset, int len) throws IOException { 635 | file.seek(pos); 636 | int n = file.read(b, offset, len); 637 | if (debugAccess) { 638 | if (showRead) System.out.println(" **read_ " + location + " = " + len + " bytes at " + pos + "; block = " + (pos / buffer.length)); 639 | debug_nseeks.incrementAndGet(); 640 | debug_nbytes.addAndGet(len); 641 | } 642 | 643 | if (extendMode && (n < len)) { 644 | //System.out.println(" read_ = "+len+" at "+pos+"; got = "+n); 645 | n = len; 646 | } 647 | return n; 648 | } 649 | 650 | /** 651 | * Read up to len bytes into an array, at a specified 652 | * offset. This will block until at least one byte has been read. 653 | * 654 | * @param b the byte array to receive the bytes. 655 | * @param off the offset in the array where copying will start. 656 | * @param len the number of bytes to copy. 657 | * @return the actual number of bytes read, or -1 if there is not 658 | * more data due to the end of the file being reached. 659 | * @throws IOException if an I/O error occurrs. 660 | */ 661 | public int read(byte b[], int off, int len) throws IOException { 662 | return readBytes(b, off, len); 663 | } 664 | 665 | /** 666 | * Read up to b.length( ) bytes into an array. This 667 | * will block until at least one byte has been read. 668 | * 669 | * @param b the byte array to receive the bytes. 670 | * @return the actual number of bytes read, or -1 if there is not 671 | * more data due to the end of the file being reached. 672 | * @throws IOException if an I/O error occurrs. 673 | */ 674 | public int read(byte b[]) throws IOException { 675 | return readBytes(b, 0, b.length); 676 | } 677 | 678 | /** 679 | * Read fully count number of bytes 680 | * 681 | * @param count how many bytes tp read 682 | * @return a byte array of length count, fully read in 683 | * @throws IOException if an I/O error occurrs. 684 | */ 685 | public byte[] readBytes(int count) throws IOException { 686 | byte[] b = new byte[count]; 687 | readFully(b); 688 | return b; 689 | } 690 | 691 | /** 692 | * Reads b.length bytes from this file into the byte 693 | * array. This method reads repeatedly from the file until all the 694 | * bytes are read. This method blocks until all the bytes are read, 695 | * the end of the stream is detected, or an exception is thrown. 696 | * 697 | * @param b the buffer into which the data is read. 698 | * @throws EOFException if this file reaches the end before reading 699 | * all the bytes. 700 | * @throws IOException if an I/O error occurs. 701 | */ 702 | public final void readFully(byte b[]) throws IOException { 703 | readFully(b, 0, b.length); 704 | } 705 | 706 | /** 707 | * Reads exactly len bytes from this file into the byte 708 | * array. This method reads repeatedly from the file until all the 709 | * bytes are read. This method blocks until all the bytes are read, 710 | * the end of the stream is detected, or an exception is thrown. 711 | * 712 | * @param b the buffer into which the data is read. 713 | * @param off the start offset of the data. 714 | * @param len the number of bytes to read. 715 | * @throws EOFException if this file reaches the end before reading 716 | * all the bytes. 717 | * @throws IOException if an I/O error occurs. 718 | */ 719 | public final void readFully(byte b[], int off, int len) throws IOException { 720 | int n = 0; 721 | while (n < len) { 722 | int count = this.read(b, off + n, len - n); 723 | if (count < 0) { 724 | throw new EOFException(); 725 | } 726 | n += count; 727 | } 728 | } 729 | 730 | /** 731 | * Skips exactly n bytes of input. 732 | * This method blocks until all the bytes are skipped, the end of 733 | * the stream is detected, or an exception is thrown. 734 | * 735 | * @param n the number of bytes to be skipped. 736 | * @return the number of bytes skipped, which is always n. 737 | * @throws EOFException if this file reaches the end before skipping 738 | * all the bytes. 739 | * @throws IOException if an I/O error occurs. 740 | */ 741 | public int skipBytes(int n) throws IOException { 742 | seek(getFilePointer() + n); 743 | return n; 744 | } 745 | 746 | /* public void skipToMultiple( int multipleOfBytes) throws IOException { 747 | long pos = getFilePointer(); 748 | int pad = (int) (pos % multipleOfBytes); 749 | if (pad != 0) pad = multipleOfBytes - pad; 750 | if (pad > 0) skipBytes(pad); 751 | } */ 752 | 753 | /** 754 | * Unread the last byte read. 755 | * This method should not be used more than once 756 | * between reading operations, or strange things might happen. 757 | */ 758 | public void unread() { 759 | filePosition--; 760 | } 761 | 762 | // 763 | // Write primitives. 764 | // 765 | 766 | /** 767 | * Write a byte to the file. If the file has not been opened for 768 | * writing, an IOException will be raised only when an attempt is 769 | * made to write the buffer to the file. 770 | * 771 | * Caveat: the effects of seek( )ing beyond the end of the file are 772 | * undefined. 773 | * 774 | * @param b write this byte 775 | * @throws IOException if an I/O error occurrs. 776 | */ 777 | public void write(int b) throws IOException { 778 | 779 | // If the file position is within the block of data... 780 | if (filePosition < dataEnd) { 781 | int pos = (int) (filePosition - bufferStart); 782 | buffer[pos] = (byte) b; 783 | bufferModified = true; 784 | filePosition++; 785 | 786 | // ...or (assuming that seek will not allow the file pointer 787 | // to move beyond the end of the file) get the correct block of 788 | // data... 789 | } else { 790 | 791 | // If there is room in the buffer, expand it... 792 | if (dataSize != buffer.length) { 793 | int pos = (int) (filePosition - bufferStart); 794 | buffer[pos] = (byte) b; 795 | bufferModified = true; 796 | filePosition++; 797 | dataSize++; 798 | dataEnd++; 799 | 800 | // ...or do another seek to get a new buffer, and start again... 801 | } else { 802 | seek(filePosition); 803 | write(b); 804 | } 805 | } 806 | } 807 | 808 | /** 809 | * Write len bytes from an array to the file. 810 | * 811 | * @param b the array containing the data. 812 | * @param off the offset in the array to the data. 813 | * @param len the length of the data. 814 | * @throws IOException if an I/O error occurrs. 815 | */ 816 | public void writeBytes(byte b[], int off, int len) throws IOException { 817 | // If the amount of data is small (less than a full buffer)... 818 | if (len < buffer.length) { 819 | 820 | // If any of the data fits within the buffer... 821 | int spaceInBuffer = 0; 822 | int copyLength = 0; 823 | if (filePosition >= bufferStart) { 824 | spaceInBuffer = (int) ((bufferStart + buffer.length) - filePosition); 825 | } 826 | 827 | if (spaceInBuffer > 0) { 828 | // Copy as much as possible to the buffer. 829 | copyLength = (spaceInBuffer > len) ? len : spaceInBuffer; 830 | System.arraycopy(b, off, buffer, (int) (filePosition - bufferStart), copyLength); 831 | bufferModified = true; 832 | long myDataEnd = filePosition + copyLength; 833 | dataEnd = (myDataEnd > dataEnd) ? myDataEnd : dataEnd; 834 | dataSize = (int) (dataEnd - bufferStart); 835 | filePosition += copyLength; 836 | ///System.out.println("--copy to buffer "+copyLength+" "+len); 837 | } 838 | 839 | // If there is any data remaining, move to the new position and copy to 840 | // the new buffer. 841 | if (copyLength < len) { 842 | //System.out.println("--need more "+copyLength+" "+len+" space= "+spaceInBuffer); 843 | seek(filePosition); // triggers a flush 844 | System.arraycopy(b, off + copyLength, buffer, (int) (filePosition - bufferStart), len - copyLength); 845 | bufferModified = true; 846 | long myDataEnd = filePosition + (len - copyLength); 847 | dataEnd = (myDataEnd > dataEnd) ? myDataEnd : dataEnd; 848 | dataSize = (int) (dataEnd - bufferStart); 849 | filePosition += (len - copyLength); 850 | } 851 | 852 | // ...or write a lot of data... 853 | } else { 854 | 855 | // Flush the current buffer, and write this data to the file. 856 | if (bufferModified) { 857 | flush(); 858 | } 859 | file.seek(filePosition); // moved per Steve Cerruti; Jan 14, 2005 860 | file.write(b, off, len); 861 | //System.out.println("--write at "+filePosition+" "+len); 862 | 863 | filePosition += len; 864 | bufferStart = filePosition; // an empty buffer 865 | dataSize = 0; 866 | dataEnd = bufferStart + dataSize; 867 | } 868 | } 869 | 870 | /** 871 | * Writes b.length bytes from the specified byte array 872 | * starting at offset off to this file. 873 | * 874 | * @param b the data. 875 | * @throws IOException if an I/O error occurs. 876 | */ 877 | public void write(byte b[]) throws IOException { 878 | writeBytes(b, 0, b.length); 879 | } 880 | 881 | /** 882 | * Writes len bytes from the specified byte array 883 | * starting at offset off to this file. 884 | * 885 | * @param b the data. 886 | * @param off the start offset in the data. 887 | * @param len the number of bytes to write. 888 | * @throws IOException if an I/O error occurs. 889 | */ 890 | public void write(byte b[], int off, int len) throws IOException { 891 | writeBytes(b, off, len); 892 | } 893 | 894 | // 895 | // DataInput methods. 896 | // 897 | 898 | /** 899 | * Reads a boolean from this file. This method reads a 900 | * single byte from the file. A value of 0 represents 901 | * false. Any other value represents true. 902 | * This method blocks until the byte is read, the end of the stream 903 | * is detected, or an exception is thrown. 904 | * 905 | * @return the boolean value read. 906 | * @throws EOFException if this file has reached the end. 907 | * @throws IOException if an I/O error occurs. 908 | */ 909 | public final boolean readBoolean() throws IOException { 910 | int ch = this.read(); 911 | if (ch < 0) { 912 | throw new EOFException(); 913 | } 914 | return (ch != 0); 915 | } 916 | 917 | /** 918 | * Reads a signed 8-bit value from this file. This method reads a 919 | * byte from the file. If the byte read is b, where 920 | * 0 <= b <= 255, 921 | * then the result is: 922 | * 923 | * (byte)(b) 924 | * 925 | * 926 | * This method blocks until the byte is read, the end of the stream 927 | * is detected, or an exception is thrown. 928 | * 929 | * @return the next byte of this file as a signed 8-bit 930 | * byte. 931 | * @throws EOFException if this file has reached the end. 932 | * @throws IOException if an I/O error occurs. 933 | */ 934 | public final byte readByte() throws IOException { 935 | int ch = this.read(); 936 | if (ch < 0) { 937 | throw new EOFException(); 938 | } 939 | return (byte) (ch); 940 | } 941 | 942 | /** 943 | * Reads an unsigned 8-bit number from this file. This method reads 944 | * a byte from this file and returns that byte. 945 | * 946 | * This method blocks until the byte is read, the end of the stream 947 | * is detected, or an exception is thrown. 948 | * 949 | * @return the next byte of this file, interpreted as an unsigned 950 | * 8-bit number. 951 | * @throws EOFException if this file has reached the end. 952 | * @throws IOException if an I/O error occurs. 953 | */ 954 | public final int readUnsignedByte() throws IOException { 955 | int ch = this.read(); 956 | if (ch < 0) { 957 | throw new EOFException(); 958 | } 959 | return ch; 960 | } 961 | 962 | /** 963 | * Reads a signed 16-bit number from this file. The method reads 2 964 | * bytes from this file. If the two bytes read, in order, are 965 | * b1 and b2, where each of the two values is 966 | * between 0 and 255, inclusive, then the 967 | * result is equal to: 968 | * 969 | * (short)((b1 << 8) | b2) 970 | * 971 | * 972 | * This method blocks until the two bytes are read, the end of the 973 | * stream is detected, or an exception is thrown. 974 | * 975 | * @return the next two bytes of this file, interpreted as a signed 976 | * 16-bit number. 977 | * @throws EOFException if this file reaches the end before reading 978 | * two bytes. 979 | * @throws IOException if an I/O error occurs. 980 | */ 981 | public final short readShort() throws IOException { 982 | int ch1 = this.read(); 983 | int ch2 = this.read(); 984 | if ((ch1 | ch2) < 0) { 985 | throw new EOFException(); 986 | } 987 | if (bigEndian) { 988 | return (short) ((ch1 << 8) + (ch2)); 989 | } else { 990 | return (short) ((ch2 << 8) + (ch1)); 991 | } 992 | } 993 | 994 | /** 995 | * Read an array of shorts 996 | * 997 | * @param pa read into this array 998 | * @param start starting at pa[start] 999 | * @param n read this many elements 1000 | * @throws IOException on read error 1001 | */ 1002 | public final void readShort(short[] pa, int start, int n) throws IOException { 1003 | for (int i = 0; i < n; i++) { 1004 | pa[start + i] = readShort(); 1005 | } 1006 | } 1007 | 1008 | /** 1009 | * Reads an unsigned 16-bit number from this file. This method reads 1010 | * two bytes from the file. If the bytes read, in order, are 1011 | * b1 and b2, where 1012 | * 0 <= b1, b2 <= 255, 1013 | * then the result is equal to: 1014 | * 1015 | * (b1 << 8) | b2 1016 | * 1017 | * 1018 | * This method blocks until the two bytes are read, the end of the 1019 | * stream is detected, or an exception is thrown. 1020 | * 1021 | * @return the next two bytes of this file, interpreted as an unsigned 1022 | * 16-bit integer. 1023 | * @throws EOFException if this file reaches the end before reading 1024 | * two bytes. 1025 | * @throws IOException if an I/O error occurs. 1026 | */ 1027 | public final int readUnsignedShort() throws IOException { 1028 | int ch1 = this.read(); 1029 | int ch2 = this.read(); 1030 | if ((ch1 | ch2) < 0) { 1031 | throw new EOFException(); 1032 | } 1033 | if (bigEndian) { 1034 | return ((ch1 << 8) + (ch2)); 1035 | } else { 1036 | return ((ch2 << 8) + (ch1)); 1037 | } 1038 | } 1039 | 1040 | 1041 | /* 1042 | * Reads a signed 24-bit integer from this file. This method reads 3 1043 | * bytes from the file. If the bytes read, in order, are b1, 1044 | * b2, and b3, where 1045 | * 0 <= b1, b2, b3 <= 255, 1046 | * then the result is equal to: 1047 | * 1048 | * (b1 << 16) | (b2 << 8) + (b3 << 0) 1049 | * 1050 | * 1051 | * This method blocks until the three bytes are read, the end of the 1052 | * stream is detected, or an exception is thrown. 1053 | */ 1054 | 1055 | /** 1056 | * Reads a Unicode character from this file. This method reads two 1057 | * bytes from the file. If the bytes read, in order, are 1058 | * b1 and b2, where 1059 | * 0 <= b1, b2 <= 255, 1060 | * then the result is equal to: 1061 | * 1062 | * (char)((b1 << 8) | b2) 1063 | * 1064 | * 1065 | * This method blocks until the two bytes are read, the end of the 1066 | * stream is detected, or an exception is thrown. 1067 | * 1068 | * @return the next two bytes of this file as a Unicode character. 1069 | * @throws EOFException if this file reaches the end before reading 1070 | * two bytes. 1071 | * @throws IOException if an I/O error occurs. 1072 | */ 1073 | public final char readChar() throws IOException { 1074 | int ch1 = this.read(); 1075 | int ch2 = this.read(); 1076 | if ((ch1 | ch2) < 0) { 1077 | throw new EOFException(); 1078 | } 1079 | if (bigEndian) { 1080 | return (char) ((ch1 << 8) + (ch2)); 1081 | } else { 1082 | return (char) ((ch2 << 8) + (ch1)); 1083 | } 1084 | } 1085 | 1086 | /** 1087 | * Reads a signed 32-bit integer from this file. This method reads 4 1088 | * bytes from the file. If the bytes read, in order, are b1, 1089 | * b2, b3, and b4, where 1090 | * 0 <= b1, b2, b3, b4 <= 255, 1091 | * then the result is equal to: 1092 | * 1093 | * (b1 << 24) | (b2 << 16) + (b3 << 8) + b4 1094 | * 1095 | * 1096 | * This method blocks until the four bytes are read, the end of the 1097 | * stream is detected, or an exception is thrown. 1098 | * 1099 | * @return the next four bytes of this file, interpreted as an 1100 | * int. 1101 | * @throws EOFException if this file reaches the end before reading 1102 | * four bytes. 1103 | * @throws IOException if an I/O error occurs. 1104 | */ 1105 | public final int readInt() throws IOException { 1106 | int ch1 = this.read(); 1107 | int ch2 = this.read(); 1108 | int ch3 = this.read(); 1109 | int ch4 = this.read(); 1110 | if ((ch1 | ch2 | ch3 | ch4) < 0) { 1111 | throw new EOFException(); 1112 | } 1113 | 1114 | if (bigEndian) { 1115 | return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4)); 1116 | } else { 1117 | return ((ch4 << 24) + (ch3 << 16) + (ch2 << 8) + (ch1)); 1118 | } 1119 | } 1120 | 1121 | /** 1122 | * Read an integer at the given position, bypassing all buffering. 1123 | * 1124 | * @param pos read a byte at this position 1125 | * @return The int that was read 1126 | * @throws IOException if an I/O error occurs. 1127 | */ 1128 | public final int readIntUnbuffered(long pos) throws IOException { 1129 | byte[] bb = new byte[4]; 1130 | read_(pos, bb, 0, 4); 1131 | int ch1 = bb[0] & 0xff; 1132 | int ch2 = bb[1] & 0xff; 1133 | int ch3 = bb[2] & 0xff; 1134 | int ch4 = bb[3] & 0xff; 1135 | if ((ch1 | ch2 | ch3 | ch4) < 0) { 1136 | throw new EOFException(); 1137 | } 1138 | 1139 | if (bigEndian) { 1140 | return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4)); 1141 | } else { 1142 | return ((ch4 << 24) + (ch3 << 16) + (ch2 << 8) + (ch1)); 1143 | } 1144 | } 1145 | 1146 | 1147 | /** 1148 | * Read an array of ints 1149 | * 1150 | * @param pa read into this array 1151 | * @param start starting at pa[start] 1152 | * @param n read this many elements 1153 | * @throws IOException on read error 1154 | */ 1155 | public final void readInt(int[] pa, int start, int n) throws IOException { 1156 | for (int i = 0; i < n; i++) { 1157 | pa[start + i] = readInt(); 1158 | } 1159 | } 1160 | 1161 | /** 1162 | * Reads a signed 64-bit integer from this file. This method reads eight 1163 | * bytes from the file. If the bytes read, in order, are 1164 | * b1, b2, b3, 1165 | * b4, b5, b6, 1166 | * b7, and b8, where: 1167 | * 1168 | * 0 <= b1, b2, b3, b4, b5, b6, b7, b8 <=255, 1169 | * 1170 | * 1171 | * then the result is equal to: 1172 | * 1173 | * ((long)b1 << 56) + ((long)b2 << 48) 1174 | * + ((long)b3 << 40) + ((long)b4 << 32) 1175 | * + ((long)b5 << 24) + ((long)b6 << 16) 1176 | * + ((long)b7 << 8) + b8 1177 | * 1178 | * 1179 | * This method blocks until the eight bytes are read, the end of the 1180 | * stream is detected, or an exception is thrown. 1181 | * 1182 | * @return the next eight bytes of this file, interpreted as a 1183 | * long. 1184 | * @throws EOFException if this file reaches the end before reading 1185 | * eight bytes. 1186 | * @throws IOException if an I/O error occurs. 1187 | */ 1188 | public final long readLong() throws IOException { 1189 | if (bigEndian) { 1190 | return ((long) (readInt()) << 32) + (readInt() & 0xFFFFFFFFL); // tested ok 1191 | } else { 1192 | return ((readInt() & 0xFFFFFFFFL) + ((long) readInt() << 32)); // not tested yet ?? 1193 | } 1194 | 1195 | /* int ch1 = this.read(); 1196 | int ch2 = this.read(); 1197 | int ch3 = this.read(); 1198 | int ch4 = this.read(); 1199 | int ch5 = this.read(); 1200 | int ch6 = this.read(); 1201 | int ch7 = this.read(); 1202 | int ch8 = this.read(); 1203 | if ((ch1 | ch2 | ch3 | ch4 | ch5 | ch6 | ch7 | ch8) < 0) 1204 | throw new EOFException(); 1205 | 1206 | if (bigEndian) 1207 | return ((long)(ch1 << 56)) + (ch2 << 48) + (ch3 << 40) + (ch4 << 32) + (ch5 << 24) + (ch6 << 16) + (ch7 << 8) + (ch8 << 0)); 1208 | else 1209 | return ((long)(ch8 << 56) + (ch7 << 48) + (ch6 << 40) + (ch5 << 32) + (ch4 << 24) + (ch3 << 16) + (ch2 << 8) + (ch1 << 0)); 1210 | */ 1211 | } 1212 | 1213 | /** 1214 | * Read an array of longs 1215 | * 1216 | * @param pa read into this array 1217 | * @param start starting at pa[start] 1218 | * @param n read this many elements 1219 | * @throws IOException on read error 1220 | */ 1221 | public final void readLong(long[] pa, int start, int n) throws IOException { 1222 | for (int i = 0; i < n; i++) { 1223 | pa[start + i] = readLong(); 1224 | } 1225 | } 1226 | 1227 | 1228 | /** 1229 | * Reads a float from this file. This method reads an 1230 | * int value as if by the readInt method 1231 | * and then converts that int to a float 1232 | * using the intBitsToFloat method in class 1233 | * Float. 1234 | * 1235 | * This method blocks until the four bytes are read, the end of the 1236 | * stream is detected, or an exception is thrown. 1237 | * 1238 | * @return the next four bytes of this file, interpreted as a 1239 | * float. 1240 | * @throws EOFException if this file reaches the end before reading 1241 | * four bytes. 1242 | * @throws IOException if an I/O error occurs. 1243 | * @see java.io.RandomAccessFile#readInt() 1244 | * @see java.lang.Float#intBitsToFloat(int) 1245 | */ 1246 | public final float readFloat() throws IOException { 1247 | return Float.intBitsToFloat(readInt()); 1248 | } 1249 | 1250 | /** 1251 | * Read an array of floats 1252 | * 1253 | * @param pa read into this array 1254 | * @param start starting at pa[start] 1255 | * @param n read this many elements 1256 | * @throws IOException on read error 1257 | */ 1258 | public final void readFloat(float[] pa, int start, int n) throws IOException { 1259 | for (int i = 0; i < n; i++) { 1260 | pa[start + i] = Float.intBitsToFloat(readInt()); 1261 | } 1262 | } 1263 | 1264 | 1265 | /** 1266 | * Reads a double from this file. This method reads a 1267 | * long value as if by the readLong method 1268 | * and then converts that long to a double 1269 | * using the longBitsToDouble method in 1270 | * class Double. 1271 | * 1272 | * This method blocks until the eight bytes are read, the end of the 1273 | * stream is detected, or an exception is thrown. 1274 | * 1275 | * @return the next eight bytes of this file, interpreted as a 1276 | * double. 1277 | * @throws EOFException if this file reaches the end before reading 1278 | * eight bytes. 1279 | * @throws IOException if an I/O error occurs. 1280 | * @see java.io.RandomAccessFile#readLong() 1281 | * @see java.lang.Double#longBitsToDouble(long) 1282 | */ 1283 | public final double readDouble() throws IOException { 1284 | return Double.longBitsToDouble(readLong()); 1285 | } 1286 | 1287 | /** 1288 | * Read an array of doubles 1289 | * 1290 | * @param pa read into this array 1291 | * @param start starting at pa[start] 1292 | * @param n read this many elements 1293 | * @throws IOException on read error 1294 | */ 1295 | public final void readDouble(double[] pa, int start, int n) throws IOException { 1296 | for (int i = 0; i < n; i++) { 1297 | pa[start + i] = Double.longBitsToDouble(readLong()); 1298 | } 1299 | } 1300 | 1301 | /** 1302 | * Reads the next line of text from this file. This method 1303 | * successively reads bytes from the file until it reaches the end of 1304 | * a line of text. 1305 | * 1306 | * 1307 | * A line of text is terminated by a carriage-return character 1308 | * ('\r'), a newline character ('\n'), a 1309 | * carriage-return character immediately followed by a newline 1310 | * character, or the end of the input stream. The line-terminating 1311 | * character(s), if any, are included as part of the string returned. 1312 | * 1313 | * 1314 | * This method blocks until a newline character is read, a carriage 1315 | * return and the byte following it are read (to see if it is a 1316 | * newline), the end of the stream is detected, or an exception is thrown. 1317 | * 1318 | * @return the next line of text from this file. 1319 | * @throws IOException if an I/O error occurs. 1320 | */ 1321 | public final String readLine() throws IOException { 1322 | StringBuilder input = new StringBuilder(); 1323 | int c; 1324 | 1325 | while (((c = read()) != -1) && (c != '\n')) { 1326 | input.append((char) c); 1327 | } 1328 | if ((c == -1) && (input.length() == 0)) { 1329 | return null; 1330 | } 1331 | return input.toString(); 1332 | } 1333 | 1334 | /** 1335 | * Reads in a string from this file. The string has been encoded 1336 | * using a modified UTF-8 format. 1337 | * 1338 | * The first two bytes are read as if by 1339 | * readUnsignedShort. This value gives the number of 1340 | * following bytes that are in the encoded string, not 1341 | * the length of the resulting string. The following bytes are then 1342 | * interpreted as bytes encoding characters in the UTF-8 format 1343 | * and are converted into characters. 1344 | * 1345 | * This method blocks until all the bytes are read, the end of the 1346 | * stream is detected, or an exception is thrown. 1347 | * 1348 | * @return a Unicode string. 1349 | * @throws EOFException if this file reaches the end before 1350 | * reading all the bytes. 1351 | * @throws IOException if an I/O error occurs. 1352 | * @throws UTFDataFormatException if the bytes do not represent 1353 | * valid UTF-8 encoding of a Unicode string. 1354 | * @see java.io.RandomAccessFile#readUnsignedShort() 1355 | */ 1356 | public final String readUTF() throws IOException { 1357 | return DataInputStream.readUTF(this); 1358 | } 1359 | 1360 | /** 1361 | * Read a String of knoen length. 1362 | * 1363 | * @param nbytes number of bytes to read 1364 | * @return String wrapping the bytes. 1365 | * @throws IOException if an I/O error occurs. 1366 | */ 1367 | public String readString(int nbytes) throws IOException { 1368 | byte[] data = new byte[nbytes]; 1369 | readFully(data); 1370 | return new String(data); 1371 | } 1372 | 1373 | // 1374 | // DataOutput methods. 1375 | // 1376 | 1377 | /** 1378 | * Writes a boolean to the file as a 1-byte value. The 1379 | * value true is written out as the value 1380 | * (byte)1; the value false is written out 1381 | * as the value (byte)0. 1382 | * 1383 | * @param v a boolean value to be written. 1384 | * @throws IOException if an I/O error occurs. 1385 | */ 1386 | public final void writeBoolean(boolean v) throws IOException { 1387 | write(v ? 1 : 0); 1388 | } 1389 | 1390 | /** 1391 | * Write an array of booleans 1392 | * 1393 | * @param pa write from this array 1394 | * @param start starting with this element in the array 1395 | * @param n write this number of elements 1396 | * @throws IOException on read error 1397 | */ 1398 | public final void writeBoolean(boolean[] pa, int start, int n) throws IOException { 1399 | for (int i = 0; i < n; i++) { 1400 | writeBoolean(pa[start + i]); 1401 | } 1402 | } 1403 | 1404 | /** 1405 | * Writes a byte to the file as a 1-byte value. 1406 | * 1407 | * @param v a byte value to be written. 1408 | * @throws IOException if an I/O error occurs. 1409 | */ 1410 | public final void writeByte(int v) throws IOException { 1411 | write(v); 1412 | } 1413 | 1414 | /** 1415 | * Writes a short to the file as two bytes, high byte first. 1416 | * 1417 | * @param v a short to be written. 1418 | * @throws IOException if an I/O error occurs. 1419 | */ 1420 | public final void writeShort(int v) throws IOException { 1421 | write((v >>> 8) & 0xFF); 1422 | write((v) & 0xFF); 1423 | } 1424 | 1425 | /** 1426 | * Write an array of shorts 1427 | * 1428 | * @param pa write from this array 1429 | * @param start starting with this element in the array 1430 | * @param n this number of elements 1431 | * @throws IOException on read error 1432 | */ 1433 | public final void writeShort(short[] pa, int start, int n) throws IOException { 1434 | for (int i = 0; i < n; i++) { 1435 | writeShort(pa[start + i]); 1436 | } 1437 | } 1438 | 1439 | /** 1440 | * Writes a char to the file as a 2-byte value, high 1441 | * byte first. 1442 | * 1443 | * @param v a char value to be written. 1444 | * @throws IOException if an I/O error occurs. 1445 | */ 1446 | public final void writeChar(int v) throws IOException { 1447 | write((v >>> 8) & 0xFF); 1448 | write((v) & 0xFF); 1449 | } 1450 | 1451 | /** 1452 | * Write an array of chars 1453 | * 1454 | * @param pa write from this array 1455 | * @param start starting with this element in the array 1456 | * @param n this number of elements 1457 | * @throws IOException on read error 1458 | */ 1459 | public final void writeChar(char[] pa, int start, int n) throws IOException { 1460 | for (int i = 0; i < n; i++) { 1461 | writeChar(pa[start + i]); 1462 | } 1463 | } 1464 | 1465 | 1466 | /** 1467 | * Writes an int to the file as four bytes, high byte first. 1468 | * 1469 | * @param v an int to be written. 1470 | * @throws IOException if an I/O error occurs. 1471 | */ 1472 | public final void writeInt(int v) throws IOException { 1473 | write((v >>> 24) & 0xFF); 1474 | write((v >>> 16) & 0xFF); 1475 | write((v >>> 8) & 0xFF); 1476 | write((v) & 0xFF); 1477 | } 1478 | 1479 | /** 1480 | * Write an array of ints 1481 | * 1482 | * @param pa write from this array 1483 | * @param start starting with this element in the array 1484 | * @param n write this number of elements 1485 | * @throws IOException on read error 1486 | */ 1487 | public final void writeInt(int[] pa, int start, int n) throws IOException { 1488 | for (int i = 0; i < n; i++) { 1489 | writeInt(pa[start + i]); 1490 | } 1491 | } 1492 | 1493 | /** 1494 | * Writes a long to the file as eight bytes, high byte first. 1495 | * 1496 | * @param v a long to be written. 1497 | * @throws IOException if an I/O error occurs. 1498 | */ 1499 | public final void writeLong(long v) throws IOException { 1500 | write((int) (v >>> 56) & 0xFF); 1501 | write((int) (v >>> 48) & 0xFF); 1502 | write((int) (v >>> 40) & 0xFF); 1503 | write((int) (v >>> 32) & 0xFF); 1504 | write((int) (v >>> 24) & 0xFF); 1505 | write((int) (v >>> 16) & 0xFF); 1506 | write((int) (v >>> 8) & 0xFF); 1507 | write((int) (v) & 0xFF); 1508 | } 1509 | 1510 | /** 1511 | * Write an array of longs 1512 | * 1513 | * @param pa write from this array 1514 | * @param start starting with this element in the array 1515 | * @param n write this number of elements 1516 | * @throws IOException on read error 1517 | */ 1518 | public final void writeLong(long[] pa, int start, int n) throws IOException { 1519 | for (int i = 0; i < n; i++) { 1520 | writeLong(pa[start + i]); 1521 | } 1522 | } 1523 | 1524 | /** 1525 | * Converts the float argument to an int using the 1526 | * floatToIntBits method in class Float, 1527 | * and then writes that int value to the file as a 1528 | * 4-byte quantity, high byte first. 1529 | * 1530 | * @param v a float value to be written. 1531 | * @throws IOException if an I/O error occurs. 1532 | * @see java.lang.Float#floatToIntBits(float) 1533 | */ 1534 | public final void writeFloat(float v) throws IOException { 1535 | writeInt(Float.floatToIntBits(v)); 1536 | } 1537 | 1538 | /** 1539 | * Write an array of floats 1540 | * 1541 | * @param pa write from this array 1542 | * @param start starting with this element in the array 1543 | * @param n write this number of elements 1544 | * @throws IOException on read error 1545 | */ 1546 | public final void writeFloat(float[] pa, int start, int n) throws IOException { 1547 | for (int i = 0; i < n; i++) { 1548 | writeFloat(pa[start + i]); 1549 | } 1550 | } 1551 | 1552 | 1553 | /** 1554 | * Converts the double argument to a long using the 1555 | * doubleToLongBits method in class Double, 1556 | * and then writes that long value to the file as an 1557 | * 8-byte quantity, high byte first. 1558 | * 1559 | * @param v a double value to be written. 1560 | * @throws IOException if an I/O error occurs. 1561 | * @see java.lang.Double#doubleToLongBits(double) 1562 | */ 1563 | public final void writeDouble(double v) throws IOException { 1564 | writeLong(Double.doubleToLongBits(v)); 1565 | } 1566 | 1567 | /** 1568 | * Write an array of doubles 1569 | * 1570 | * @param pa write from this array 1571 | * @param start starting with this element in the array 1572 | * @param n write this number of elements 1573 | * @throws IOException on read error 1574 | */ 1575 | public final void writeDouble(double[] pa, int start, int n) throws IOException { 1576 | for (int i = 0; i < n; i++) { 1577 | writeDouble(pa[start + i]); 1578 | } 1579 | } 1580 | 1581 | /** 1582 | * Writes the string to the file as a sequence of bytes. Each 1583 | * character in the string is written out, in sequence, by discarding 1584 | * its high eight bits. 1585 | * 1586 | * @param s a string of bytes to be written. 1587 | * @throws IOException if an I/O error occurs. 1588 | */ 1589 | public final void writeBytes(String s) throws IOException { 1590 | int len = s.length(); 1591 | for (int i = 0; i < len; i++) { 1592 | write((byte) s.charAt(i)); 1593 | } 1594 | } 1595 | 1596 | /** 1597 | * Writes the character array to the file as a sequence of bytes. Each 1598 | * character in the string is written out, in sequence, by discarding 1599 | * its high eight bits. 1600 | * 1601 | * @param b a character array of bytes to be written. 1602 | * @param off the index of the first character to write. 1603 | * @param len the number of characters to write. 1604 | * @throws IOException if an I/O error occurs. 1605 | */ 1606 | public final void writeBytes(char b[], int off, int len) throws IOException { 1607 | for (int i = off; i < len; i++) { 1608 | write((byte) b[i]); 1609 | } 1610 | } 1611 | 1612 | /** 1613 | * Writes a string to the file as a sequence of characters. Each 1614 | * character is written to the data output stream as if by the 1615 | * writeChar method. 1616 | * 1617 | * @param s a String value to be written. 1618 | * @throws IOException if an I/O error occurs. 1619 | * @see java.io.RandomAccessFile#writeChar(int) 1620 | */ 1621 | public final void writeChars(String s) throws IOException { 1622 | int len = s.length(); 1623 | for (int i = 0; i < len; i++) { 1624 | int v = s.charAt(i); 1625 | write((v >>> 8) & 0xFF); 1626 | write((v) & 0xFF); 1627 | } 1628 | } 1629 | 1630 | /** 1631 | * Writes a string to the file using UTF-8 encoding in a 1632 | * machine-independent manner. 1633 | * 1634 | * First, two bytes are written to the file as if by the 1635 | * writeShort method giving the number of bytes to 1636 | * follow. This value is the number of bytes actually written out, 1637 | * not the length of the string. Following the length, each character 1638 | * of the string is output, in sequence, using the UTF-8 encoding 1639 | * for each character. 1640 | * 1641 | * @param str a string to be written. 1642 | * @throws IOException if an I/O error occurs. 1643 | */ 1644 | public final void writeUTF(String str) throws IOException { 1645 | int strlen = str.length(); 1646 | int utflen = 0; 1647 | 1648 | for (int i = 0; i < strlen; i++) { 1649 | int c = str.charAt(i); 1650 | if ((c >= 0x0001) && (c <= 0x007F)) { 1651 | utflen++; 1652 | } else if (c > 0x07FF) { 1653 | utflen += 3; 1654 | } else { 1655 | utflen += 2; 1656 | } 1657 | } 1658 | if (utflen > 65535) { 1659 | throw new UTFDataFormatException(); 1660 | } 1661 | 1662 | write((utflen >>> 8) & 0xFF); 1663 | write((utflen) & 0xFF); 1664 | for (int i = 0; i < strlen; i++) { 1665 | int c = str.charAt(i); 1666 | if ((c >= 0x0001) && (c <= 0x007F)) { 1667 | write(c); 1668 | } else if (c > 0x07FF) { 1669 | write(0xE0 | ((c >> 12) & 0x0F)); 1670 | write(0x80 | ((c >> 6) & 0x3F)); 1671 | write(0x80 | ((c) & 0x3F)); 1672 | } else { 1673 | write(0xC0 | ((c >> 6) & 0x1F)); 1674 | write(0x80 | ((c) & 0x3F)); 1675 | } 1676 | } 1677 | } 1678 | 1679 | /** 1680 | * Create a string representation of this object. 1681 | * 1682 | * @return a string representation of the state of the object. 1683 | */ 1684 | public String toString() { 1685 | return "fp=" + filePosition + ", bs=" + bufferStart + ", de=" 1686 | + dataEnd + ", ds=" + dataSize + ", bl=" + buffer.length 1687 | + ", readonly=" + readonly + ", bm=" + bufferModified; 1688 | } 1689 | 1690 | ///////////////////////////////////////////////// 1691 | 1692 | /** 1693 | * Search forward from the current pos, looking for a match. 1694 | * 1695 | * @param match the match to look for. 1696 | * @param maxBytes maximum number of bytes to search. use -1 for all 1697 | * @return true if found, file position will be at the start of the match. 1698 | * @throws IOException on read error 1699 | */ 1700 | public boolean searchForward(KMPMatch match, int maxBytes) throws IOException { 1701 | long start = getFilePointer(); 1702 | long last = (maxBytes < 0) ? length() : Math.min(length(), start + maxBytes); 1703 | long needToScan = last - start; 1704 | 1705 | // check what ever is now in the buffer 1706 | int bytesAvailable = (int) (dataEnd - filePosition); 1707 | if (bytesAvailable < 1) { 1708 | seek(filePosition); // read a new buffer 1709 | bytesAvailable = (int) (dataEnd - filePosition); 1710 | } 1711 | int bufStart = (int) (filePosition - bufferStart); 1712 | int scanBytes = (int) Math.min(bytesAvailable, needToScan); 1713 | int pos = match.indexOf(buffer, bufStart, scanBytes); 1714 | if (pos >= 0) { 1715 | seek(bufferStart + pos); 1716 | return true; 1717 | } 1718 | 1719 | int matchLen = match.getMatchLength(); 1720 | needToScan -= scanBytes - matchLen; 1721 | 1722 | while (needToScan > matchLen) { 1723 | readBuffer(dataEnd - matchLen); // force new buffer 1724 | 1725 | scanBytes = (int) Math.min(buffer.length, needToScan); 1726 | pos = match.indexOf(buffer, 0, scanBytes); 1727 | if (pos > 0) { 1728 | seek(bufferStart + pos); 1729 | return true; 1730 | } 1731 | 1732 | needToScan -= scanBytes - matchLen; 1733 | } 1734 | 1735 | // failure 1736 | seek(last); 1737 | return false; 1738 | } 1739 | 1740 | } 1741 | 1742 | 1743 | -------------------------------------------------------------------------------- /src/test/java/info/fetter/logstashforwarder/FileReaderTest.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import static org.apache.log4j.Level.*; 21 | import info.fetter.logstashforwarder.util.AdapterException; 22 | 23 | import java.io.File; 24 | import java.io.IOException; 25 | import java.util.ArrayList; 26 | import java.util.HashMap; 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | import org.apache.commons.io.FileUtils; 31 | import org.apache.log4j.BasicConfigurator; 32 | import org.apache.log4j.Logger; 33 | import org.apache.log4j.spi.RootLogger; 34 | import org.junit.AfterClass; 35 | import org.junit.BeforeClass; 36 | import org.junit.Test; 37 | 38 | public class FileReaderTest { 39 | Logger logger = Logger.getLogger(FileReaderTest.class); 40 | 41 | @BeforeClass 42 | public static void setUpBeforeClass() throws Exception { 43 | BasicConfigurator.configure(); 44 | RootLogger.getRootLogger().setLevel(TRACE); 45 | } 46 | 47 | @AfterClass 48 | public static void tearDownAfterClass() throws Exception { 49 | BasicConfigurator.resetConfiguration(); 50 | } 51 | 52 | @Test 53 | public void testFileReader1() throws IOException, InterruptedException, AdapterException { 54 | FileReader reader = new FileReader(2); 55 | reader.setAdapter(new MockProtocolAdapter()); 56 | List fileList = new ArrayList(1); 57 | File file1 = new File("testFileReader1.txt"); 58 | FileUtils.write(file1, "testFileReader1 line1\n"); 59 | FileUtils.write(file1, " nl line12\n", true); 60 | FileUtils.write(file1, "testFileReader1 line2\n", true); 61 | FileUtils.write(file1, "testFileReader1 line3\n", true); 62 | Thread.sleep(500); 63 | FileState state = new FileState(file1); 64 | fileList.add(state); 65 | state.setFields(new Event().addField("testFileReader1", "testFileReader1")); 66 | Map m = new HashMap(); 67 | m.put("pattern", " nl"); 68 | m.put("negate", "false"); 69 | state.setMultiline(new Multiline(m)); 70 | reader.readFiles(fileList); 71 | reader.readFiles(fileList); 72 | reader.readFiles(fileList); 73 | //FileUtils.forceDelete(file1); 74 | } 75 | 76 | @Test 77 | public void testFileReader2() throws IOException, InterruptedException, AdapterException { 78 | FileReader reader = new FileReader(2); 79 | reader.setAdapter(new MockProtocolAdapter()); 80 | List fileList = new ArrayList(1); 81 | File file1 = new File("testFileReader1.txt"); 82 | FileUtils.write(file1, "testFileReader1 line1\n"); 83 | FileUtils.write(file1, " nl line12\n", true); 84 | FileUtils.write(file1, "testFileReader1 line2\n", true); 85 | FileUtils.write(file1, "testFileReader1 line3\n", true); 86 | Thread.sleep(500); 87 | FileState state = new FileState(file1); 88 | fileList.add(state); 89 | state.setFields(new Event().addField("testFileReader1", "testFileReader1")); 90 | Map m = new HashMap(); 91 | m.put("pattern", "testFileReader1"); 92 | m.put("negate", "true"); 93 | state.setMultiline(new Multiline(m)); 94 | reader.readFiles(fileList); 95 | reader.readFiles(fileList); 96 | reader.readFiles(fileList); 97 | //FileUtils.forceDelete(file1); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/test/java/info/fetter/logstashforwarder/FileWatcherTest.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import static org.apache.log4j.Level.*; 21 | 22 | import java.io.File; 23 | import java.io.IOException; 24 | import java.util.HashMap; 25 | import java.util.Map; 26 | 27 | import org.apache.commons.io.FileUtils; 28 | import org.apache.log4j.BasicConfigurator; 29 | import org.apache.log4j.Logger; 30 | import org.apache.log4j.spi.RootLogger; 31 | import org.junit.AfterClass; 32 | import org.junit.BeforeClass; 33 | import org.junit.Test; 34 | 35 | public class FileWatcherTest { 36 | Logger logger = Logger.getLogger(FileWatcherTest.class); 37 | 38 | @BeforeClass 39 | public static void setUpBeforeClass() throws Exception { 40 | BasicConfigurator.configure(); 41 | RootLogger.getRootLogger().setLevel(TRACE); 42 | } 43 | 44 | @AfterClass 45 | public static void tearDownAfterClass() throws Exception { 46 | BasicConfigurator.resetConfiguration(); 47 | } 48 | 49 | //@Test 50 | public void testFileWatch() throws InterruptedException, IOException { 51 | FileWatcher watcher = new FileWatcher(); 52 | watcher.addFilesToWatch("./test.txt", new Event().addField("test", "test"), FileWatcher.ONE_DAY, null, null); 53 | for(int i = 0; i < 100; i++) { 54 | Thread.sleep(1000); 55 | watcher.checkFiles(); 56 | } 57 | } 58 | 59 | //@Test 60 | public void testFileWatchWithMultilines() throws InterruptedException, IOException { 61 | FileWatcher watcher = new FileWatcher(); 62 | Multiline multiline = new Multiline(); 63 | watcher.addFilesToWatch("./test.txt", new Event().addField("test", "test"), FileWatcher.ONE_DAY, multiline, null); 64 | for(int i = 0; i < 100; i++) { 65 | Thread.sleep(1000); 66 | watcher.checkFiles(); 67 | } 68 | } 69 | 70 | //@Test 71 | public void testWildcardWatch() throws InterruptedException, IOException { 72 | if(System.getProperty("os.name").toLowerCase().contains("win")) { 73 | logger.warn("Not executing this test on windows"); 74 | return; 75 | } 76 | FileWatcher watcher = new FileWatcher(); 77 | watcher.addFilesToWatch("./testFileWatcher*.txt", new Event().addField("test", "test"), FileWatcher.ONE_DAY, null, null); 78 | watcher.initialize(); 79 | 80 | File file1 = new File("testFileWatcher1.txt"); 81 | File file2 = new File("testFileWatcher2.txt"); 82 | //File file3 = new File("test3.txt"); 83 | //File file4 = new File("test4.txt"); 84 | 85 | //File testDir = new File("testFileWatcher"); 86 | //FileUtils.forceMkdir(new File("test")); 87 | 88 | watcher.checkFiles(); 89 | Thread.sleep(100); 90 | FileUtils.write(file1, "file 1 line 1\n", true); 91 | Thread.sleep(100); 92 | watcher.checkFiles(); 93 | FileUtils.write(file1, "file 1 line 2\n", true); 94 | //FileUtils.write(file2, "file 2 line 1\n", true); 95 | Thread.sleep(1000); 96 | watcher.checkFiles(); 97 | // FileUtils.moveFileToDirectory(file1, testDir, true); 98 | // FileUtils.write(file2, "file 2 line 2\n", true); 99 | FileUtils.moveFile(file1, file2); 100 | // FileUtils.write(file2, "file 3 line 1\n", true); 101 | // 102 | Thread.sleep(1000); 103 | watcher.checkFiles(); 104 | // 105 | // 106 | watcher.close(); 107 | FileUtils.deleteQuietly(file1); 108 | FileUtils.deleteQuietly(file2); 109 | // FileUtils.forceDelete(testDir); 110 | 111 | 112 | 113 | } 114 | 115 | @Test 116 | public void testWildcardWatchMultiline() throws InterruptedException, IOException { 117 | if(System.getProperty("os.name").toLowerCase().contains("win")) { 118 | logger.warn("Not executing this test on windows"); 119 | return; 120 | } 121 | FileWatcher watcher = new FileWatcher(); 122 | Map m = new HashMap(); 123 | m.put("pattern", " nl"); 124 | m.put("negate", "false"); 125 | Multiline multiline = new Multiline(m); 126 | watcher.addFilesToWatch("./testFileWatcher*.txt", new Event().addField("test", "test"), FileWatcher.ONE_DAY, multiline, null); 127 | watcher.initialize(); 128 | 129 | File file1 = new File("testFileWatcher1.txt"); 130 | File file2 = new File("testFileWatcher2.txt"); 131 | //File file3 = new File("test3.txt"); 132 | //File file4 = new File("test4.txt"); 133 | 134 | //File testDir = new File("testFileWatcher"); 135 | //FileUtils.forceMkdir(new File("test")); 136 | 137 | watcher.checkFiles(); 138 | Thread.sleep(100); 139 | FileUtils.write(file1, "file 1 line 1\n nl line 1-2", true); 140 | Thread.sleep(100); 141 | watcher.checkFiles(); 142 | FileUtils.write(file1, "file 1 line 2\n", true); 143 | Thread.sleep(100); 144 | watcher.checkFiles(); 145 | FileUtils.write(file1, " nl line 3\n", true); 146 | //FileUtils.write(file2, "file 2 line 1\n", true); 147 | Thread.sleep(1000); 148 | watcher.checkFiles(); 149 | // FileUtils.moveFileToDirectory(file1, testDir, true); 150 | // FileUtils.write(file2, "file 2 line 2\n", true); 151 | FileUtils.moveFile(file1, file2); 152 | // FileUtils.write(file2, "file 3 line 1\n", true); 153 | // 154 | Thread.sleep(1000); 155 | watcher.checkFiles(); 156 | // 157 | // 158 | watcher.close(); 159 | FileUtils.deleteQuietly(file1); 160 | FileUtils.deleteQuietly(file2); 161 | // FileUtils.forceDelete(testDir); 162 | 163 | 164 | 165 | } 166 | 167 | @Test 168 | public void dummy() {} 169 | } 170 | -------------------------------------------------------------------------------- /src/test/java/info/fetter/logstashforwarder/InputReaderTest.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import static org.apache.log4j.Level.*; 21 | import info.fetter.logstashforwarder.util.AdapterException; 22 | 23 | import java.io.IOException; 24 | import java.io.PipedInputStream; 25 | import java.io.PipedOutputStream; 26 | import java.io.PrintWriter; 27 | import org.apache.log4j.BasicConfigurator; 28 | import org.apache.log4j.Logger; 29 | import org.apache.log4j.spi.RootLogger; 30 | import org.junit.AfterClass; 31 | import org.junit.BeforeClass; 32 | import org.junit.Test; 33 | import static org.junit.Assert.*; 34 | 35 | public class InputReaderTest { 36 | Logger logger = Logger.getLogger(InputReaderTest.class); 37 | 38 | @BeforeClass 39 | public static void setUpBeforeClass() throws Exception { 40 | BasicConfigurator.configure(); 41 | RootLogger.getRootLogger().setLevel(TRACE); 42 | } 43 | 44 | @AfterClass 45 | public static void tearDownAfterClass() throws Exception { 46 | BasicConfigurator.resetConfiguration(); 47 | } 48 | 49 | @Test 50 | public void testInputReader1() throws IOException, InterruptedException, AdapterException { 51 | int numberOfEvents = 0; 52 | PipedInputStream in = new PipedInputStream(); 53 | PipedOutputStream out = new PipedOutputStream(in); 54 | PrintWriter writer = new PrintWriter(out); 55 | InputReader reader = new InputReader(2, in); 56 | MockProtocolAdapter adapter = new MockProtocolAdapter(); 57 | reader.setAdapter(adapter); 58 | 59 | numberOfEvents = reader.readInput(); 60 | assertEquals(0, numberOfEvents); 61 | 62 | writer.println("line1"); 63 | writer.flush(); 64 | numberOfEvents = reader.readInput(); 65 | assertEquals(1, numberOfEvents); 66 | assertArrayEquals("line1".getBytes(), adapter.getLastEvents().get(0).getValue("line")); 67 | 68 | writer.print("line2"); 69 | writer.flush(); 70 | numberOfEvents = reader.readInput(); 71 | assertEquals(0, numberOfEvents); 72 | 73 | writer.println(); 74 | writer.flush(); 75 | numberOfEvents = reader.readInput(); 76 | assertEquals(1, numberOfEvents); 77 | assertArrayEquals("line2".getBytes(), adapter.getLastEvents().get(0).getValue("line")); 78 | 79 | writer.println("line3"); 80 | writer.println("line4"); 81 | writer.println("line5"); 82 | writer.flush(); 83 | numberOfEvents = reader.readInput(); 84 | assertEquals(2, numberOfEvents); 85 | assertArrayEquals("line3".getBytes(), adapter.getLastEvents().get(0).getValue("line")); 86 | assertArrayEquals("line4".getBytes(), adapter.getLastEvents().get(1).getValue("line")); 87 | 88 | numberOfEvents = reader.readInput(); 89 | assertEquals(1, numberOfEvents); 90 | assertArrayEquals("line5".getBytes(), adapter.getLastEvents().get(0).getValue("line")); 91 | 92 | numberOfEvents = reader.readInput(); 93 | assertEquals(0, numberOfEvents); 94 | 95 | assertEquals(0, in.available()); 96 | 97 | writer.close(); 98 | } 99 | 100 | @Test 101 | public void testInputReaderCloseStream() throws AdapterException, IOException { 102 | int numberOfEvents = 0; 103 | PipedInputStream in = new PipedInputStream(); 104 | PipedOutputStream out = new PipedOutputStream(in); 105 | PrintWriter writer = new PrintWriter(out); 106 | InputReader reader = new InputReader(2, in); 107 | MockProtocolAdapter adapter = new MockProtocolAdapter(); 108 | reader.setAdapter(adapter); 109 | 110 | numberOfEvents = reader.readInput(); 111 | assertEquals(0, numberOfEvents); 112 | 113 | writer.println("line1"); 114 | writer.flush(); 115 | numberOfEvents = reader.readInput(); 116 | assertEquals(1, numberOfEvents); 117 | assertArrayEquals("line1".getBytes(), adapter.getLastEvents().get(0).getValue("line")); 118 | 119 | writer.close(); 120 | in.close(); 121 | 122 | numberOfEvents = reader.readInput(); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/test/java/info/fetter/logstashforwarder/MockProtocolAdapter.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import org.apache.log4j.Logger; 24 | 25 | public class MockProtocolAdapter implements ProtocolAdapter { 26 | private static Logger logger = Logger.getLogger(MockProtocolAdapter.class); 27 | private List lastEvents; 28 | 29 | public int sendEvents(List eventList) { 30 | for(Event event : eventList) { 31 | logger.trace("Event :"); 32 | for(String key : event.getKeyValues().keySet()) { 33 | logger.trace("-- " + key + ":" + new String(event.getKeyValues().get(key))); 34 | } 35 | } 36 | lastEvents = new ArrayList(eventList); 37 | return eventList.size(); 38 | } 39 | 40 | public List getLastEvents() { 41 | return lastEvents; 42 | } 43 | 44 | public void close() { 45 | // not implemented 46 | } 47 | 48 | public String getServer() { 49 | // TODO Auto-generated method stub 50 | return ""; 51 | } 52 | 53 | public int getPort() { 54 | // TODO Auto-generated method stub 55 | return 0; 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/test/java/info/fetter/logstashforwarder/RegistrarTest.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import static org.apache.log4j.Level.DEBUG; 21 | 22 | import java.io.File; 23 | import java.io.IOException; 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | 27 | import org.apache.log4j.BasicConfigurator; 28 | import org.apache.log4j.Logger; 29 | import org.apache.log4j.spi.RootLogger; 30 | import org.junit.AfterClass; 31 | import org.junit.BeforeClass; 32 | import org.junit.Test; 33 | 34 | import com.fasterxml.jackson.core.JsonGenerationException; 35 | import com.fasterxml.jackson.core.JsonParseException; 36 | import com.fasterxml.jackson.databind.JsonMappingException; 37 | 38 | public class RegistrarTest { 39 | Logger logger = Logger.getLogger(RegistrarTest.class); 40 | 41 | @BeforeClass 42 | public static void setUpBeforeClass() throws Exception { 43 | BasicConfigurator.configure(); 44 | RootLogger.getRootLogger().setLevel(DEBUG); 45 | } 46 | 47 | @AfterClass 48 | public static void tearDownAfterClass() throws Exception { 49 | BasicConfigurator.resetConfiguration(); 50 | } 51 | 52 | @Test 53 | public void testReadState1() throws JsonParseException, JsonMappingException, IOException { 54 | FileState[] states = Registrar.readStateFromJson(new File(RegistrarTest.class.getClassLoader().getResource("state1.json").getFile())); 55 | for(FileState state : states) { 56 | logger.debug("Loaded state : " + state); 57 | } 58 | } 59 | 60 | @Test 61 | public void testWriteState2() throws JsonGenerationException, JsonMappingException, IOException { 62 | FileState state1 = new FileState(); 63 | state1.setDirectory("/directory1"); 64 | state1.setFileName("file1"); 65 | state1.setPointer(1234); 66 | state1.setSignature(123456); 67 | state1.setSignatureLength(135); 68 | FileState state2 = new FileState(); 69 | state2.setDirectory("/directory2"); 70 | state2.setFileName("file2"); 71 | state2.setPointer(4321); 72 | state2.setSignature(654321); 73 | state2.setSignatureLength(531); 74 | List stateList = new ArrayList(2); 75 | stateList.add(state1); 76 | stateList.add(state2); 77 | File file = new File("state2.json"); 78 | logger.debug("Writing to file : " + file.getCanonicalPath()); 79 | Registrar.writeStateToJson(file, stateList); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/test/java/info/fetter/logstashforwarder/config/ConfigurationManagerTest.java: -------------------------------------------------------------------------------- 1 | package info.fetter.logstashforwarder.config; 2 | 3 | /* 4 | * Copyright 2015 Didier Fetter 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | * 18 | */ 19 | 20 | import static org.apache.log4j.Level.DEBUG; 21 | import static org.junit.Assert.*; 22 | 23 | import java.io.File; 24 | import java.io.IOException; 25 | 26 | import org.apache.log4j.BasicConfigurator; 27 | import org.apache.log4j.Logger; 28 | import org.apache.log4j.spi.RootLogger; 29 | import org.junit.AfterClass; 30 | import org.junit.BeforeClass; 31 | import org.junit.Test; 32 | 33 | import com.fasterxml.jackson.core.JsonParseException; 34 | import com.fasterxml.jackson.databind.JsonMappingException; 35 | 36 | public class ConfigurationManagerTest { 37 | Logger logger = Logger.getLogger(ConfigurationManagerTest.class); 38 | 39 | @BeforeClass 40 | public static void setUpBeforeClass() throws Exception { 41 | BasicConfigurator.configure(); 42 | RootLogger.getRootLogger().setLevel(DEBUG); 43 | } 44 | 45 | @AfterClass 46 | public static void tearDownAfterClass() throws Exception { 47 | BasicConfigurator.resetConfiguration(); 48 | } 49 | 50 | @Test 51 | public void testReadConfig1() throws JsonParseException, JsonMappingException, IOException { 52 | ConfigurationManager manager = new ConfigurationManager(new File(ConfigurationManagerTest.class.getClassLoader().getResource("config1.json").getFile())); 53 | manager.readConfiguration(); 54 | logger.debug(manager.getConfig().toString()); 55 | for(FilesSection files : manager.getConfig().getFiles()) { 56 | logger.debug("File Section"); 57 | for(String path : files.getPaths()) { 58 | logger.debug(" - Path : " + path); 59 | } 60 | logger.debug(" - Multiline : " + files.getMultiline()); 61 | logger.debug(" - Dead time : " + files.getDeadTimeInSeconds()); 62 | if(files.getDeadTime().equals("24h")) { 63 | assertEquals(86400, files.getDeadTimeInSeconds()); 64 | } else if(files.getDeadTime().equals("12h")) { 65 | assertEquals(43200, files.getDeadTimeInSeconds()); 66 | } else if(files.getDeadTime().equals("8h32m50s")) { 67 | assertEquals(30770, files.getDeadTimeInSeconds()); 68 | } 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/test/resources/config1.json: -------------------------------------------------------------------------------- 1 | { 2 | // The network section covers network configuration :) 3 | "network": { 4 | // A list of downstream servers listening for our messages. 5 | // logstash-forwarder will pick one at random and only switch if 6 | // the selected one appears to be dead or unresponsive 7 | "servers": [ "localhost:5043" ], 8 | 9 | // The path to your client ssl certificate (optional) 10 | "ssl certificate": "./logstash-forwarder.crt", 11 | // The path to your client ssl key (optional) 12 | "ssl key": "./logstash-forwarder.key", 13 | 14 | // The path to your trusted ssl CA file. This is used 15 | // to authenticate your downstream server. 16 | "ssl ca": "./logstash-forwarder.crt", 17 | 18 | // Network timeout in seconds. This is most important for 19 | // logstash-forwarder determining whether to stop waiting for an 20 | // acknowledgement from the downstream server. If an timeout is reached, 21 | // logstash-forwarder will assume the connection or server is bad and 22 | // will connect to a server chosen at random from the servers list. 23 | "timeout": 15 24 | }, 25 | 26 | // The list of files configurations 27 | "files": [ 28 | // An array of hashes. Each hash tells what paths to watch and 29 | // what fields to annotate on events from those paths. 30 | { 31 | "paths": [ 32 | // single paths are fine 33 | "/var/log/messages", 34 | // globs are fine too, they will be periodically evaluated 35 | // to see if any new files match the wildcard. 36 | "/var/log/*.log" 37 | ], 38 | 39 | // A dictionary of fields to annotate on each event. 40 | "fields": { "type": "syslog" } 41 | }, { 42 | // A path of "-" means stdin. 43 | "paths": [ "-" ], 44 | "fields": { "type": "stdin" } 45 | }, { 46 | "paths": [ 47 | "/var/log/apache/httpd-*.log" 48 | ], 49 | "fields": { "type": "apache" }, 50 | "dead time": "12h" 51 | }, { 52 | "paths": [ 53 | "/var/log/apache/error-*.log" 54 | ], 55 | "fields": { "type": "error" }, 56 | "multiline": { "pattern": "^[0-9]{4}", "negate": "true" }, 57 | "dead time": "8h32m50s" 58 | } 59 | ] 60 | } -------------------------------------------------------------------------------- /src/test/resources/state1.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"fileName":"file1","directory":"/directory1","pointer":12345,"signature":654321,"signatureLength":135}, 3 | {"fileName":"file2","directory":"/directory2","pointer":23456,"signature":7654,"signatureLength":246} 4 | ] --------------------------------------------------------------------------------
By Russ Rew, based on 55 | * BufferedRandomAccessFile by Alex McManus, based on Sun's source code 56 | * for java.io.RandomAccessFile. For Alex McManus version from which 57 | * this derives, see his 58 | * Freeware Java Classes. 59 | *
bufferStart + dataSize
len
nbytes
b.length( )
b.length
n
off
boolean
0
false
true
b
0 <= b <= 255
923 | * (byte)(b) 924 | *
byte
b1
b2
255
969 | * (short)((b1 << 8) | b2) 970 | *
0 <= b1, b2 <= 255
1015 | * (b1 << 8) | b2 1016 | *
b3
0 <= b1, b2, b3 <= 255
1048 | * (b1 << 16) | (b2 << 8) + (b3 << 0) 1049 | *
1062 | * (char)((b1 << 8) | b2) 1063 | *
b4
0 <= b1, b2, b3, b4 <= 255
1093 | * (b1 << 24) | (b2 << 16) + (b3 << 8) + b4 1094 | *
int
b5
b6
b7
b8,
1168 | * 0 <= b1, b2, b3, b4, b5, b6, b7, b8 <=255, 1169 | *
1173 | * ((long)b1 << 56) + ((long)b2 << 48) 1174 | * + ((long)b3 << 40) + ((long)b4 << 32) 1175 | * + ((long)b5 << 24) + ((long)b6 << 16) 1176 | * + ((long)b7 << 8) + b8 1177 | *
long
float
readInt
intBitsToFloat
Float
double
readLong
longBitsToDouble
Double
'\r'
'\n'
readUnsignedShort
(byte)1
(byte)0
short
char
floatToIntBits
doubleToLongBits
writeChar
String
writeShort