├── .gitignore ├── lib └── sigar-1.6.4 │ ├── sigar.jar │ ├── sigar-x86-winnt.dll │ ├── sigar-x86-winnt.lib │ ├── libsigar-ppc-aix-5.so │ ├── libsigar-ppc-linux.so │ ├── libsigar-x86-linux.so │ ├── sigar-amd64-winnt.dll │ ├── libsigar-amd64-linux.so │ ├── libsigar-ia64-hpux-11.sl │ ├── libsigar-ia64-linux.so │ ├── libsigar-pa-hpux-11.sl │ ├── libsigar-ppc64-aix-5.so │ ├── libsigar-ppc64-linux.so │ ├── libsigar-s390x-linux.so │ ├── libsigar-x86-solaris.so │ ├── libsigar-amd64-solaris.so │ ├── libsigar-sparc-solaris.so │ ├── libsigar-x86-freebsd-5.so │ ├── libsigar-x86-freebsd-6.so │ ├── libsigar-amd64-freebsd-6.so │ ├── libsigar-sparc64-solaris.so │ ├── libsigar-universal-macosx.dylib │ └── libsigar-universal64-macosx.dylib ├── src ├── main │ ├── resources │ │ ├── collector-version.properties │ │ ├── log4j2.xml │ │ └── ValidationMessages.properties │ ├── scripts │ │ ├── graylog-collector-script-config.sh │ │ ├── graylog-collector.bat │ │ └── graylog-collector.sh │ ├── java │ │ └── org │ │ │ └── graylog │ │ │ └── collector │ │ │ ├── cli │ │ │ ├── commands │ │ │ │ ├── CollectorCommand.java │ │ │ │ ├── CollectorHelp.java │ │ │ │ └── Version.java │ │ │ └── Main.java │ │ │ ├── buffer │ │ │ ├── BufferConsumer.java │ │ │ ├── Buffer.java │ │ │ ├── BufferModule.java │ │ │ ├── MessageBufferConfiguration.java │ │ │ ├── MessageBuffer.java │ │ │ └── BufferProcessor.java │ │ │ ├── config │ │ │ ├── Configuration.java │ │ │ ├── ConfigurationError.java │ │ │ ├── ConfigurationModule.java │ │ │ ├── constraints │ │ │ │ ├── IsOneOf.java │ │ │ │ └── IsOneOfValidator.java │ │ │ ├── ConfigurationParser.java │ │ │ ├── ConfigurationUtils.java │ │ │ └── ConfigurationValidator.java │ │ │ ├── file │ │ │ ├── naming │ │ │ │ ├── FileNamingStrategy.java │ │ │ │ ├── ExactFileStrategy.java │ │ │ │ └── NumberSuffixStrategy.java │ │ │ ├── splitters │ │ │ │ ├── ContentSplitter.java │ │ │ │ └── NewlineChunkSplitter.java │ │ │ ├── FileModule.java │ │ │ ├── PathSet.java │ │ │ ├── FileChunk.java │ │ │ ├── ChunkBufferStore.java │ │ │ ├── SinglePathSet.java │ │ │ └── ChunkProcessor.java │ │ │ ├── inputs │ │ │ ├── InputService.java │ │ │ ├── Input.java │ │ │ ├── file │ │ │ │ ├── ValidFileInputConfiguration.java │ │ │ │ ├── FileInput.java │ │ │ │ └── FileInputConfigurationValidator.java │ │ │ ├── InputsModule.java │ │ │ ├── eventlog │ │ │ │ ├── WindowsEventlogInputConfiguration.java │ │ │ │ ├── WindowsEventlogInput.java │ │ │ │ └── WindowsEventlogHandler.java │ │ │ └── InputConfiguration.java │ │ │ ├── outputs │ │ │ ├── OutputService.java │ │ │ ├── Output.java │ │ │ ├── OutputsModule.java │ │ │ ├── stdout │ │ │ │ ├── StdoutOutputConfiguration.java │ │ │ │ └── StdoutOutput.java │ │ │ ├── OutputRouter.java │ │ │ └── OutputConfiguration.java │ │ │ ├── utils │ │ │ ├── Utils.java │ │ │ ├── CollectorIdModule.java │ │ │ ├── MemoryReporterModule.java │ │ │ ├── CollectorHostNameModule.java │ │ │ ├── CollectorHostNameConfiguration.java │ │ │ ├── CollectorIdConfiguration.java │ │ │ ├── CollectorHostNameProvider.java │ │ │ ├── MemoryReporterServiceConfiguration.java │ │ │ ├── CollectorHostNameSupplier.java │ │ │ └── CollectorId.java │ │ │ ├── services │ │ │ ├── ServicesModule.java │ │ │ ├── ServiceManagerProvider.java │ │ │ └── CollectorServiceManager.java │ │ │ ├── annotations │ │ │ ├── CollectorHostName.java │ │ │ └── GraylogServerURL.java │ │ │ ├── metrics │ │ │ ├── MetricsModule.java │ │ │ ├── MetricServiceConfiguration.java │ │ │ └── MetricService.java │ │ │ ├── heartbeat │ │ │ ├── HeartbeatModule.java │ │ │ ├── CollectorRegistrationService.java │ │ │ ├── CollectorRegistrationServiceProvider.java │ │ │ ├── CollectorNodeDetailsSummary.java │ │ │ ├── CollectorRegistrationRequestProvider.java │ │ │ ├── CollectorRegistrationRequest.java │ │ │ └── HeartbeatService.java │ │ │ ├── serverapi │ │ │ ├── ServerApiModule.java │ │ │ ├── ServerURLProvider.java │ │ │ └── RestAdapterProvider.java │ │ │ ├── guice │ │ │ ├── CollectorInjector.java │ │ │ └── CollectorModule.java │ │ │ ├── MessageFields.java │ │ │ ├── CollectorVersion.java │ │ │ └── Message.java │ └── assembly │ │ └── collector.xml └── test │ └── java │ └── org │ └── graylog │ └── collector │ ├── utils │ ├── CollectorHostNameProviderTest.java │ ├── CollectorHostNameConfigurationTest.java │ └── CollectorHostNameSupplierTest.java │ ├── file │ ├── CollectingBuffer.java │ ├── ChunkBufferStoreTest.java │ ├── MultithreadedBaseTest.java │ ├── naming │ │ └── FileNamingStrategyTest.java │ ├── SinglePathSetTest.java │ └── Utils.java │ ├── config │ ├── constraints │ │ └── IsOneOfValidatorTest.java │ └── ConfigurationParserTest.java │ └── MessageFieldsTest.java ├── bin └── windows │ ├── graylog-collector-service-x64.exe │ ├── graylog-collector-service-x86.exe │ └── graylog-collector-service-manager.exe ├── .travis.yml ├── development └── forbidden-apis │ └── signatures.txt ├── CONTRIBUTING.md ├── CHANGELOG.md └── config └── collector.conf.example /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea/ 2 | /config/*.conf 3 | /config/collector-id 4 | /logs/ 5 | /target/ 6 | *.iml 7 | -------------------------------------------------------------------------------- /lib/sigar-1.6.4/sigar.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/sigar.jar -------------------------------------------------------------------------------- /lib/sigar-1.6.4/sigar-x86-winnt.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/sigar-x86-winnt.dll -------------------------------------------------------------------------------- /lib/sigar-1.6.4/sigar-x86-winnt.lib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/sigar-x86-winnt.lib -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-ppc-aix-5.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-ppc-aix-5.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-ppc-linux.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-ppc-linux.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-x86-linux.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-x86-linux.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/sigar-amd64-winnt.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/sigar-amd64-winnt.dll -------------------------------------------------------------------------------- /src/main/resources/collector-version.properties: -------------------------------------------------------------------------------- 1 | version=${project.version} 2 | commit-id=${buildNumber} 3 | timestamp=${timestamp} -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-amd64-linux.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-amd64-linux.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-ia64-hpux-11.sl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-ia64-hpux-11.sl -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-ia64-linux.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-ia64-linux.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-pa-hpux-11.sl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-pa-hpux-11.sl -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-ppc64-aix-5.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-ppc64-aix-5.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-ppc64-linux.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-ppc64-linux.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-s390x-linux.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-s390x-linux.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-x86-solaris.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-x86-solaris.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-amd64-solaris.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-amd64-solaris.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-sparc-solaris.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-sparc-solaris.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-x86-freebsd-5.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-x86-freebsd-5.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-x86-freebsd-6.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-x86-freebsd-6.so -------------------------------------------------------------------------------- /bin/windows/graylog-collector-service-x64.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/bin/windows/graylog-collector-service-x64.exe -------------------------------------------------------------------------------- /bin/windows/graylog-collector-service-x86.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/bin/windows/graylog-collector-service-x86.exe -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-amd64-freebsd-6.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-amd64-freebsd-6.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-sparc64-solaris.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-sparc64-solaris.so -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-universal-macosx.dylib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-universal-macosx.dylib -------------------------------------------------------------------------------- /bin/windows/graylog-collector-service-manager.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/bin/windows/graylog-collector-service-manager.exe -------------------------------------------------------------------------------- /lib/sigar-1.6.4/libsigar-universal64-macosx.dylib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/graylog-labs/collector/HEAD/lib/sigar-1.6.4/libsigar-universal64-macosx.dylib -------------------------------------------------------------------------------- /src/main/scripts/graylog-collector-script-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | COLLECTOR_JAVA_DEFAULT_OPTS="${collector.jvm-opts} -Djava.library.path=${collector.share-path}/lib/sigar" 4 | COLLECTOR_DEFAULT_JAR="${collector.jar-path}" 5 | 6 | # For Debian/Ubuntu based systems. 7 | if [ -f "/etc/default/graylog-collector" ]; then 8 | . "/etc/default/graylog-collector" 9 | fi 10 | 11 | # For RedHat/Fedora based systems. 12 | if [ -f "/etc/sysconfig/graylog-collector" ]; then 13 | . "/etc/sysconfig/graylog-collector" 14 | fi 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: java 3 | jdk: 4 | - oraclejdk7 5 | - oraclejdk8 6 | after_success: 7 | - mvn -DskipTests -B assembly:single 8 | deploy: 9 | provider: s3 10 | access_key_id: AKIAJBL3SCAJTENB5ZQA 11 | secret_access_key: 12 | secure: EY9WZbf/e1I7Ft3qfhBXMYpzn1CGOzCp1hjMQK8NsO8nJEYtT5V/RRsqsgBNr0bQF3FHxcidqRi4CoY0cS5c71pLff9lYa9Dkyi5oJ0W8/rzGMu69WNhNmUWrpwxs0IL91/st6L1/okoSSb0WXCPLRFG5t8/xKE1YY6XKC4DM9I= 13 | bucket: graylog2-builds 14 | region: eu-west-1 15 | skip_cleanup: true 16 | local-dir: target/assembly 17 | on: 18 | repo: Graylog2/collector 19 | jdk: oraclejdk7 20 | branch: 21 | - master 22 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/main/scripts/graylog-collector.bat: -------------------------------------------------------------------------------- 1 | :: Graylog Collector startup script for Windows 2 | 3 | @ECHO OFF 4 | 5 | SETLOCAL ENABLEEXTENSIONS ENABLEDELAYEDEXPANSION 6 | TITLE Graylog Collector ${project.version} 7 | 8 | IF DEFINED JAVA_HOME goto :continue 9 | 10 | :: JAVA_HOME needs to be set to find the JVM. 11 | :jvmError 12 | ECHO JAVA_HOME not set! 1>&2 13 | EXIT /B 1 14 | 15 | :: Execute the JAR. 16 | :continue 17 | set COLLECTOR_BIN_DIR=%~dp0 18 | 19 | :: Get root directory of the Collector. 20 | FOR %%D in ("%COLLECTOR_BIN_DIR%..") DO SET COLLECTOR_ROOT=%%~dpfD 21 | 22 | SET COLLECTOR_JAR=%COLLECTOR_ROOT%\${project.artifactId}.jar 23 | SET COLLECTOR_JVM_OPTIONS=-Djava.library.path="%COLLECTOR_ROOT%\lib\sigar" -Dfile.encoding=UTF-8 ${collector.jvm-opts} 24 | 25 | "%JAVA_HOME%\bin\java" %COLLECTOR_JVM_OPTIONS% -jar "%COLLECTOR_JAR%" %* 26 | 27 | ENDLOCAL 28 | -------------------------------------------------------------------------------- /development/forbidden-apis/signatures.txt: -------------------------------------------------------------------------------- 1 | @defaultMessage Use a custom thread factory to ensure proper thread naming. 2 | java.util.concurrent.Executors#defaultThreadFactory() 3 | java.util.concurrent.Executors#newCachedThreadPool() 4 | java.util.concurrent.Executors#newFixedThreadPool(int) 5 | java.util.concurrent.Executors#newScheduledThreadPool(int) 6 | java.util.concurrent.Executors#newSingleThreadExecutor() 7 | java.util.concurrent.Executors#newSingleThreadScheduledExecutor() 8 | java.util.concurrent.Executors#privilegedThreadFactory() 9 | 10 | @defaultMessage Do not create a DateTime without an explicit time zone. 11 | org.joda.time.DateTime#() 12 | org.joda.time.DateTime#(long) 13 | org.joda.time.DateTime#(int, int, int, int, int) 14 | org.joda.time.DateTime#(int, int, int, int, int, int) 15 | org.joda.time.DateTime#(int, int, int, int, int, int, int) 16 | org.joda.time.DateTime#now() 17 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/cli/commands/CollectorCommand.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.cli.commands; 18 | 19 | public interface CollectorCommand extends Runnable { 20 | void stop(); 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/buffer/BufferConsumer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.buffer; 18 | 19 | import org.graylog.collector.Message; 20 | 21 | public interface BufferConsumer { 22 | void process(Message message); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/buffer/Buffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.buffer; 18 | 19 | import org.graylog.collector.Message; 20 | 21 | public interface Buffer { 22 | void insert(Message message); 23 | 24 | Message remove(); 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/config/Configuration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.config; 18 | 19 | import java.util.Map; 20 | 21 | public interface Configuration { 22 | String getId(); 23 | 24 | Map toStringValues(); 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/file/naming/FileNamingStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file.naming; 18 | 19 | import java.nio.file.Path; 20 | 21 | public interface FileNamingStrategy { 22 | 23 | boolean pathMatches(Path path); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/inputs/InputService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.inputs; 18 | 19 | import com.google.common.util.concurrent.AbstractService; 20 | 21 | public abstract class InputService extends AbstractService implements Input { 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/outputs/OutputService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.outputs; 18 | 19 | import com.google.common.util.concurrent.AbstractService; 20 | 21 | public abstract class OutputService extends AbstractService implements Output { 22 | } 23 | -------------------------------------------------------------------------------- /src/main/resources/ValidationMessages.properties: -------------------------------------------------------------------------------- 1 | org.graylog.collector.config.constraints.IsOneOf.message="${validatedValue}" is not one of: ${validValues} 2 | org.graylog.collector.inputs.file.ValidFileInputConfiguration.message=The file input configuration is invalid 3 | org.graylog.collector.inputs.file.ValidFileInputConfiguration.invalidPattern.message=Invalid content splitter pattern: "${value}" 4 | org.graylog.collector.inputs.file.ValidFileInputConfiguration.missingPattern.message=Missing content splitter pattern. 5 | org.graylog.collector.inputs.file.ValidFileInputConfiguration.unsupportedCharset.message=Character set not supported: "${value}" 6 | org.graylog.collector.inputs.file.ValidFileInputConfiguration.illegalCharset.message=Invalid character set value: "${value}" 7 | org.graylog.collector.inputs.file.ValidFileInputConfiguration.readerBufferSizeTooSmall.message=Reader buffer size too small: "${value}" 8 | org.graylog.collector.inputs.file.ValidFileInputConfiguration.readerIntervalTooSmall.message=Reader interval too small: "${value}" 9 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/cli/commands/CollectorHelp.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.cli.commands; 18 | 19 | import io.airlift.airline.Help; 20 | 21 | public class CollectorHelp extends Help implements CollectorCommand { 22 | @Override 23 | public void stop() { 24 | // nothing to stop 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/utils/Utils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import java.util.Locale; 20 | 21 | public class Utils { 22 | private static final boolean IS_WINDOWS = System.getProperty("os.name").toLowerCase(Locale.US).startsWith("windows"); 23 | 24 | public static boolean isWindows() { 25 | return IS_WINDOWS; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/config/ConfigurationError.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.config; 18 | 19 | public class ConfigurationError { 20 | private final String messsage; 21 | 22 | public ConfigurationError(String messsage) { 23 | this.messsage = messsage; 24 | } 25 | 26 | public String getMesssage() { 27 | return messsage; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/utils/CollectorIdModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import org.graylog.collector.guice.CollectorModule; 20 | 21 | public class CollectorIdModule extends CollectorModule { 22 | @Override 23 | protected void configure() { 24 | bind(CollectorIdConfiguration.class); 25 | bind(CollectorId.class); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/services/ServicesModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.services; 18 | 19 | import com.google.common.util.concurrent.ServiceManager; 20 | import org.graylog.collector.guice.CollectorModule; 21 | 22 | public class ServicesModule extends CollectorModule { 23 | @Override 24 | protected void configure() { 25 | bind(CollectorServiceManager.class); 26 | bind(ServiceManager.class).toProvider(ServiceManagerProvider.class); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/annotations/CollectorHostName.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.annotations; 18 | 19 | import com.google.inject.BindingAnnotation; 20 | 21 | import java.lang.annotation.ElementType; 22 | import java.lang.annotation.Retention; 23 | import java.lang.annotation.RetentionPolicy; 24 | import java.lang.annotation.Target; 25 | 26 | @Target(ElementType.PARAMETER) 27 | @Retention(RetentionPolicy.RUNTIME) 28 | @BindingAnnotation 29 | public @interface CollectorHostName { 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/annotations/GraylogServerURL.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.annotations; 18 | 19 | import com.google.inject.BindingAnnotation; 20 | 21 | import java.lang.annotation.ElementType; 22 | import java.lang.annotation.Retention; 23 | import java.lang.annotation.RetentionPolicy; 24 | import java.lang.annotation.Target; 25 | 26 | @Target(ElementType.PARAMETER) 27 | @Retention(RetentionPolicy.RUNTIME) 28 | @BindingAnnotation 29 | public @interface GraylogServerURL { 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/outputs/Output.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.outputs; 18 | 19 | import org.graylog.collector.Message; 20 | import org.graylog.collector.config.Configuration; 21 | 22 | import java.util.Set; 23 | 24 | public interface Output { 25 | String getId(); 26 | 27 | Set getInputs(); 28 | 29 | void write(Message message); 30 | 31 | interface Factory { 32 | T create(C configuration); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/buffer/BufferModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.buffer; 18 | 19 | import com.google.inject.Scopes; 20 | import org.graylog.collector.guice.CollectorModule; 21 | 22 | public class BufferModule extends CollectorModule { 23 | @Override 24 | protected void configure() { 25 | bind(MessageBufferConfiguration.class); 26 | bind(Buffer.class).to(MessageBuffer.class).in(Scopes.SINGLETON); 27 | 28 | registerService(BufferProcessor.class); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/utils/MemoryReporterModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import com.google.inject.Scopes; 20 | import org.graylog.collector.guice.CollectorModule; 21 | 22 | public class MemoryReporterModule extends CollectorModule { 23 | @Override 24 | protected void configure() { 25 | bind(MemoryReporterService.class).in(Scopes.SINGLETON); 26 | bind(MemoryReporterServiceConfiguration.class); 27 | 28 | registerService(MemoryReporterService.class); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/utils/CollectorHostNameModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import org.graylog.collector.annotations.CollectorHostName; 20 | import org.graylog.collector.guice.CollectorModule; 21 | 22 | public class CollectorHostNameModule extends CollectorModule { 23 | @Override 24 | protected void configure() { 25 | bind(CollectorHostNameConfiguration.class); 26 | bind(String.class).annotatedWith(CollectorHostName.class).toProvider(CollectorHostNameProvider.class); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/metrics/MetricsModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.metrics; 18 | 19 | import com.codahale.metrics.MetricRegistry; 20 | import com.google.inject.Scopes; 21 | import org.graylog.collector.guice.CollectorModule; 22 | 23 | public class MetricsModule extends CollectorModule { 24 | @Override 25 | protected void configure() { 26 | bind(MetricRegistry.class).in(Scopes.SINGLETON); 27 | bind(MetricServiceConfiguration.class); 28 | 29 | registerService(MetricService.class); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/heartbeat/HeartbeatModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.heartbeat; 18 | 19 | import org.graylog.collector.guice.CollectorModule; 20 | 21 | public class HeartbeatModule extends CollectorModule { 22 | @Override 23 | protected void configure() { 24 | bind(CollectorRegistrationService.class).toProvider(CollectorRegistrationServiceProvider.class); 25 | bind(CollectorRegistrationRequest.class).toProvider(CollectorRegistrationRequestProvider.class); 26 | registerService(HeartbeatService.class); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/inputs/Input.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.inputs; 18 | 19 | import org.graylog.collector.config.Configuration; 20 | import org.graylog.collector.file.ChunkReader; 21 | 22 | import java.util.Set; 23 | 24 | public interface Input { 25 | String getId(); 26 | 27 | Set getOutputs(); 28 | 29 | // TODO Check if needed and for what it was used. 30 | void setReaderFinished(ChunkReader chunkReader); 31 | 32 | interface Factory { 33 | T create(C configuration); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/serverapi/ServerApiModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.serverapi; 18 | 19 | import org.graylog.collector.annotations.GraylogServerURL; 20 | import org.graylog.collector.guice.CollectorModule; 21 | import retrofit.RestAdapter; 22 | 23 | public class ServerApiModule extends CollectorModule { 24 | @Override 25 | protected void configure() { 26 | bind(String.class).annotatedWith(GraylogServerURL.class).toProvider(ServerURLProvider.class); 27 | bind(RestAdapter.class).toProvider(RestAdapterProvider.class).asEagerSingleton(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/scripts/graylog-collector.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # resolve links - $0 may be a softlink 4 | COLLECTOR_BIN="$0" 5 | 6 | while [ -h "$COLLECTOR_BIN" ]; do 7 | ls=$(ls -ld "$COLLECTOR_BIN") 8 | link=$(expr "$ls" : '.*-> \(.*\)$') 9 | if expr "$link" : '/.*' > /dev/null; then 10 | COLLECTOR_BIN="$link" 11 | else 12 | COLLECTOR_BIN=$(dirname "$COLLECTOR_BIN")/"$link" 13 | fi 14 | done 15 | 16 | COLLECTOR_ROOT="$(dirname $(dirname $COLLECTOR_BIN))" 17 | COLLECTOR_DEFAULT_JAR="$COLLECTOR_ROOT/graylog-collector.jar" 18 | 19 | COLLECTOR_JAVA_DEFAULT_OPTS="${collector.jvm-opts} -Djava.library.path=$COLLECTOR_ROOT/lib/sigar" 20 | 21 | if [ -f "${collector.script-config}" ]; then 22 | . "${collector.script-config}" 23 | fi 24 | 25 | COLLECTOR_JAR=${COLLECTOR_JAR:="$COLLECTOR_DEFAULT_JAR"} 26 | 27 | COLLECTOR_JAVA_CMD=${COLLECTOR_JAVA_CMD:=$(which java)} 28 | COLLECTOR_JAVA_OPTS="${COLLECTOR_JAVA_OPTS:="$COLLECTOR_JAVA_DEFAULT_OPTS"}" 29 | 30 | die() { 31 | echo $* 32 | exit 1 33 | } 34 | 35 | if [ -n "$JAVA_HOME" ]; then 36 | # Try to use $JAVA_HOME 37 | if [ -x "$JAVA_HOME"/bin/java ]; then 38 | COLLECTOR_JAVA_CMD="$JAVA_HOME"/bin/java 39 | else 40 | die "$JAVA_HOME"/bin/java is not executable 41 | fi 42 | fi 43 | 44 | exec $COLLECTOR_JAVA_CMD $COLLECTOR_JAVA_OPTS -jar $COLLECTOR_JAR "$@" 45 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/file/splitters/ContentSplitter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file.splitters; 18 | 19 | import io.netty.buffer.ByteBuf; 20 | 21 | import java.nio.charset.Charset; 22 | 23 | public abstract class ContentSplitter { 24 | 25 | public abstract Iterable split(ByteBuf buffer, Charset charset, boolean includeRemainingData); 26 | 27 | public Iterable split(ByteBuf buffer, Charset charset) { 28 | return split(buffer, charset, false); 29 | } 30 | 31 | public Iterable splitRemaining(ByteBuf buffer, Charset charset) { 32 | return split(buffer, charset, true); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/heartbeat/CollectorRegistrationService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.heartbeat; 18 | 19 | import retrofit.client.Response; 20 | import retrofit.http.Body; 21 | import retrofit.http.PUT; 22 | import retrofit.http.Path; 23 | 24 | public interface CollectorRegistrationService { 25 | @PUT("/plugins/org.graylog.plugins.collector/collectors/{collectorId}") 26 | Response register(@Path("collectorId") String collectorId, @Body CollectorRegistrationRequest request); 27 | 28 | @PUT("/system/collectors/{collectorId}") 29 | Response legacyRegister(@Path("collectorId") String collectorId, @Body CollectorRegistrationRequest request); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/buffer/MessageBufferConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.buffer; 18 | 19 | import com.typesafe.config.Config; 20 | 21 | import javax.inject.Inject; 22 | 23 | public class MessageBufferConfiguration { 24 | private static int SIZE = 128; 25 | 26 | private final int size; 27 | 28 | @Inject 29 | public MessageBufferConfiguration(Config config) { 30 | if (config.hasPath("message-buffer-size")) { 31 | this.size = config.getInt("message-buffer-size"); 32 | } else { 33 | this.size = SIZE; 34 | } 35 | } 36 | 37 | public int getSize() { 38 | return size; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/utils/CollectorHostNameConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import com.typesafe.config.Config; 20 | 21 | import javax.annotation.Nullable; 22 | import javax.inject.Inject; 23 | 24 | public class CollectorHostNameConfiguration { 25 | private static final String CONFIG_PATH = "host-name"; 26 | private final String hostName; 27 | 28 | @Inject 29 | public CollectorHostNameConfiguration(Config config) { 30 | this.hostName = config.hasPath(CONFIG_PATH) ? config.getString(CONFIG_PATH) : null; 31 | } 32 | 33 | @Nullable 34 | public String getHostName() { 35 | return hostName; 36 | } 37 | } 38 | 39 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/guice/CollectorInjector.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.guice; 18 | 19 | import com.google.inject.Guice; 20 | import com.google.inject.Injector; 21 | import com.google.inject.Module; 22 | import com.google.inject.Stage; 23 | 24 | public class CollectorInjector { 25 | public static Injector createInjector(Module... modules) { 26 | final Injector injector = Guice.createInjector(Stage.PRODUCTION, new CollectorModule() { 27 | @Override 28 | protected void configure() { 29 | binder().requireExplicitBindings(); 30 | } 31 | }); 32 | 33 | return injector.createChildInjector(modules); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/config/ConfigurationModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.config; 18 | 19 | import com.google.inject.Scopes; 20 | import com.typesafe.config.Config; 21 | import org.graylog.collector.guice.CollectorModule; 22 | 23 | import java.io.File; 24 | 25 | public class ConfigurationModule extends CollectorModule { 26 | private final Config config; 27 | 28 | public ConfigurationModule(File configFile) { 29 | this.config = ConfigurationParser.parse(configFile); 30 | } 31 | 32 | @Override 33 | protected void configure() { 34 | bind(Config.class).toInstance(config); 35 | bind(ConfigurationRegistry.class).in(Scopes.SINGLETON); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/heartbeat/CollectorRegistrationServiceProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.heartbeat; 18 | 19 | import retrofit.RestAdapter; 20 | 21 | import javax.inject.Inject; 22 | import javax.inject.Provider; 23 | 24 | public class CollectorRegistrationServiceProvider implements Provider { 25 | private final RestAdapter restAdapter; 26 | 27 | @Inject 28 | public CollectorRegistrationServiceProvider(RestAdapter restAdapter) { 29 | this.restAdapter = restAdapter; 30 | } 31 | 32 | @Override 33 | public CollectorRegistrationService get() { 34 | return this.restAdapter.create(CollectorRegistrationService.class); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/cli/commands/Version.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.cli.commands; 18 | 19 | import io.airlift.airline.Command; 20 | import org.graylog.collector.CollectorVersion; 21 | 22 | @Command(name = "version", description = "Show version information on STDOUT") 23 | public class Version implements CollectorCommand { 24 | @Override 25 | public void run() { 26 | final CollectorVersion v = CollectorVersion.CURRENT; 27 | final String message = String.format("Graylog Collector v%s (commit=%s, timestamp=%s)", 28 | v.version(), v.commitIdShort(), v.timestamp()); 29 | 30 | System.out.println(message); 31 | } 32 | 33 | @Override 34 | public void stop() { 35 | // nothing to stop 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/utils/CollectorIdConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import com.typesafe.config.Config; 20 | 21 | import javax.inject.Inject; 22 | 23 | public class CollectorIdConfiguration { 24 | private static final String collectorIdStatement = "collector-id"; 25 | private final String collectorId; 26 | 27 | @Inject 28 | public CollectorIdConfiguration(Config config) { 29 | if (config.hasPath(collectorIdStatement)) { 30 | this.collectorId = config.getString(collectorIdStatement); 31 | } else { 32 | this.collectorId = "file:config/collector-id"; 33 | } 34 | } 35 | 36 | public String getCollectorId() { 37 | return collectorId; 38 | } 39 | } 40 | 41 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/config/constraints/IsOneOf.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.config.constraints; 18 | 19 | import javax.validation.Constraint; 20 | import javax.validation.Payload; 21 | import java.lang.annotation.ElementType; 22 | import java.lang.annotation.Retention; 23 | import java.lang.annotation.RetentionPolicy; 24 | import java.lang.annotation.Target; 25 | 26 | @Target({ElementType.FIELD}) 27 | @Retention(RetentionPolicy.RUNTIME) 28 | @Constraint(validatedBy = IsOneOfValidator.class) 29 | public @interface IsOneOf { 30 | String message() default "{org.graylog.collector.config.constraints.IsOneOf.message}"; 31 | 32 | String[] value(); 33 | 34 | Class[] groups() default {}; 35 | 36 | Class[] payload() default {}; 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/file/FileModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file; 18 | 19 | import com.google.inject.Scopes; 20 | import org.graylog.collector.guice.CollectorModule; 21 | 22 | import java.io.IOException; 23 | import java.nio.file.FileSystems; 24 | import java.nio.file.WatchService; 25 | 26 | public class FileModule extends CollectorModule { 27 | @Override 28 | protected void configure() { 29 | try { 30 | bind(WatchService.class).toInstance(FileSystems.getDefault().newWatchService()); 31 | } catch (IOException e) { 32 | throw new RuntimeException("Unable to create WatchService"); 33 | } 34 | 35 | bind(FileObserver.class).in(Scopes.SINGLETON); 36 | registerService(FileObserver.class); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/serverapi/ServerURLProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.serverapi; 18 | 19 | import com.typesafe.config.Config; 20 | 21 | import javax.inject.Inject; 22 | import javax.inject.Provider; 23 | 24 | public class ServerURLProvider implements Provider { 25 | private static final String serverURLStatement = "server-url"; 26 | private final Config config; 27 | 28 | @Inject 29 | public ServerURLProvider(Config config) { 30 | this.config = config; 31 | } 32 | 33 | @Override 34 | public String get() { 35 | if (config.hasPath(serverURLStatement)) { 36 | return config.getString(serverURLStatement); 37 | } else { 38 | return "http://localhost:12900"; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/inputs/file/ValidFileInputConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.inputs.file; 18 | 19 | import javax.validation.Constraint; 20 | import javax.validation.Payload; 21 | import java.lang.annotation.ElementType; 22 | import java.lang.annotation.Retention; 23 | import java.lang.annotation.RetentionPolicy; 24 | import java.lang.annotation.Target; 25 | 26 | @Target({ElementType.TYPE, ElementType.ANNOTATION_TYPE}) 27 | @Retention(RetentionPolicy.RUNTIME) 28 | @Constraint(validatedBy = FileInputConfigurationValidator.class) 29 | public @interface ValidFileInputConfiguration { 30 | String message() default "{org.graylog.collector.inputs.file.ValidFileInputConfiguration.message}"; 31 | 32 | Class[] groups() default {}; 33 | 34 | Class[] payload() default {}; 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/file/PathSet.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file; 18 | 19 | import java.io.IOException; 20 | import java.nio.file.Path; 21 | import java.util.Set; 22 | 23 | public interface PathSet { 24 | /** 25 | * Get the root path of the path set. 26 | * @return the root path 27 | */ 28 | Path getRootPath(); 29 | 30 | /** 31 | * Checks if the given path is in the path set. 32 | * 33 | * @param path 34 | * @return true if given path is in the path set, false otherwise 35 | */ 36 | boolean isInSet(Path path); 37 | 38 | /** 39 | * Returns all files of the path set that exist in the file system. 40 | * 41 | * @return existing path set files in the file system 42 | * @throws IOException 43 | */ 44 | Set getPaths() throws IOException; 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/heartbeat/CollectorNodeDetailsSummary.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.heartbeat; 18 | 19 | import com.fasterxml.jackson.annotation.JsonAutoDetect; 20 | import com.fasterxml.jackson.annotation.JsonCreator; 21 | import com.fasterxml.jackson.annotation.JsonProperty; 22 | import com.google.auto.value.AutoValue; 23 | 24 | import javax.validation.constraints.NotNull; 25 | import javax.validation.constraints.Size; 26 | 27 | @AutoValue 28 | @JsonAutoDetect 29 | public abstract class CollectorNodeDetailsSummary { 30 | 31 | @JsonProperty("operating_system") 32 | @NotNull 33 | @Size(min = 1) 34 | public abstract String operatingSystem(); 35 | 36 | @JsonCreator 37 | public static CollectorNodeDetailsSummary create(@JsonProperty("operating_system") String operatingSystem) { 38 | return new AutoValue_CollectorNodeDetailsSummary(operatingSystem); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/heartbeat/CollectorRegistrationRequestProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.heartbeat; 18 | 19 | import javax.inject.Inject; 20 | import javax.inject.Provider; 21 | import java.net.InetAddress; 22 | import java.net.UnknownHostException; 23 | 24 | public class CollectorRegistrationRequestProvider implements Provider { 25 | private final String operatingSystem; 26 | private final String hostname; 27 | 28 | @Inject 29 | public CollectorRegistrationRequestProvider() throws UnknownHostException { 30 | this.operatingSystem = System.getProperty("os.name", "unknown"); 31 | this.hostname = InetAddress.getLocalHost().getHostName(); 32 | } 33 | 34 | @Override 35 | public CollectorRegistrationRequest get() { 36 | return CollectorRegistrationRequest.create(hostname, CollectorNodeDetailsSummary.create(operatingSystem)); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/utils/CollectorHostNameProviderTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import org.junit.Test; 20 | 21 | import static org.junit.Assert.assertEquals; 22 | import static org.mockito.Mockito.mock; 23 | import static org.mockito.Mockito.only; 24 | import static org.mockito.Mockito.verify; 25 | import static org.mockito.Mockito.when; 26 | 27 | public class CollectorHostNameProviderTest { 28 | @Test 29 | public void providerMemoizesHostName() throws Exception { 30 | final CollectorHostNameSupplier supplier = mock(CollectorHostNameSupplier.class); 31 | when(supplier.get()).thenReturn("foobar.example.net"); 32 | 33 | final CollectorHostNameProvider provider = new CollectorHostNameProvider(supplier); 34 | 35 | assertEquals("foobar.example.net", provider.get()); 36 | assertEquals("foobar.example.net", provider.get()); 37 | verify(supplier, only()).get(); 38 | } 39 | } -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/file/FileChunk.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file; 18 | 19 | import io.netty.buffer.ByteBuf; 20 | 21 | import java.nio.file.Path; 22 | 23 | public class FileChunk { 24 | 25 | private final Path path; 26 | private final ByteBuf chunkBuffer; 27 | private final long id; 28 | 29 | public FileChunk(Path path, ByteBuf chunkBuffer, long id) { 30 | this.path = path; 31 | this.chunkBuffer = chunkBuffer; 32 | this.id = id; 33 | } 34 | 35 | public static FileChunk finalChunk(Path path) { 36 | return new FileChunk(path, null, -1); 37 | } 38 | 39 | public Path getPath() { 40 | return path; 41 | } 42 | 43 | public ByteBuf getChunkBuffer() { 44 | return chunkBuffer; 45 | } 46 | 47 | public long getId() { 48 | return id; 49 | } 50 | 51 | public boolean isFinalChunk() { 52 | return chunkBuffer == null; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/config/ConfigurationParser.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.config; 18 | 19 | import com.typesafe.config.Config; 20 | import com.typesafe.config.ConfigFactory; 21 | 22 | import java.io.File; 23 | 24 | public class ConfigurationParser { 25 | public static class Error extends RuntimeException { 26 | public Error(String message) { 27 | super(message); 28 | } 29 | } 30 | 31 | public static Config parse(File configFile) { 32 | Config config = null; 33 | 34 | if (configFile.exists() && configFile.canRead()) { 35 | config = ConfigFactory.parseFile(configFile); 36 | 37 | if (config.isEmpty()) { 38 | throw new Error("Empty configuration!"); 39 | } 40 | } else { 41 | throw new Error("Configuration file " + configFile + " does not exist or is not readable!"); 42 | } 43 | 44 | return config; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/utils/CollectorHostNameProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import com.google.common.annotations.VisibleForTesting; 20 | import com.google.common.base.Supplier; 21 | import com.google.common.base.Suppliers; 22 | 23 | import javax.inject.Inject; 24 | import javax.inject.Provider; 25 | 26 | public class CollectorHostNameProvider implements Provider { 27 | private final Supplier hostName; 28 | 29 | @Inject 30 | public CollectorHostNameProvider(CollectorHostNameConfiguration config, CollectorId collectorId) { 31 | this(new CollectorHostNameSupplier(config.getHostName(), collectorId)); 32 | } 33 | 34 | @VisibleForTesting 35 | protected CollectorHostNameProvider(CollectorHostNameSupplier supplier) { 36 | this.hostName = Suppliers.memoize(supplier); 37 | } 38 | 39 | @Override 40 | public String get() { 41 | return hostName.get(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/outputs/OutputsModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.outputs; 18 | 19 | import org.graylog.collector.guice.CollectorModule; 20 | import org.graylog.collector.outputs.gelf.GelfOutput; 21 | import org.graylog.collector.outputs.gelf.GelfOutputConfiguration; 22 | import org.graylog.collector.outputs.stdout.StdoutOutput; 23 | import org.graylog.collector.outputs.stdout.StdoutOutputConfiguration; 24 | 25 | public class OutputsModule extends CollectorModule { 26 | @Override 27 | protected void configure() { 28 | registerOutput("gelf", 29 | GelfOutput.class, 30 | GelfOutput.Factory.class, 31 | GelfOutputConfiguration.class, 32 | GelfOutputConfiguration.Factory.class); 33 | 34 | registerOutput("stdout", 35 | StdoutOutput.class, 36 | StdoutOutput.Factory.class, 37 | StdoutOutputConfiguration.class, 38 | StdoutOutputConfiguration.Factory.class); 39 | 40 | registerBufferConsumer(OutputRouter.class); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/utils/MemoryReporterServiceConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import com.typesafe.config.Config; 20 | 21 | import javax.inject.Inject; 22 | import java.util.concurrent.TimeUnit; 23 | 24 | public class MemoryReporterServiceConfiguration { 25 | private long interval = 1000L; 26 | private boolean enable = false; 27 | 28 | @Inject 29 | public MemoryReporterServiceConfiguration(final Config config) { 30 | if (config.hasPath("debug")) { 31 | final Config debug = config.getConfig("debug"); 32 | 33 | if (debug.hasPath("memory-reporter")) { 34 | this.enable = debug.getBoolean("memory-reporter"); 35 | } 36 | 37 | if (debug.hasPath("memory-reporter-interval")) { 38 | this.interval = debug.getDuration("memory-reporter-interval", TimeUnit.MILLISECONDS); 39 | } 40 | } 41 | } 42 | 43 | public long getInterval() { 44 | return interval; 45 | } 46 | 47 | public boolean isEnable() { 48 | return enable; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/metrics/MetricServiceConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.metrics; 18 | 19 | import com.typesafe.config.Config; 20 | import org.joda.time.Duration; 21 | 22 | import javax.inject.Inject; 23 | import java.util.concurrent.TimeUnit; 24 | 25 | public class MetricServiceConfiguration { 26 | private boolean enableLog = false; 27 | private Duration reportDuration = new Duration(60000); 28 | 29 | @Inject 30 | public MetricServiceConfiguration(Config config) { 31 | if (config.hasPath("metrics")) { 32 | final Config metrics = config.getConfig("metrics"); 33 | 34 | this.enableLog = metrics.hasPath("enable-logging") && metrics.getBoolean("enable-logging"); 35 | 36 | if (metrics.hasPath("log-duration")) { 37 | this.reportDuration = new Duration(metrics.getDuration("log-duration", TimeUnit.MILLISECONDS)); 38 | } 39 | } 40 | } 41 | 42 | public boolean isEnableLog() { 43 | return enableLog; 44 | } 45 | 46 | public Duration getReportDuration() { 47 | return reportDuration; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/heartbeat/CollectorRegistrationRequest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.heartbeat; 18 | 19 | import com.fasterxml.jackson.annotation.JsonAutoDetect; 20 | import com.fasterxml.jackson.annotation.JsonCreator; 21 | import com.fasterxml.jackson.annotation.JsonProperty; 22 | import com.google.auto.value.AutoValue; 23 | 24 | import javax.validation.Valid; 25 | import javax.validation.constraints.NotNull; 26 | import javax.validation.constraints.Size; 27 | 28 | @AutoValue 29 | @JsonAutoDetect 30 | public abstract class CollectorRegistrationRequest { 31 | @JsonProperty("node_id") 32 | @NotNull 33 | @Size(min = 1) 34 | public abstract String nodeId(); 35 | 36 | @JsonProperty("node_details") 37 | public abstract CollectorNodeDetailsSummary nodeDetails(); 38 | 39 | @JsonCreator 40 | public static CollectorRegistrationRequest create(@JsonProperty("node_id") String nodeId, 41 | @JsonProperty("node_details") @Valid CollectorNodeDetailsSummary nodeDetails) { 42 | return new AutoValue_CollectorRegistrationRequest(nodeId, nodeDetails); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/utils/CollectorHostNameConfigurationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import com.typesafe.config.Config; 20 | import com.typesafe.config.ConfigFactory; 21 | import org.junit.Test; 22 | 23 | import java.util.Collections; 24 | 25 | import static org.junit.Assert.assertEquals; 26 | import static org.junit.Assert.assertNull; 27 | 28 | public class CollectorHostNameConfigurationTest { 29 | @Test 30 | public void getHostNameReturnsNullIfHostNameNotOverridden() { 31 | final Config config = ConfigFactory.empty(); 32 | final CollectorHostNameConfiguration hostNameConfiguration = new CollectorHostNameConfiguration(config); 33 | assertNull(hostNameConfiguration.getHostName()); 34 | } 35 | 36 | @Test 37 | public void getHostNameReturnsHostNameIfHostNameOverridden() { 38 | final Config config = ConfigFactory.parseMap(Collections.singletonMap("host-name", "foobar.example.net")); 39 | final CollectorHostNameConfiguration hostNameConfiguration = new CollectorHostNameConfiguration(config); 40 | assertEquals("foobar.example.net", hostNameConfiguration.getHostName()); 41 | } 42 | } -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/inputs/InputsModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.inputs; 18 | 19 | import org.graylog.collector.guice.CollectorModule; 20 | import org.graylog.collector.inputs.eventlog.WindowsEventlogInput; 21 | import org.graylog.collector.inputs.eventlog.WindowsEventlogInputConfiguration; 22 | import org.graylog.collector.inputs.file.FileInput; 23 | import org.graylog.collector.inputs.file.FileInputConfiguration; 24 | import org.graylog.collector.utils.Utils; 25 | 26 | public class InputsModule extends CollectorModule { 27 | @Override 28 | protected void configure() { 29 | registerInput("file", 30 | FileInput.class, 31 | FileInput.Factory.class, 32 | FileInputConfiguration.class, 33 | FileInputConfiguration.Factory.class); 34 | 35 | if (Utils.isWindows()) { 36 | registerInput("windows-eventlog", 37 | WindowsEventlogInput.class, 38 | WindowsEventlogInput.Factory.class, 39 | WindowsEventlogInputConfiguration.class, 40 | WindowsEventlogInputConfiguration.Factory.class); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/services/ServiceManagerProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.services; 18 | 19 | import com.google.common.collect.ImmutableSet; 20 | import com.google.common.util.concurrent.Service; 21 | import com.google.common.util.concurrent.ServiceManager; 22 | import com.google.inject.Provider; 23 | import org.graylog.collector.config.ConfigurationRegistry; 24 | 25 | import javax.inject.Inject; 26 | import java.util.Set; 27 | 28 | public class ServiceManagerProvider implements Provider { 29 | private final Set services; 30 | private final ConfigurationRegistry configuration; 31 | 32 | @Inject 33 | public ServiceManagerProvider(Set services, ConfigurationRegistry configuration) { 34 | this.services = services; 35 | this.configuration = configuration; 36 | } 37 | 38 | @Override 39 | public ServiceManager get() { 40 | final ImmutableSet allServices = ImmutableSet.builder() 41 | .addAll(services) 42 | .addAll(configuration.getServices()) 43 | .build(); 44 | 45 | return new ServiceManager(allServices); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/outputs/stdout/StdoutOutputConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.outputs.stdout; 18 | 19 | import com.google.inject.assistedinject.Assisted; 20 | import com.typesafe.config.Config; 21 | import org.graylog.collector.config.ConfigurationUtils; 22 | import org.graylog.collector.outputs.OutputConfiguration; 23 | 24 | import javax.inject.Inject; 25 | 26 | public class StdoutOutputConfiguration extends OutputConfiguration { 27 | private final StdoutOutput.Factory outputFactory; 28 | 29 | public interface Factory extends OutputConfiguration.Factory { 30 | @Override 31 | StdoutOutputConfiguration create(String id, Config config); 32 | } 33 | 34 | @Inject 35 | public StdoutOutputConfiguration(@Assisted String id, 36 | @Assisted Config output, 37 | StdoutOutput.Factory outputFactory) { 38 | super(id, output); 39 | this.outputFactory = outputFactory; 40 | } 41 | 42 | @Override 43 | public StdoutOutput createOutput() { 44 | return outputFactory.create(this); 45 | } 46 | 47 | @Override 48 | public String toString() { 49 | return ConfigurationUtils.toString(this); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/utils/CollectorHostNameSupplierTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import org.junit.Before; 20 | import org.junit.Test; 21 | import org.junit.runner.RunWith; 22 | import org.mockito.Mock; 23 | import org.mockito.runners.MockitoJUnitRunner; 24 | 25 | import static org.junit.Assert.assertEquals; 26 | import static org.junit.Assert.assertNotNull; 27 | import static org.mockito.Mockito.when; 28 | 29 | @RunWith(MockitoJUnitRunner.class) 30 | public class CollectorHostNameSupplierTest { 31 | @Mock 32 | public CollectorId collectorId; 33 | 34 | @Before 35 | public void setUp() throws Exception { 36 | when(collectorId.toString()).thenReturn("cafebabedeadbeef"); 37 | } 38 | 39 | @Test 40 | public void getReturnsDefaultHostName() throws Exception { 41 | final CollectorHostNameSupplier supplier = new CollectorHostNameSupplier("foobar.example.net", collectorId); 42 | assertEquals("foobar.example.net", supplier.get()); 43 | } 44 | 45 | @Test 46 | public void getDetectsHostNameIfDefaultIsMissing() throws Exception { 47 | final CollectorHostNameSupplier supplier = new CollectorHostNameSupplier(null, collectorId); 48 | final String hostName = supplier.get(); 49 | assertNotNull(hostName); 50 | } 51 | } -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/file/CollectingBuffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file; 18 | 19 | import com.google.common.collect.Queues; 20 | import org.graylog.collector.Message; 21 | import org.graylog.collector.buffer.Buffer; 22 | 23 | import java.util.concurrent.BlockingQueue; 24 | 25 | class CollectingBuffer implements Buffer { 26 | 27 | private final BlockingQueue messages = Queues.newArrayBlockingQueue(16); 28 | private boolean outOfCapacity = false; 29 | private boolean processingDisabled = false; 30 | 31 | @Override 32 | public void insert(Message message) { 33 | messages.add(message); 34 | } 35 | 36 | @Override 37 | public Message remove() { 38 | return null; 39 | } 40 | 41 | public boolean isOutOfCapacity() { 42 | return outOfCapacity; 43 | } 44 | 45 | public void setOutOfCapacity(boolean outOfCapacity) { 46 | this.outOfCapacity = outOfCapacity; 47 | } 48 | 49 | public boolean isProcessingDisabled() { 50 | return processingDisabled; 51 | } 52 | 53 | public void setProcessingDisabled(boolean processingDisabled) { 54 | this.processingDisabled = processingDisabled; 55 | } 56 | 57 | public BlockingQueue getMessageQueue() { 58 | return messages; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/services/CollectorServiceManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.services; 18 | 19 | import com.google.common.collect.ImmutableMultimap; 20 | import com.google.common.util.concurrent.Service; 21 | import com.google.common.util.concurrent.ServiceManager; 22 | import org.graylog.collector.config.ConfigurationRegistry; 23 | 24 | import javax.inject.Inject; 25 | 26 | public class CollectorServiceManager { 27 | private final ServiceManager serviceManager; 28 | private final ConfigurationRegistry configuration; 29 | 30 | @Inject 31 | public CollectorServiceManager(ServiceManager serviceManager, ConfigurationRegistry configuration) { 32 | this.serviceManager = serviceManager; 33 | this.configuration = configuration; 34 | } 35 | 36 | public ConfigurationRegistry getConfiguration() { 37 | return configuration; 38 | } 39 | 40 | public void start() { 41 | serviceManager.startAsync().awaitHealthy(); 42 | } 43 | 44 | public void stop() { 45 | serviceManager.stopAsync().awaitStopped(); 46 | } 47 | 48 | public void awaitStopped() { 49 | serviceManager.awaitStopped(); 50 | } 51 | 52 | public ImmutableMultimap servicesByState() { 53 | return serviceManager.servicesByState(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/serverapi/RestAdapterProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.serverapi; 18 | 19 | import com.fasterxml.jackson.databind.ObjectMapper; 20 | import org.graylog.collector.CollectorVersion; 21 | import org.graylog.collector.annotations.GraylogServerURL; 22 | import retrofit.RequestInterceptor; 23 | import retrofit.RestAdapter; 24 | import retrofit.converter.JacksonConverter; 25 | 26 | import javax.inject.Inject; 27 | import javax.inject.Provider; 28 | 29 | public class RestAdapterProvider implements Provider { 30 | private final String graylogServerURL; 31 | 32 | @Inject 33 | public RestAdapterProvider(@GraylogServerURL String graylogServerURL) { 34 | this.graylogServerURL = graylogServerURL; 35 | } 36 | 37 | @Override 38 | public RestAdapter get() { 39 | return new RestAdapter.Builder() 40 | .setEndpoint(graylogServerURL) 41 | .setConverter(new JacksonConverter(new ObjectMapper())) 42 | .setRequestInterceptor(new RequestInterceptor() { 43 | @Override 44 | public void intercept(RequestFacade request) { 45 | request.addHeader("User-Agent", "Graylog Collector " + CollectorVersion.CURRENT); 46 | request.addHeader("X-Graylog-Collector-Version", CollectorVersion.CURRENT.version()); 47 | } 48 | }) 49 | .build(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/config/ConfigurationUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.config; 18 | 19 | import com.google.common.base.Function; 20 | import com.google.common.base.Joiner; 21 | import com.google.common.collect.Iterables; 22 | 23 | import javax.annotation.Nullable; 24 | import java.util.Locale; 25 | import java.util.Map; 26 | 27 | public class ConfigurationUtils { 28 | public static String toString(Configuration configurationObject) { 29 | return toString(configurationObject, configurationObject); 30 | } 31 | 32 | public static String toString(Configuration configurationObject, Object nameClass) { 33 | final Map values = configurationObject.toStringValues(); 34 | final Iterable strings = Iterables.transform(values.entrySet(), new Function, String>() { 35 | @Nullable 36 | @Override 37 | public String apply(@Nullable Map.Entry input) { 38 | if (input == null) { 39 | return ""; 40 | } 41 | return String.format(Locale.getDefault(), "%s='%s'", input.getKey(), input.getValue()); 42 | } 43 | }); 44 | 45 | final StringBuffer sb = new StringBuffer(nameClass.getClass().getSimpleName()); 46 | sb.append('{'); 47 | sb.append(Joiner.on(", ").join(strings)); 48 | sb.append('}'); 49 | 50 | return sb.toString(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/config/constraints/IsOneOfValidator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.config.constraints; 18 | 19 | import com.google.common.base.Joiner; 20 | import org.hibernate.validator.constraintvalidation.HibernateConstraintValidatorContext; 21 | 22 | import javax.validation.ConstraintValidator; 23 | import javax.validation.ConstraintValidatorContext; 24 | import java.util.Arrays; 25 | 26 | public class IsOneOfValidator implements ConstraintValidator { 27 | private String[] strings; 28 | 29 | @Override 30 | public void initialize(IsOneOf constraintAnnotation) { 31 | this.strings = constraintAnnotation.value(); 32 | } 33 | 34 | @Override 35 | public boolean isValid(String value, ConstraintValidatorContext context) { 36 | if (value == null) { 37 | return true; 38 | } 39 | final boolean valid = Arrays.asList(strings).contains(value); 40 | 41 | if (!valid) { 42 | HibernateConstraintValidatorContext hibernateContext = context.unwrap(HibernateConstraintValidatorContext.class); 43 | hibernateContext.disableDefaultConstraintViolation(); 44 | 45 | hibernateContext.addExpressionVariable("validValues", Joiner.on(" ").join(strings)) 46 | .buildConstraintViolationWithTemplate(hibernateContext.getDefaultConstraintMessageTemplate()) 47 | .addConstraintViolation(); 48 | } 49 | 50 | return valid; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/config/ConfigurationValidator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.config; 18 | 19 | import com.google.common.collect.Lists; 20 | 21 | import javax.validation.ConstraintViolation; 22 | import javax.validation.Validation; 23 | import javax.validation.Validator; 24 | import java.util.Collections; 25 | import java.util.List; 26 | import java.util.Locale; 27 | import java.util.Set; 28 | 29 | public class ConfigurationValidator { 30 | private static final Validator VALIDATOR = Validation.buildDefaultValidatorFactory().getValidator(); 31 | 32 | private final List errors = Lists.newArrayList(); 33 | 34 | public boolean isValid(Configuration configuration) { 35 | return doValidate(configuration); 36 | } 37 | 38 | private boolean doValidate(Object obj) { 39 | final Set> constraintViolations = VALIDATOR.validate(obj); 40 | 41 | if (constraintViolations.size() > 0) { 42 | for (ConstraintViolation violation : constraintViolations) { 43 | final String msg = String.format(Locale.getDefault(), "%s (%s)", violation.getMessage(), violation.getPropertyPath().toString()); 44 | errors.add(new ConfigurationError(msg)); 45 | } 46 | 47 | return false; 48 | } 49 | 50 | return true; 51 | } 52 | 53 | public List getErrors() { 54 | return Collections.unmodifiableList(errors); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/outputs/stdout/StdoutOutput.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.outputs.stdout; 18 | 19 | import com.google.inject.assistedinject.Assisted; 20 | import org.graylog.collector.Message; 21 | import org.graylog.collector.config.ConfigurationUtils; 22 | import org.graylog.collector.outputs.OutputService; 23 | 24 | import javax.inject.Inject; 25 | import java.util.Set; 26 | 27 | public class StdoutOutput extends OutputService { 28 | public interface Factory extends OutputService.Factory { 29 | StdoutOutput create(StdoutOutputConfiguration configuration); 30 | } 31 | 32 | private final StdoutOutputConfiguration configuration; 33 | 34 | @Inject 35 | public StdoutOutput(@Assisted StdoutOutputConfiguration configuration) { 36 | this.configuration = configuration; 37 | } 38 | 39 | @Override 40 | protected void doStart() { 41 | notifyStarted(); 42 | } 43 | 44 | @Override 45 | protected void doStop() { 46 | notifyStopped(); 47 | } 48 | 49 | @Override 50 | public void write(Message message) { 51 | System.out.println("MESSAGE: " + message); 52 | } 53 | 54 | @Override 55 | public String toString() { 56 | return ConfigurationUtils.toString(configuration, this); 57 | } 58 | 59 | @Override 60 | public String getId() { 61 | return configuration.getId(); 62 | } 63 | 64 | @Override 65 | public Set getInputs() { 66 | return configuration.getInputs(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/file/naming/ExactFileStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file.naming; 18 | 19 | import com.google.common.base.Function; 20 | import com.google.common.base.Predicate; 21 | import com.google.common.collect.ImmutableSet; 22 | import com.google.common.collect.Iterables; 23 | 24 | import javax.annotation.Nullable; 25 | import java.nio.file.Path; 26 | import java.util.Set; 27 | 28 | public class ExactFileStrategy implements FileNamingStrategy { 29 | 30 | private final Iterable basePaths; 31 | 32 | public ExactFileStrategy(Path basePath) { 33 | this(ImmutableSet.of(basePath)); 34 | } 35 | 36 | public ExactFileStrategy(Set basePaths) { 37 | this.basePaths = Iterables.transform(basePaths, new Function() { 38 | @Nullable 39 | @Override 40 | public Path apply(Path path) { 41 | return path.normalize().toAbsolutePath(); 42 | } 43 | }); 44 | } 45 | 46 | @Override 47 | public boolean pathMatches(final Path path) { 48 | return Iterables.any(basePaths, new Predicate() { 49 | @Override 50 | public boolean apply(@Nullable Path basePath) { 51 | if (basePath == null) { 52 | return false; 53 | } 54 | 55 | Path normalizedPath = path.normalize(); 56 | normalizedPath = basePath.getParent().resolve(normalizedPath); 57 | 58 | return basePath.equals(normalizedPath); 59 | } 60 | }); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/file/ChunkBufferStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file; 18 | 19 | import com.google.common.collect.Maps; 20 | import io.netty.buffer.ByteBuf; 21 | import io.netty.buffer.Unpooled; 22 | 23 | import java.nio.file.Path; 24 | import java.util.concurrent.ConcurrentMap; 25 | 26 | /** 27 | * Stores {@link ByteBuf} chunks for a path. It appends new buffers to existing ones. 28 | * 29 | * This class is thread-safe. 30 | */ 31 | public class ChunkBufferStore { 32 | private final ConcurrentMap buffersPerFile = Maps.newConcurrentMap(); 33 | 34 | public ChunkBufferStore() { 35 | } 36 | 37 | /** 38 | * Returns the {@link ByteBuf} for the given path, null if there is no buffer for the path. 39 | * 40 | * @param path the path 41 | * @return the buffer for the given path or null 42 | */ 43 | public ByteBuf get(final Path path) { 44 | return buffersPerFile.get(path); 45 | } 46 | 47 | /** 48 | * Stores the {@link ByteBuf} for the given path. If there is a buffer for the path already, the new buffer 49 | * will be appended to the existing one. 50 | * 51 | * @param path the path 52 | * @param chunk the buffer 53 | */ 54 | public void put(final Path path, final ByteBuf chunk) { 55 | synchronized (this) { 56 | final ByteBuf buf = get(path); 57 | 58 | if (buf == null) { 59 | buffersPerFile.put(path, chunk); 60 | } else { 61 | buffersPerFile.put(path, Unpooled.wrappedBuffer(buf, chunk)); 62 | } 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/outputs/OutputRouter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.outputs; 18 | 19 | import org.graylog.collector.Message; 20 | import org.graylog.collector.buffer.BufferConsumer; 21 | import org.graylog.collector.config.ConfigurationRegistry; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import javax.inject.Inject; 26 | import java.util.Set; 27 | 28 | public class OutputRouter implements BufferConsumer { 29 | private static final Logger LOG = LoggerFactory.getLogger(OutputRouter.class); 30 | private final Set outputs; 31 | 32 | @Inject 33 | public OutputRouter(ConfigurationRegistry configuration) { 34 | this.outputs = configuration.getOutputs(); 35 | } 36 | 37 | @Override 38 | public void process(Message message) { 39 | LOG.debug("Routing message to outputs. {}", message); 40 | 41 | for (Output output : outputs) { 42 | final Set outputInputs = output.getInputs(); 43 | final Set messageOutputs = message.getOutputs(); 44 | 45 | if (outputInputs.isEmpty() && messageOutputs.isEmpty()) { 46 | output.write(message); 47 | } else if (outputInputs.contains(message.getInput()) || messageOutputs.contains(output.getId())) { 48 | output.write(message); 49 | } 50 | } 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | final StringBuffer sb = new StringBuffer("OutputRouter{"); 56 | sb.append("outputs=").append(outputs); 57 | sb.append('}'); 58 | return sb.toString(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Graylog 2 | 3 | Thank you very much for considering contributing to the Graylog project! 4 | We <3 our community. 5 | 6 | To make handling all the community contributions as easy as possible for us 7 | we ask you to follow these steps as good as possible: 8 | 9 | ## Bug reports 10 | 11 | 1. Search the issues of the [graylog2-server](https://github.com/Graylog2/graylog2-server), 12 | [graylog2-web-interface](https://github.com/Graylog2/graylog2-web-interface) 13 | and [graylog-collector](https://github.com/Graylog2/collector) repositories for 14 | opened or closed issues to avoid duplicating effort. 15 | 2. Be clear about the issue. It is always better to include too much 16 | information than too little. Include screenshots explaining the problem 17 | if you can. 18 | 3. Always include the versions of affected Graylog components you are running. 19 | 4. Provide exact steps to reproduce the issue if at any possible. 20 | 21 | ## Pull requests 22 | 23 | 1. Please sign the [Individual Contributor Assignment Agreement](https://s3.amazonaws.com/graylog2public/Individual+Contributor+Assignment+Agreement+-+Graylog.pdf) 24 | before sending pull requests. We cannot accept code without this. 25 | 2. Send the signed agreement to contributor@graylog.com 26 | 3. Search the issues of the [graylog2-server](https://github.com/Graylog2/graylog2-server), 27 | [graylog2-web-interface](https://github.com/Graylog2/graylog2-web-interface) 28 | and [graylog-collector](https://github.com/Graylog2/collector) repositories for 29 | opened or closed issues to avoid duplicating effort. 30 | 4. Contact the Graylog, Inc team via [any communication channel](https://www.graylog.org/community-support/) 31 | (contact form, mailing list, IRC, Github issue tracker) before working 32 | on a big change to make sure that there are chances of acceptance. 33 | 5. Include tests if at any possible. 34 | 6. Submit the pull request. 35 | 36 | ## Feature requests 37 | 38 | We are always happy about any feature request! Do not hesitate to create as 39 | many feature request issues as you want. 40 | 41 | 1. Search the existing feature requests in our [Product Ideas](https://www.graylog.org/product-ideas/) 42 | page to avoid creating duplicate ones. 43 | 2. Explain exactly what feature you’d like to see and always include a use 44 | case for why you think it is useful. 45 | -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/config/constraints/IsOneOfValidatorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.config.constraints; 18 | 19 | import org.junit.BeforeClass; 20 | import org.junit.Test; 21 | 22 | import javax.validation.ConstraintViolation; 23 | import javax.validation.Validation; 24 | import javax.validation.Validator; 25 | import javax.validation.ValidatorFactory; 26 | import java.util.Set; 27 | 28 | import static org.junit.Assert.assertEquals; 29 | 30 | public class IsOneOfValidatorTest { 31 | private static class TestObject { 32 | @IsOneOf({"hello", "world"}) 33 | private final String theValue; 34 | 35 | public TestObject(String theValue) { 36 | this.theValue = theValue; 37 | } 38 | } 39 | 40 | private static Validator validator; 41 | 42 | @BeforeClass 43 | public static void setUpClass() throws Exception { 44 | ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); 45 | validator = factory.getValidator(); 46 | } 47 | 48 | @Test 49 | public void testConstraint() throws Exception { 50 | assertEquals(0, validate("hello").size()); 51 | assertEquals(0, validate("world").size()); 52 | assertEquals(1, validate("wat?").size()); 53 | assertEquals("\"wat?\" is not one of: hello world", validate("wat?").iterator().next().getMessage()); 54 | 55 | // Check different case. 56 | assertEquals(1, validate("Hello").size()); 57 | assertEquals(1, validate("WORLD").size()); 58 | } 59 | 60 | private Set> validate(String value) { 61 | return validator.validate(new TestObject(value)); 62 | } 63 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Graylog Collector ChangeLog 2 | =========================== 3 | 4 | ## v0.5.0 (2016-05-03) 5 | 6 | * Add UDP support to GELF output. (#59) 7 | * Allow to override reported host name (#71) 8 | * Use canonical host name as message source in WindowsEventlogInput (#72) 9 | * Add compatiblity with Graylog 2.x (#85) 10 | * Improve error message if node ID could not be persisted 11 | 12 | ## v0.4.2 (2016-01-04) 13 | 14 | * Fix concurrency issue in MessageBuilder/MessageFields. 15 | 16 | ## v0.4.1 (2015-09-07) 17 | 18 | * Fix file rotation detection on platforms with a slow watchservice 19 | implementation like Mac OS X and AIX. 20 | 21 | ## v0.4.0 (2015-07-23) 22 | 23 | * Log operating system version and platform on startup. 24 | * More resource usage improvements by using less threads per input/output. 25 | * Add static message-fields support. (#45) 26 | 27 | ## v0.3.0 (2015-07-14) 28 | 29 | * Lots of bug fixes regarding resource usage. 30 | * Wildcard support for file inputs. (#24, #42) 31 | * The GELF output does not send a `level` field for messages from file inputs 32 | anymore. It was hardcoded to INFO before. 33 | * Improved example configuration file. 34 | 35 | ## v0.2.5 (2015-07-01) 36 | 37 | * Fix classpath for Windows service to unbreak Sigar. (#39) 38 | 39 | ## v0.2.4 (2015-06-24) 40 | 41 | * Fix several quoting issues for Windows start/service scripts. (#28) 42 | * Rename some variables in Linux startup scripts. 43 | 44 | ## v0.2.3 (2015-06-23) 45 | 46 | * Replace readlink usage with manually resolving symbolic links. (#20) 47 | * Do not stop the collector if the configured file does not exist. (#33) 48 | 49 | ## v0.2.2 (2015-06-02) 50 | 51 | * Improve Windows batch script compatibility. (#15) 52 | 53 | ## v0.2.1 (2015-05-20) 54 | 55 | * Fixed problem with release infrastructure. 56 | 57 | ## v0.2.0 (2015-05-20) 58 | 59 | * Support for reading Windows eventlog. 60 | * Connect to Graylog server API and send heartbeats. 61 | * Make content splitter for file input configurable. 62 | * Auto detect line endings in newline content splitter. (CRLF vs. LF) 63 | * Add scripts to run on Windows and to install as a Windows service. 64 | * Add charset support to the file input. 65 | * Improved file input regarding following files. 66 | * Improved performance when reading from files. 67 | 68 | ## v0.1.1 (2015-02-18) 69 | 70 | * Fix validation messages. 71 | 72 | ## v0.1.0 (2015-02-18) 73 | 74 | * Initial release. 75 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/outputs/OutputConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.outputs; 18 | 19 | import com.google.common.base.Joiner; 20 | import com.google.common.base.Splitter; 21 | import com.google.common.collect.Sets; 22 | import com.typesafe.config.Config; 23 | import org.graylog.collector.config.Configuration; 24 | import org.hibernate.validator.constraints.NotBlank; 25 | 26 | import javax.validation.constraints.NotNull; 27 | import java.util.Collections; 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | import java.util.Set; 31 | 32 | public abstract class OutputConfiguration implements Configuration { 33 | public interface Factory { 34 | C create(String id, Config config); 35 | } 36 | 37 | @NotBlank 38 | private final String id; 39 | 40 | @NotNull 41 | private Set inputs = Sets.newHashSet(); 42 | 43 | public OutputConfiguration(String id, Config config) { 44 | this.id = id; 45 | 46 | if (config.hasPath("inputs")) { 47 | this.inputs = Sets.newHashSet(Splitter.on(",").omitEmptyStrings().trimResults().split(config.getString("inputs"))); 48 | } 49 | } 50 | 51 | public abstract OutputService createOutput(); 52 | 53 | @Override 54 | public String getId() { 55 | return id; 56 | } 57 | 58 | public Set getInputs() { 59 | return inputs; 60 | } 61 | 62 | @Override 63 | public Map toStringValues() { 64 | return Collections.unmodifiableMap(new HashMap() { 65 | { 66 | put("id", getId()); 67 | put("inputs", Joiner.on(",").join(getInputs())); 68 | } 69 | }); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/utils/CollectorHostNameSupplier.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import com.google.common.base.Supplier; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import javax.annotation.Nullable; 24 | import java.net.InetAddress; 25 | import java.net.UnknownHostException; 26 | 27 | import static java.util.Objects.requireNonNull; 28 | 29 | public class CollectorHostNameSupplier implements Supplier { 30 | private static final Logger LOG = LoggerFactory.getLogger(CollectorHostNameSupplier.class); 31 | 32 | private final String defaultHostName; 33 | private final CollectorId collectorId; 34 | 35 | public CollectorHostNameSupplier(@Nullable String defaultHostName, CollectorId collectorId) { 36 | this.defaultHostName = defaultHostName; 37 | this.collectorId = requireNonNull(collectorId); 38 | } 39 | 40 | @Override 41 | public String get() { 42 | return defaultHostName == null ? detectHostname() : defaultHostName; 43 | } 44 | 45 | private String detectHostname() { 46 | String hostname; 47 | try { 48 | hostname = InetAddress.getLocalHost().getHostName(); 49 | } catch (UnknownHostException e) { 50 | hostname = System.getenv("HOSTNAME"); 51 | if (hostname == null) { 52 | hostname = System.getenv("COMPUTERNAME"); 53 | } 54 | if (hostname == null) { 55 | hostname = "unknown-" + collectorId.toString(); 56 | LOG.warn("Unable to detect the local host name, falling back to \"{}\". " 57 | + "Use the \"host-name\" configuration setting to override.", hostname); 58 | } 59 | } 60 | 61 | return hostname; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/MessageFieldsTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector; 18 | 19 | import org.junit.Test; 20 | 21 | import java.util.HashMap; 22 | 23 | import static org.assertj.core.api.Assertions.assertThat; 24 | import static org.junit.Assert.assertEquals; 25 | 26 | public class MessageFieldsTest { 27 | @Test 28 | public void testFields() { 29 | final MessageFields fields = new MessageFields(); 30 | 31 | fields.put("bool", true); 32 | fields.put("int", 123); 33 | fields.put("long", 1500L); 34 | fields.put("double", 1.4D); 35 | fields.put("string", "string"); 36 | 37 | final HashMap map = new HashMap() { 38 | { 39 | put("bool", true); 40 | put("int", 123); 41 | put("long", 1500L); 42 | put("double", 1.4D); 43 | put("string", "string"); 44 | } 45 | }; 46 | 47 | assertEquals(map, fields.asMap()); 48 | } 49 | 50 | @Test 51 | public void testCopy() throws Exception { 52 | final MessageFields fields = new MessageFields(); 53 | final MessageFields copy = fields.copy(); 54 | 55 | fields.put("boolean", true); 56 | fields.put("number", 1); 57 | fields.put("string", "foo"); 58 | copy.put("boolean", false); 59 | copy.put("number", 2); 60 | copy.put("string", "bar"); 61 | 62 | assertThat(fields.asMap().get("boolean")).isEqualTo(true); 63 | assertThat(fields.asMap().get("number")).isEqualTo(1); 64 | assertThat(fields.asMap().get("string")).isEqualTo("foo"); 65 | assertThat(copy.asMap().get("boolean")).isEqualTo(false); 66 | assertThat(copy.asMap().get("number")).isEqualTo(2); 67 | assertThat(copy.asMap().get("string")).isEqualTo("bar"); 68 | } 69 | } -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/file/SinglePathSet.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file; 18 | 19 | import com.google.common.base.MoreObjects; 20 | import com.google.common.collect.ImmutableSet; 21 | 22 | import java.io.IOException; 23 | import java.nio.file.FileSystem; 24 | import java.nio.file.FileSystems; 25 | import java.nio.file.Files; 26 | import java.nio.file.Path; 27 | import java.util.Collections; 28 | import java.util.Set; 29 | 30 | import static com.google.common.base.Preconditions.checkNotNull; 31 | 32 | public class SinglePathSet implements PathSet { 33 | private final Path path; 34 | 35 | public SinglePathSet(final String path) { 36 | this(path, FileSystems.getDefault()); 37 | } 38 | 39 | public SinglePathSet(final String path, final FileSystem fileSystem) { 40 | this.path = fileSystem.getPath(checkNotNull(path)).toAbsolutePath(); 41 | } 42 | 43 | @Override 44 | public Path getRootPath() { 45 | return path.getParent(); 46 | } 47 | 48 | @Override 49 | public boolean isInSet(Path path) { 50 | return path != null && this.path.equals(path.toAbsolutePath()); 51 | } 52 | 53 | @Override 54 | public Set getPaths() throws IOException { 55 | return Files.exists(path) ? ImmutableSet.of(path) : Collections.emptySet(); 56 | } 57 | 58 | @Override 59 | public boolean equals(Object o) { 60 | if (this == o) return true; 61 | if (o == null || getClass() != o.getClass()) return false; 62 | 63 | SinglePathSet that = (SinglePathSet) o; 64 | 65 | return path.equals(that.path); 66 | } 67 | 68 | @Override 69 | public int hashCode() { 70 | return path.hashCode(); 71 | } 72 | 73 | @Override 74 | public String toString() { 75 | return MoreObjects.toStringHelper(this).add("path", path).toString(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/buffer/MessageBuffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.buffer; 18 | 19 | import com.codahale.metrics.Meter; 20 | import com.codahale.metrics.MetricRegistry; 21 | import com.google.common.collect.Queues; 22 | import org.graylog.collector.Message; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import javax.inject.Inject; 27 | import java.util.concurrent.BlockingQueue; 28 | 29 | import static com.codahale.metrics.MetricRegistry.name; 30 | 31 | public class MessageBuffer implements Buffer { 32 | private static final Logger LOG = LoggerFactory.getLogger(MessageBuffer.class); 33 | 34 | private final BlockingQueue queue; 35 | private final MetricRegistry metricRegistry; 36 | private final Meter inserted; 37 | private final Meter removed; 38 | 39 | @Inject 40 | public MessageBuffer(MessageBufferConfiguration config, MetricRegistry metricRegistry) { 41 | this.metricRegistry = metricRegistry; 42 | this.queue = Queues.newLinkedBlockingQueue(config.getSize()); 43 | 44 | this.inserted = metricRegistry.meter(name(getClass(), "inserted")); 45 | this.removed = metricRegistry.meter(name(getClass(), "removed")); 46 | } 47 | 48 | public void insert(Message message) { 49 | LOG.debug("Adding message to queue: {}", message); 50 | 51 | try { 52 | queue.put(message); 53 | inserted.mark(); 54 | } catch (InterruptedException e) { 55 | LOG.error("Interrupted, dropping message.", e); 56 | } 57 | } 58 | 59 | @Override 60 | public Message remove() { 61 | try { 62 | final Message message = queue.take(); 63 | 64 | if (message != null) { 65 | removed.mark(); 66 | } 67 | 68 | return message; 69 | } catch (InterruptedException e) { 70 | return null; 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/file/ChunkBufferStoreTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file; 18 | 19 | import io.netty.buffer.Unpooled; 20 | import org.junit.Test; 21 | 22 | import java.nio.file.Path; 23 | import java.nio.file.Paths; 24 | 25 | import static com.google.common.base.Charsets.UTF_8; 26 | import static org.junit.Assert.assertEquals; 27 | import static org.junit.Assert.assertNull; 28 | 29 | public class ChunkBufferStoreTest { 30 | @Test 31 | public void test() throws Exception { 32 | final ChunkBufferStore store = new ChunkBufferStore(); 33 | final Path path1 = Paths.get("/tmp/foo-1.log"); 34 | final Path path2 = Paths.get("/tmp/foo-2.log"); 35 | 36 | assertNull("Empty store should not have any buffers...", store.get(path1)); 37 | assertNull("Empty store should not have any buffers...", store.get(path2)); 38 | 39 | store.put(path1, Unpooled.copiedBuffer("log 1\n".getBytes())); 40 | assertEquals("Buffer should have correct data", "log 1\n", store.get(path1).toString(UTF_8)); 41 | 42 | store.put(path2, Unpooled.copiedBuffer("hello\n".getBytes())); 43 | assertEquals("Buffer should have correct data", "hello\n", store.get(path2).toString(UTF_8)); 44 | 45 | store.put(path1, Unpooled.copiedBuffer("log 2\n".getBytes())); 46 | assertEquals("New data should be appended to buffer", "log 1\nlog 2\n", store.get(path1).toString(UTF_8)); 47 | 48 | store.get(path1).readBytes(new byte[store.get(path1).readableBytes()]); 49 | store.put(path1, Unpooled.copiedBuffer("new log\n".getBytes())); 50 | assertEquals("Writing to truncated buffer should only have new data", "new log\n", store.get(path1).toString(UTF_8)); 51 | 52 | store.get(path2).readBytes(new byte[3], 0, 3); 53 | store.put(path2, Unpooled.copiedBuffer("new log\n".getBytes())); 54 | assertEquals("Buffer should have rest of old and the new data", "lo\nnew log\n", store.get(path2).toString(UTF_8)); 55 | } 56 | } -------------------------------------------------------------------------------- /src/main/assembly/collector.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | collector 7 | 8 | tar.gz 9 | zip 10 | 11 | 12 | 13 | ${project.basedir}/bin/windows 14 | /bin/windows 15 | 16 | *.exe 17 | 18 | 19 | 20 | ${project.basedir}/lib/sigar-${sigar.version} 21 | /lib/sigar 22 | 23 | 24 | 25 | 26 | ${project.basedir}/config/collector.conf.example 27 | /config/ 28 | 29 | 30 | ${project.build.directory}/graylog-collector-script-config.sh 31 | graylog-collector-script-config.sh 32 | /bin/ 33 | 0644 34 | 35 | 36 | ${project.build.directory}/graylog-collector.sh 37 | graylog-collector 38 | /bin/ 39 | 0755 40 | 41 | 42 | ${project.build.directory}/graylog-collector.bat 43 | graylog-collector.bat 44 | /bin/ 45 | 0755 46 | dos 47 | 48 | 49 | ${project.build.directory}/graylog-collector-service.bat 50 | graylog-collector-service.bat 51 | /bin/ 52 | 0755 53 | dos 54 | 55 | 56 | ${project.build.directory}/${project.artifactId}-${project.version}.jar 57 | ${project.artifactId}.jar 58 | / 59 | 60 | 61 | -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/config/ConfigurationParserTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.config; 18 | 19 | import com.google.common.collect.Sets; 20 | import com.typesafe.config.Config; 21 | import org.junit.Before; 22 | import org.junit.Rule; 23 | import org.junit.Test; 24 | import org.junit.rules.ExpectedException; 25 | 26 | import java.io.File; 27 | import java.nio.file.Files; 28 | import java.nio.file.attribute.PosixFilePermission; 29 | 30 | import static org.junit.Assert.assertEquals; 31 | 32 | public class ConfigurationParserTest { 33 | private File configFile; 34 | 35 | @Rule 36 | public ExpectedException thrower = ExpectedException.none(); 37 | 38 | @Before 39 | public void setUp() throws Exception { 40 | configFile = Files.createTempFile("collector", ".conf").toFile(); 41 | 42 | Files.write(configFile.toPath(), "message-buffer-size = 128".getBytes()); 43 | } 44 | 45 | @Test 46 | public void testParsing() throws Exception { 47 | final Config config = ConfigurationParser.parse(configFile); 48 | 49 | assertEquals(128, config.getInt("message-buffer-size")); 50 | } 51 | 52 | @Test 53 | public void testMissingConfigFile() throws Exception { 54 | thrower.expect(ConfigurationParser.Error.class); 55 | 56 | configFile.delete(); 57 | 58 | ConfigurationParser.parse(configFile); 59 | } 60 | 61 | @Test 62 | public void testUnreadableFile() throws Exception { 63 | thrower.expect(ConfigurationParser.Error.class); 64 | 65 | Files.setPosixFilePermissions(configFile.toPath(), Sets.newHashSet(new PosixFilePermission[]{})); 66 | 67 | ConfigurationParser.parse(configFile); 68 | } 69 | 70 | @Test 71 | public void testEmptyFile() throws Exception { 72 | thrower.expect(ConfigurationParser.Error.class); 73 | 74 | configFile.delete(); 75 | Files.write(configFile.toPath(), "".getBytes()); 76 | 77 | ConfigurationParser.parse(configFile); 78 | } 79 | } -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/file/MultithreadedBaseTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file; 18 | 19 | import com.google.common.collect.Sets; 20 | import org.junit.After; 21 | import org.junit.Before; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.util.Set; 26 | 27 | import static org.junit.Assert.assertEquals; 28 | import static org.junit.Assert.assertTrue; 29 | 30 | public class MultithreadedBaseTest { 31 | private static final Logger log = LoggerFactory.getLogger(MultithreadedBaseTest.class); 32 | ThreadLocal setupDone = new ThreadLocal() { 33 | @Override 34 | protected Boolean initialValue() { 35 | return false; 36 | } 37 | }; 38 | 39 | private ThreadLocal> assertionErrors = new ThreadLocal>() { 40 | @Override 41 | protected Set initialValue() { 42 | return Sets.newHashSet(); 43 | } 44 | }; 45 | 46 | @Before 47 | public void assertErrorTrap() { 48 | log.info("clearing background assertions {}", Thread.currentThread().getName()); 49 | assertionErrors.get().clear(); 50 | Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { 51 | @Override 52 | public void uncaughtException(Thread t, Throwable e) { 53 | if (e instanceof AssertionError) { 54 | log.info("assertion failed on thread " + t.getName(), e); 55 | assertionErrors.get().add((AssertionError) e); 56 | } 57 | } 58 | }); 59 | setupDone.set(true); 60 | } 61 | 62 | @After 63 | public void checkThreadAssertions() { 64 | assertTrue("Thread " + Thread.currentThread().getName(), setupDone.get()); 65 | log.info("checking background assertions"); 66 | assertEquals("Background threads should not fail assertions", 0, assertionErrors.get().size()); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/MessageFields.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector; 18 | 19 | import com.google.common.base.MoreObjects; 20 | import com.google.common.collect.ImmutableMap; 21 | import com.google.common.collect.Maps; 22 | 23 | import java.util.Map; 24 | 25 | public class MessageFields { 26 | private final Map numberFields = Maps.newConcurrentMap(); 27 | private final Map booleanFields = Maps.newConcurrentMap(); 28 | private final Map stringFields = Maps.newConcurrentMap(); 29 | 30 | public MessageFields(Map numberFields, 31 | Map booleanFields, 32 | Map stringFields) { 33 | this.numberFields.putAll(numberFields); 34 | this.booleanFields.putAll(booleanFields); 35 | this.stringFields.putAll(stringFields); 36 | } 37 | 38 | public MessageFields() { 39 | } 40 | 41 | public void put(String key, Number value) { 42 | numberFields.put(key, value); 43 | } 44 | 45 | public void put(String key, boolean value) { 46 | booleanFields.put(key, value); 47 | } 48 | 49 | public void put(String key, String value) { 50 | stringFields.put(key, value); 51 | } 52 | 53 | public Map asMap() { 54 | return ImmutableMap.builder() 55 | .putAll(numberFields) 56 | .putAll(booleanFields) 57 | .putAll(stringFields) 58 | .build(); 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | final MoreObjects.ToStringHelper stringHelper = MoreObjects.toStringHelper(this); 64 | 65 | for (Map.Entry entry : asMap().entrySet()) { 66 | stringHelper.add(entry.getKey(), entry.getValue()); 67 | } 68 | 69 | return stringHelper.toString(); 70 | } 71 | 72 | public MessageFields copy() { 73 | return new MessageFields(numberFields, booleanFields, stringFields); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/cli/Main.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.cli; 18 | 19 | import io.airlift.airline.Cli; 20 | import io.airlift.airline.ParseException; 21 | import org.graylog.collector.cli.commands.CollectorCommand; 22 | import org.graylog.collector.cli.commands.CollectorHelp; 23 | import org.graylog.collector.cli.commands.Run; 24 | import org.graylog.collector.cli.commands.Version; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | public class Main { 29 | static { 30 | // Hijack java.util.logging, see https://logging.apache.org/log4j/log4j-2.2/log4j-jul/index.html 31 | System.setProperty("java.util.logging.manager", org.apache.logging.log4j.jul.LogManager.class.getCanonicalName()); 32 | } 33 | 34 | private static final Logger LOG = LoggerFactory.getLogger(Main.class); 35 | private static CollectorCommand command = null; 36 | 37 | public static void main(String[] args) { 38 | final Cli.CliBuilder cliBuilder = Cli.builder("graylog-collector") 39 | .withDescription("Graylog Collector") 40 | .withDefaultCommand(CollectorHelp.class) 41 | .withCommand(CollectorHelp.class) 42 | .withCommand(Version.class) 43 | .withCommand(Run.class); 44 | 45 | final Cli cli = cliBuilder.build(); 46 | 47 | try { 48 | command = cli.parse(args); 49 | configureShutdownHook(command); 50 | command.run(); 51 | } catch (ParseException e) { 52 | LOG.error(e.getMessage()); 53 | LOG.error("Exit"); 54 | } 55 | } 56 | 57 | public static void stop(String[] args) { 58 | if (command != null) { 59 | command.stop(); 60 | } 61 | } 62 | 63 | private static void configureShutdownHook(final CollectorCommand command) { 64 | Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { 65 | @Override 66 | public void run() { 67 | command.stop(); 68 | } 69 | })); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/CollectorVersion.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector; 18 | 19 | import java.io.IOException; 20 | import java.io.InputStream; 21 | import java.util.Properties; 22 | 23 | import static com.google.common.base.Strings.isNullOrEmpty; 24 | 25 | public class CollectorVersion { 26 | public static final CollectorVersion CURRENT; 27 | 28 | static { 29 | String version = "NONE"; 30 | String commitId = "NONE"; 31 | String commitIdShort = "NONE"; 32 | String timestamp = "NONE"; 33 | 34 | final InputStream stream = CollectorVersion.class.getResourceAsStream("/collector-version.properties"); 35 | final Properties properties = new Properties(); 36 | 37 | try { 38 | properties.load(stream); 39 | 40 | version = properties.getProperty("version"); 41 | commitId = properties.getProperty("commit-id"); 42 | timestamp = properties.getProperty("timestamp"); 43 | 44 | if (!isNullOrEmpty(commitId) && !"NONE".equals(commitId)) { 45 | commitIdShort = commitId.substring(0, 7); 46 | } 47 | } catch (IOException ignored) { 48 | } 49 | 50 | CURRENT = new CollectorVersion(version, commitId, commitIdShort, timestamp); 51 | } 52 | 53 | private final String version; 54 | private final String commitId; 55 | private final String commitIdShort; 56 | private final String timestamp; 57 | 58 | public CollectorVersion(String version, String commitId, String commitIdShort, String timestamp) { 59 | this.version = version; 60 | this.commitId = commitId; 61 | this.commitIdShort = commitIdShort; 62 | this.timestamp = timestamp; 63 | } 64 | 65 | public String version() { 66 | return version; 67 | } 68 | 69 | public String commitId() { 70 | return commitId; 71 | } 72 | 73 | public String commitIdShort() { 74 | return commitIdShort; 75 | } 76 | 77 | public String timestamp() { 78 | return timestamp; 79 | } 80 | 81 | @Override 82 | public String toString() { 83 | return "v" + version() + "(commit " + commitIdShort() + ")"; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/file/naming/FileNamingStrategyTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file.naming; 18 | 19 | import org.junit.Test; 20 | 21 | import java.nio.file.FileSystem; 22 | import java.nio.file.FileSystems; 23 | import java.nio.file.Path; 24 | 25 | import static org.junit.Assert.assertFalse; 26 | import static org.junit.Assert.assertTrue; 27 | 28 | public class FileNamingStrategyTest { 29 | 30 | @Test 31 | public void testNumberSuffixMatches() { 32 | final FileSystem fs = FileSystems.getDefault(); 33 | final Path baseName = fs.getPath("/tmp", "logfile.log"); 34 | final NumberSuffixStrategy m = new NumberSuffixStrategy(baseName); 35 | 36 | assertTrue("same file matches", m.pathMatches(fs.getPath("/tmp", "logfile.log"))); 37 | assertTrue("number suffix matches", m.pathMatches(fs.getPath("/tmp", "logfile.log.1"))); 38 | assertTrue("multi-digit suffix matches", m.pathMatches(fs.getPath("/tmp", "logfile.log.1345345"))); 39 | assertFalse("separator must be '.'", m.pathMatches(fs.getPath("/tmp", "logfile.log-123"))); 40 | assertFalse("more suffixes don't match", m.pathMatches(fs.getPath("/tmp", "logfile.log.1234.gz"))); 41 | assertFalse("wrong base path doesn't match", m.pathMatches(fs.getPath("/var/log", "logfile.log.1234"))); 42 | 43 | assertTrue("paths are normalized", m.pathMatches(fs.getPath("/tmp/bar/..", "logfile.log.1"))); 44 | assertTrue("relative paths are resolved", m.pathMatches(fs.getPath("logfile.log.1"))); 45 | 46 | } 47 | 48 | @Test 49 | public void testSameFile() { 50 | final FileSystem fs = FileSystems.getDefault(); 51 | final Path baseName = fs.getPath("/tmp", "logfile.log"); 52 | final ExactFileStrategy m = new ExactFileStrategy(baseName); 53 | 54 | assertTrue("same file matches", m.pathMatches(fs.getPath("/tmp", "logfile.log"))); 55 | assertTrue("paths are normalized", m.pathMatches(fs.getPath("/tmp/foo/..", "logfile.log"))); 56 | assertFalse("only the same file name matches", m.pathMatches(fs.getPath("/tmp", "logfile.log.1"))); 57 | assertTrue("relative paths are resolved", m.pathMatches(fs.getPath("logfile.log"))); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/buffer/BufferProcessor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.buffer; 18 | 19 | import com.google.common.util.concurrent.AbstractExecutionThreadService; 20 | import org.graylog.collector.Message; 21 | import org.graylog.collector.utils.CollectorId; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import javax.inject.Inject; 26 | import java.util.Set; 27 | 28 | public class BufferProcessor extends AbstractExecutionThreadService { 29 | private static final Logger LOG = LoggerFactory.getLogger(BufferProcessor.class); 30 | 31 | private final Buffer buffer; 32 | private final Set consumers; 33 | private final CollectorId collectorId; 34 | private Thread thread; 35 | 36 | @Inject 37 | public BufferProcessor(Buffer buffer, Set consumers, CollectorId collectorId) { 38 | this.buffer = buffer; 39 | this.consumers = consumers; 40 | this.collectorId = collectorId; 41 | } 42 | 43 | @Override 44 | protected void startUp() throws Exception { 45 | this.thread = Thread.currentThread(); 46 | } 47 | 48 | @Override 49 | protected void triggerShutdown() { 50 | thread.interrupt(); 51 | } 52 | 53 | @Override 54 | protected void run() throws Exception { 55 | while (isRunning()) { 56 | final Message message = buffer.remove(); 57 | 58 | if (message != null) { 59 | // It is a bit disgusting to mutate the message object here, but for now the simplest solution for 60 | // adding common fields to it. 61 | decorateMessage(message); 62 | 63 | LOG.debug("Read message from buffer {}", message); 64 | 65 | for (final BufferConsumer consumer : consumers) { 66 | LOG.debug("Processing message with consumer {}", consumer); 67 | consumer.process(message); 68 | } 69 | } 70 | } 71 | } 72 | 73 | private void decorateMessage(Message message) { 74 | message.getFields().put("gl2_source_collector", collectorId.toString()); 75 | message.getFields().put("gl2_source_collector_input", message.getInput()); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/file/splitters/NewlineChunkSplitter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file.splitters; 18 | 19 | import com.google.common.collect.AbstractIterator; 20 | import io.netty.buffer.ByteBuf; 21 | import io.netty.buffer.ByteBufProcessor; 22 | 23 | import java.nio.charset.Charset; 24 | import java.util.Iterator; 25 | 26 | public class NewlineChunkSplitter extends ContentSplitter { 27 | @Override 28 | public Iterable split(final ByteBuf buffer, final Charset charset, final boolean includeRemainingData) { 29 | return new Iterable() { 30 | @Override 31 | public Iterator iterator() { 32 | return new AbstractIterator() { 33 | 34 | @Override 35 | protected String computeNext() { 36 | try { 37 | if (!buffer.isReadable()) { 38 | return endOfData(); 39 | } 40 | final int i = buffer.forEachByte(ByteBufProcessor.FIND_CRLF); 41 | if (i == -1) { 42 | if (includeRemainingData) { 43 | final ByteBuf remaining = buffer.readBytes(buffer.readableBytes()); 44 | return remaining.toString(charset); 45 | } else { 46 | return endOfData(); 47 | } 48 | } 49 | final ByteBuf fullLine = buffer.readBytes(i); 50 | // Strip the \r/\n bytes from the buffer. 51 | final byte readByte = buffer.readByte(); // the \r or \n byte 52 | if (readByte == '\r') { 53 | buffer.readByte(); // the \n byte if previous was \r 54 | } 55 | return fullLine.toString(charset); 56 | } finally { 57 | buffer.discardReadBytes(); 58 | } 59 | } 60 | }; 61 | } 62 | }; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /config/collector.conf.example: -------------------------------------------------------------------------------- 1 | // Graylog Collector example configuration. 2 | 3 | // URL to REST API of Graylog server this collector registers at 4 | server-url = "http://localhost:12900" 5 | 6 | // Enable registration with the Graylog server. (enabled by default) 7 | //enable-registration = true 8 | 9 | // The id used to identify this collector. Can be either a string which is used as id, 10 | // or the location of a file if prefixed with "file:". If the file does not exist, 11 | // an id will be generated and written to that file. If it exists, it is expected 12 | // to contain a single string without spaces which will be used for the id. 13 | // Defaults to "file:config/collector-id" if not specified. 14 | collector-id = "file:config/collector-id" 15 | 16 | // Override the detected local host name and send a custom host name 17 | //host-name = "my-custom-host-name" 18 | 19 | //inputs { 20 | // // A simple file input that follows /var/log/syslog. 21 | // local-syslog { 22 | // type = "file" 23 | // path = "/var/log/syslog" 24 | // charset = "utf-8" 25 | // content-splitter = "newline" 26 | // } 27 | // 28 | // // A globbing file input. Follows all *.access.log files that exist and will be created in /var/log/apache2. 29 | // // You have to split the path into path-glob-root and path-glob-pattern. A usual "/var/log/**/*.{log,txt}" 30 | // // becomes path-glob-root="/var/log" and path-glob-pattern="**/*.{log,txt}". 31 | // apache-access { 32 | // type = "file" 33 | // path-glob-root = "/var/log/apache2" 34 | // path-glob-pattern = "*.access.log" 35 | // } 36 | // 37 | // // An input to read from the given Windows event log. Only works on Windows. 38 | // // Available source-names: Application, System, Security 39 | // win-application { 40 | // type = "windows-eventlog" 41 | // source-name = "Application" 42 | // poll-interval = 1s 43 | // } 44 | //} 45 | 46 | //outputs { 47 | // // GELF output to send messages to a Graylog server. Usually only type, host and port are needed. 48 | // // The other options are for TLS support and to fine-tune the GELF client library. 49 | // gelf-tcp { 50 | // type = "gelf" 51 | // host = "127.0.0.1" 52 | // port = 12201 53 | // client-tls = false 54 | // client-tls-cert-chain-file = "/path/to/cert-chain.pem" 55 | // client-tls-verify-cert = true 56 | // client-queue-size = 512 57 | // client-connect-timeout = 5000 58 | // client-reconnect-delay = 1000 59 | // client-tcp-no-delay = true 60 | // client-send-buffer-size = 32768 61 | // } 62 | // 63 | // // GELF output to send messages to a Graylog server using UDP. 64 | // // NOTE: GELF via UDP *does not* support TLS, so the connection will be unencrypted! 65 | // gelf-udp { 66 | // type = "gelf" 67 | // protocol = "UDP" 68 | // host = "127.0.0.1" 69 | // port = 12201 70 | // client-queue-size = 512 71 | // client-send-buffer-size = 32768 72 | // } 73 | // 74 | // // Prints all messages to STDOUT. Useful for debugging. Do not enable in production usage! 75 | // console { 76 | // type = "stdout" 77 | // } 78 | //} 79 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/metrics/MetricService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.metrics; 18 | 19 | import com.codahale.metrics.MetricRegistry; 20 | import com.codahale.metrics.Reporter; 21 | import com.codahale.metrics.ScheduledReporter; 22 | import com.codahale.metrics.Slf4jReporter; 23 | import com.google.common.collect.Maps; 24 | import com.google.common.util.concurrent.AbstractIdleService; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | import javax.inject.Inject; 29 | import java.util.concurrent.ConcurrentMap; 30 | import java.util.concurrent.TimeUnit; 31 | 32 | public class MetricService extends AbstractIdleService { 33 | private static final Logger LOG = LoggerFactory.getLogger(MetricService.class); 34 | 35 | private final MetricServiceConfiguration configuration; 36 | private final MetricRegistry metricRegistry; 37 | 38 | private final ConcurrentMap, ScheduledReporter> reporter = Maps.newConcurrentMap(); 39 | 40 | @Inject 41 | public MetricService(MetricServiceConfiguration configuration, MetricRegistry metricRegistry) { 42 | this.configuration = configuration; 43 | this.metricRegistry = metricRegistry; 44 | } 45 | 46 | @Override 47 | protected void startUp() throws Exception { 48 | if (configuration.isEnableLog()) { 49 | startLoggingReporter(); 50 | } 51 | } 52 | 53 | @Override 54 | protected void shutDown() throws Exception { 55 | for (ScheduledReporter entry : reporter.values()) { 56 | LOG.debug("Stopping metrics reporter: {}", entry); 57 | entry.stop(); 58 | } 59 | } 60 | 61 | private void startLoggingReporter() { 62 | final Slf4jReporter loggingReporter = Slf4jReporter.forRegistry(metricRegistry) 63 | .convertDurationsTo(TimeUnit.MILLISECONDS) 64 | .convertRatesTo(TimeUnit.SECONDS) 65 | .outputTo(LOG) 66 | .build(); 67 | 68 | // Start the logging reporter if there is none yet. 69 | if (reporter.putIfAbsent(Slf4jReporter.class, loggingReporter) == null) { 70 | LOG.debug("Starting metrics reporter: {}", loggingReporter); 71 | reporter.get(Slf4jReporter.class).start(configuration.getReportDuration().getMillis(), TimeUnit.MILLISECONDS); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/file/naming/NumberSuffixStrategy.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file.naming; 18 | 19 | import com.google.common.base.Function; 20 | import com.google.common.base.Predicate; 21 | import com.google.common.collect.ImmutableSet; 22 | import com.google.common.collect.Iterables; 23 | 24 | import javax.annotation.Nullable; 25 | import java.nio.file.Path; 26 | import java.util.Set; 27 | 28 | public class NumberSuffixStrategy implements FileNamingStrategy { 29 | 30 | private final Iterable basePaths; 31 | 32 | public NumberSuffixStrategy(Path basePath) { 33 | this(ImmutableSet.of(basePath)); 34 | } 35 | 36 | public NumberSuffixStrategy(Set basePaths) { 37 | this.basePaths = Iterables.transform(basePaths, new Function() { 38 | @Nullable 39 | @Override 40 | public Path apply(Path path) { 41 | return path.normalize().toAbsolutePath(); 42 | } 43 | }); 44 | } 45 | 46 | @Override 47 | public boolean pathMatches(final Path path) { 48 | return Iterables.any(basePaths, new Predicate() { 49 | @Override 50 | public boolean apply(@Nullable Path basePath) { 51 | if (basePath == null) { 52 | return false; 53 | } 54 | 55 | Path normalizedPath = path.normalize(); 56 | normalizedPath = basePath.getParent().resolve(normalizedPath); 57 | // only allow files in the same directory 58 | if (!basePath.getParent().equals(normalizedPath.getParent())) { 59 | return false; 60 | } 61 | final String filename = normalizedPath.getFileName().toString(); 62 | final String baseFilename = basePath.getFileName().toString(); 63 | 64 | // same files are a match 65 | if (filename.equals(baseFilename)) { 66 | return true; 67 | } 68 | 69 | // do the files have a common beginning? if not, they aren't related. 70 | if (!filename.startsWith(baseFilename)) { 71 | return false; 72 | } 73 | 74 | // check for number suffix 75 | final String onlySuffix = filename.substring(baseFilename.length()); 76 | return onlySuffix.matches("^\\.\\d+$"); 77 | } 78 | }); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/Message.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector; 18 | 19 | import com.google.common.base.Joiner; 20 | import org.joda.time.DateTime; 21 | 22 | import java.util.Set; 23 | 24 | public class Message { 25 | public enum Level { 26 | EMERGENCY, ALERT, CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG 27 | } 28 | 29 | private final String message; 30 | private final String source; 31 | private final DateTime timestamp; 32 | private final String input; 33 | private final Set outputs; 34 | private final MessageFields fields; 35 | private final Level level; 36 | 37 | public Message(String message, String source, DateTime timestamp, Level level, String input, Set outputs) { 38 | this(message, source, timestamp, level, input, outputs, new MessageFields()); 39 | } 40 | 41 | public Message(String message, String source, DateTime timestamp, Level level, String input, Set outputs, MessageFields fields) { 42 | this.source = source; 43 | this.message = message; 44 | this.timestamp = timestamp; 45 | this.level = level; 46 | this.input = input; 47 | this.outputs = outputs; 48 | this.fields = fields; 49 | } 50 | 51 | public DateTime getTimestamp() { 52 | return timestamp; 53 | } 54 | 55 | public String getMessage() { 56 | return message; 57 | } 58 | 59 | public String getSource() { 60 | return source; 61 | } 62 | 63 | public Level getLevel() { 64 | return level; 65 | } 66 | 67 | public String getInput() { 68 | return input; 69 | } 70 | 71 | public Set getOutputs() { 72 | return outputs; 73 | } 74 | 75 | public MessageFields getFields() { 76 | return fields; 77 | } 78 | 79 | @Override 80 | public String toString() { 81 | final StringBuffer sb = new StringBuffer("Message{"); 82 | sb.append("timestamp=").append(getTimestamp()); 83 | sb.append(", level='").append(getLevel()).append('\''); 84 | sb.append(", message='").append(getMessage()).append('\''); 85 | sb.append(", source='").append(getSource()).append('\''); 86 | sb.append(", input='").append(getInput()).append('\''); 87 | sb.append(", outputs=").append(Joiner.on(",").join(getOutputs())); 88 | sb.append(", fields=").append(getFields().asMap()); 89 | sb.append('}'); 90 | return sb.toString(); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/file/SinglePathSetTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file; 18 | 19 | import com.google.common.collect.ImmutableSet; 20 | import com.google.common.jimfs.Configuration; 21 | import com.google.common.jimfs.Jimfs; 22 | import org.junit.Before; 23 | import org.junit.Test; 24 | import org.mockito.MockitoAnnotations; 25 | 26 | import java.nio.file.FileSystem; 27 | import java.nio.file.Files; 28 | import java.nio.file.Path; 29 | 30 | import static org.junit.Assert.assertEquals; 31 | import static org.junit.Assert.assertTrue; 32 | 33 | public class SinglePathSetTest { 34 | @Before 35 | public void setUp() throws Exception { 36 | MockitoAnnotations.initMocks(this); 37 | } 38 | 39 | private FileSystem newUnixFileSystem() { 40 | return Jimfs.newFileSystem(Configuration.unix()); 41 | } 42 | 43 | private FileSystem newWindowsFileSystem() { 44 | return Jimfs.newFileSystem(Configuration.windows()); 45 | } 46 | 47 | @Test 48 | public void testUnix() throws Exception { 49 | final FileSystem fileSystem = newUnixFileSystem(); 50 | final Path path = fileSystem.getPath("/var/log/syslog"); 51 | final PathSet pathSet = new SinglePathSet(path.toString(), fileSystem); 52 | 53 | assertEquals(path.getParent(), pathSet.getRootPath()); 54 | assertTrue(pathSet.isInSet(path)); 55 | assertTrue("Path list should be empty without any files in the file system", 56 | pathSet.getPaths().isEmpty()); 57 | 58 | Files.createDirectories(path.getParent()); 59 | Files.createFile(path); 60 | 61 | assertEquals("Path list should not be empty after creating the file", 62 | ImmutableSet.of(path), pathSet.getPaths()); 63 | } 64 | 65 | @Test 66 | public void testWindows() throws Exception { 67 | final FileSystem fileSystem = newWindowsFileSystem(); 68 | final Path path = fileSystem.getPath("C:\\logs\\application.log"); 69 | final PathSet pathSet = new SinglePathSet(path.toString(), fileSystem); 70 | 71 | assertEquals(path.getParent(), pathSet.getRootPath()); 72 | assertTrue(pathSet.isInSet(path)); 73 | assertTrue("Path list should be empty without any files in the file system", 74 | pathSet.getPaths().isEmpty()); 75 | 76 | Files.createDirectories(path.getParent()); 77 | Files.createFile(path); 78 | 79 | assertEquals("Path list should not be empty after creating the file", 80 | ImmutableSet.of(path), pathSet.getPaths()); 81 | } 82 | } -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/inputs/eventlog/WindowsEventlogInputConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.inputs.eventlog; 18 | 19 | import com.google.inject.assistedinject.Assisted; 20 | import com.typesafe.config.Config; 21 | import org.graylog.collector.config.ConfigurationUtils; 22 | import org.graylog.collector.inputs.InputConfiguration; 23 | import org.graylog.collector.inputs.InputService; 24 | 25 | import javax.inject.Inject; 26 | import javax.validation.constraints.NotNull; 27 | import java.util.Collections; 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | import java.util.concurrent.TimeUnit; 31 | 32 | public class WindowsEventlogInputConfiguration extends InputConfiguration { 33 | 34 | public interface Factory extends InputConfiguration.Factory { 35 | @Override 36 | WindowsEventlogInputConfiguration create(String id, Config config); 37 | } 38 | 39 | @NotNull 40 | private final String sourceName; 41 | 42 | private final long pollInterval; 43 | 44 | private WindowsEventlogInput.Factory inputFactory; 45 | 46 | @Inject 47 | public WindowsEventlogInputConfiguration(@Assisted String id, 48 | @Assisted Config config, 49 | WindowsEventlogInput.Factory inputFactory) { 50 | super(id, config); 51 | this.inputFactory = inputFactory; 52 | 53 | if (config.hasPath("source-name")) { 54 | this.sourceName = config.getString("source-name"); 55 | } else { 56 | this.sourceName = "Application"; 57 | } 58 | 59 | if (config.hasPath("poll-interval")) { 60 | this.pollInterval = config.getDuration("poll-interval", TimeUnit.MILLISECONDS); 61 | } else { 62 | this.pollInterval = 1000L; 63 | } 64 | } 65 | 66 | public String getSourceName() { 67 | return sourceName; 68 | } 69 | 70 | public long getPollInterval() { 71 | return pollInterval; 72 | } 73 | 74 | @Override 75 | public InputService createInput() { 76 | return inputFactory.create(this); 77 | } 78 | 79 | @Override 80 | public Map toStringValues() { 81 | return Collections.unmodifiableMap(new HashMap(super.toStringValues()) { 82 | { 83 | put("sourceName", getSourceName()); 84 | put("pollInterval", String.valueOf(getPollInterval())); 85 | } 86 | }); 87 | } 88 | 89 | @Override 90 | public String toString() { 91 | return ConfigurationUtils.toString(this); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/inputs/eventlog/WindowsEventlogInput.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.inputs.eventlog; 18 | 19 | import com.google.inject.assistedinject.Assisted; 20 | import org.graylog.collector.MessageBuilder; 21 | import org.graylog.collector.annotations.CollectorHostName; 22 | import org.graylog.collector.buffer.Buffer; 23 | import org.graylog.collector.config.ConfigurationUtils; 24 | import org.graylog.collector.file.ChunkReader; 25 | import org.graylog.collector.inputs.InputService; 26 | import org.hyperic.sigar.win32.EventLogThread; 27 | 28 | import javax.inject.Inject; 29 | import java.util.Set; 30 | 31 | public class WindowsEventlogInput extends InputService { 32 | private final WindowsEventlogInputConfiguration configuration; 33 | private final Buffer buffer; 34 | private final EventLogThread logThread; 35 | private final String collectorHostName; 36 | 37 | public interface Factory extends InputService.Factory { 38 | WindowsEventlogInput create(WindowsEventlogInputConfiguration configuration); 39 | } 40 | 41 | @Inject 42 | public WindowsEventlogInput(@Assisted WindowsEventlogInputConfiguration configuration, Buffer buffer, 43 | @CollectorHostName String collectorHostName) { 44 | this.configuration = configuration; 45 | this.buffer = buffer; 46 | this.logThread = EventLogThread.getInstance(configuration.getSourceName()); 47 | this.collectorHostName = collectorHostName; 48 | } 49 | 50 | @Override 51 | protected void doStart() { 52 | final MessageBuilder messageBuilder = new MessageBuilder() 53 | .input(getId()) 54 | .outputs(getOutputs()) 55 | .source(collectorHostName) 56 | .fields(configuration.getMessageFields()); 57 | 58 | logThread.add(new WindowsEventlogHandler(messageBuilder, buffer)); 59 | logThread.setInterval(configuration.getPollInterval()); 60 | logThread.doStart(); 61 | 62 | notifyStarted(); 63 | } 64 | 65 | @Override 66 | public void doStop() { 67 | logThread.doStop(); 68 | notifyStopped(); 69 | } 70 | 71 | @Override 72 | public String getId() { 73 | return configuration.getId(); 74 | } 75 | 76 | @Override 77 | public Set getOutputs() { 78 | return configuration.getOutputs(); 79 | } 80 | 81 | @Override 82 | public void setReaderFinished(ChunkReader chunkReader) { 83 | 84 | } 85 | 86 | @Override 87 | public String toString() { 88 | return ConfigurationUtils.toString(configuration, this); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/utils/CollectorId.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.utils; 18 | 19 | import com.google.common.base.Charsets; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import javax.inject.Inject; 24 | import javax.inject.Singleton; 25 | import java.io.File; 26 | import java.io.FileNotFoundException; 27 | import java.io.IOException; 28 | import java.nio.charset.StandardCharsets; 29 | import java.nio.file.Files; 30 | import java.nio.file.NoSuchFileException; 31 | import java.nio.file.Path; 32 | import java.nio.file.Paths; 33 | import java.util.List; 34 | import java.util.UUID; 35 | 36 | @Singleton 37 | public class CollectorId { 38 | private static final Logger LOG = LoggerFactory.getLogger(CollectorId.class); 39 | 40 | private final String id; 41 | 42 | @Inject 43 | public CollectorId(CollectorIdConfiguration config) { 44 | final String configuredCollectorId = config.getCollectorId(); 45 | if (configuredCollectorId.startsWith("file:")) { 46 | final String[] splittedConfig = configuredCollectorId.split("^file:"); 47 | if (splittedConfig.length < 2) { 48 | throw new RuntimeException("Invalid specified file location for collector id: " + configuredCollectorId); 49 | } 50 | this.id = readOrGenerate(splittedConfig[1]); 51 | } else { 52 | this.id = configuredCollectorId; 53 | } 54 | } 55 | 56 | private String readOrGenerate(String filename) { 57 | try { 58 | String read = read(filename); 59 | 60 | if (read == null || read.isEmpty()) { 61 | return generate(filename); 62 | } 63 | 64 | LOG.info("Collector ID: {}", read); 65 | return read; 66 | } catch (FileNotFoundException | NoSuchFileException e) { 67 | return generate(filename); 68 | } catch (IOException e) { 69 | throw new RuntimeException("Unable to read node id from file: ", e); 70 | } 71 | } 72 | 73 | private String read(String filename) throws IOException { 74 | final List lines = Files.readAllLines(Paths.get(filename), StandardCharsets.UTF_8); 75 | 76 | return lines.size() > 0 ? lines.get(0) : ""; 77 | } 78 | 79 | private String generate(String filename) { 80 | String generated = this.randomId(); 81 | LOG.info("No node ID file found. Generated: {}", generated); 82 | 83 | try { 84 | persist(generated, filename); 85 | } catch (IOException e1) { 86 | LOG.debug("Could not persist node ID: ", e1); 87 | throw new RuntimeException("Unable to persist node ID to " + filename, e1); 88 | } 89 | 90 | return generated; 91 | } 92 | 93 | private void persist(String nodeId, String filename) throws IOException { 94 | final Path path = new File(filename).getAbsoluteFile().toPath(); 95 | 96 | if (path.getParent() != null) { 97 | Files.createDirectories(path.getParent()); 98 | } 99 | 100 | Files.write(path, nodeId.getBytes(Charsets.UTF_8)); 101 | } 102 | 103 | /** 104 | * {@inheritDoc} 105 | */ 106 | @Override 107 | public String toString() { 108 | return id; 109 | } 110 | 111 | 112 | private String randomId() { 113 | return UUID.randomUUID().toString(); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/file/ChunkProcessor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file; 18 | 19 | import com.google.common.util.concurrent.AbstractExecutionThreadService; 20 | import io.netty.buffer.ByteBuf; 21 | import org.graylog.collector.Message; 22 | import org.graylog.collector.MessageBuilder; 23 | import org.graylog.collector.buffer.Buffer; 24 | import org.graylog.collector.file.splitters.ContentSplitter; 25 | import org.joda.time.DateTime; 26 | import org.joda.time.DateTimeZone; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import java.nio.charset.Charset; 31 | import java.nio.file.Path; 32 | import java.util.concurrent.BlockingQueue; 33 | import java.util.concurrent.TimeUnit; 34 | 35 | public class ChunkProcessor extends AbstractExecutionThreadService { 36 | 37 | private static final Logger log = LoggerFactory.getLogger(ChunkProcessor.class); 38 | 39 | private final Buffer buffer; 40 | private final MessageBuilder messageBuilder; 41 | private final BlockingQueue chunkQueue; 42 | 43 | private final ContentSplitter splitter; 44 | private final Charset charset; 45 | 46 | @Override 47 | protected void run() throws Exception { 48 | while (isRunning()) { 49 | final FileChunk chunk = chunkQueue.poll(500, TimeUnit.MILLISECONDS); 50 | if (chunk != null) { 51 | process(chunk); 52 | } 53 | } 54 | } 55 | 56 | private ChunkBufferStore chunkBufferStore = new ChunkBufferStore(); 57 | 58 | public ChunkProcessor(Buffer buffer, MessageBuilder messageBuilder, BlockingQueue chunkQueue, ContentSplitter splitter, final Charset charset) { 59 | this.buffer = buffer; 60 | this.messageBuilder = messageBuilder; 61 | this.chunkQueue = chunkQueue; 62 | this.splitter = splitter; 63 | this.charset = charset; 64 | } 65 | 66 | public void process(FileChunk chunk) { 67 | final Path path = chunk.getPath(); 68 | 69 | if (chunk.isFinalChunk()) { 70 | // we've reached the EOF and aren't in follow mode 71 | log.debug("[{}] Processing final chunk.", path); 72 | final ByteBuf channelBuffer = chunkBufferStore.get(path); 73 | final Iterable messages = splitter.splitRemaining(channelBuffer, charset); 74 | 75 | createMessages(path, messages); 76 | 77 | // nothing more to do. 78 | stopAsync(); 79 | return; 80 | } 81 | log.debug("[{}] Processing {} bytes chunk (pos {})", path, chunk.getChunkBuffer().readableBytes(), chunk.getId()); 82 | 83 | chunkBufferStore.put(path, chunk.getChunkBuffer()); 84 | 85 | createMessages(path, splitter.split(chunkBufferStore.get(path), charset, false)); 86 | } 87 | 88 | private void createMessages(Path path, Iterable messages) { 89 | for (String messageString : messages) { 90 | if (messageString.isEmpty()) { 91 | // skip completely empty messages, they contain no useful information 92 | continue; 93 | } 94 | 95 | final Message message = messageBuilder.copy() 96 | .message(messageString) 97 | .timestamp(DateTime.now(DateTimeZone.UTC)) 98 | .level(null) // Do not send a level in the GELF message for data read from files. 99 | .build(); 100 | 101 | message.getFields().put("source_file", path.toFile().getAbsolutePath()); 102 | 103 | buffer.insert(message); 104 | } 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /src/test/java/org/graylog/collector/file/Utils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.file; 18 | 19 | import java.io.BufferedOutputStream; 20 | import java.io.File; 21 | import java.io.FileOutputStream; 22 | import java.io.IOException; 23 | import java.nio.file.Files; 24 | import java.nio.file.Path; 25 | import java.util.Random; 26 | 27 | public class Utils { 28 | 29 | public static class LogFile { 30 | 31 | private final long initialSize; 32 | private final int maxLineLength; 33 | private final int lineLengthDeviation; 34 | 35 | private Path path; 36 | private BufferedOutputStream bos; 37 | private Random random; 38 | 39 | public LogFile(long initialSize, int maxLineLength, int lineLengthDeviation) throws IOException { 40 | this.initialSize = initialSize; 41 | this.maxLineLength = maxLineLength; 42 | this.lineLengthDeviation = lineLengthDeviation; 43 | 44 | random = new Random(); 45 | createInitialFile(); 46 | } 47 | 48 | private void createInitialFile() throws IOException { 49 | long size = initialSize; 50 | path = Files.createTempFile("gl2-", ".log"); 51 | final File file = path.toFile(); 52 | file.deleteOnExit(); 53 | bos = new BufferedOutputStream(new FileOutputStream(file)); 54 | 55 | while (size > 0) { 56 | final int actualLineLength = appendRandomLine(); 57 | size -= actualLineLength; // line and NL char 58 | } 59 | flush(); 60 | } 61 | 62 | public int appendRandomLine() throws IOException { 63 | final RandomString randomString = new RandomString(maxLineLength); 64 | final int actualLineLength = maxLineLength - random.nextInt(lineLengthDeviation); 65 | bos.write(randomString.nextString().substring(0, actualLineLength).getBytes()); 66 | bos.write("\n".getBytes()); 67 | return actualLineLength + 1; 68 | } 69 | 70 | public int appendLine(String line) throws IOException { 71 | bos.write(line.getBytes()); 72 | bos.write("\n".getBytes()); 73 | return line.length() + 1; // line and NL char 74 | } 75 | 76 | public void close() throws IOException { 77 | bos.close(); 78 | } 79 | 80 | public void flush() throws IOException { 81 | bos.flush(); 82 | } 83 | 84 | public Path getPath() { 85 | return path; 86 | } 87 | } 88 | 89 | 90 | // blatantly stolen from http://stackoverflow.com/questions/41107/how-to-generate-a-random-alpha-numeric-string 91 | public static class RandomString { 92 | private static final char[] symbols = new char[36]; 93 | 94 | static { 95 | for (int idx = 0; idx < 10; ++idx) 96 | symbols[idx] = (char) ('0' + idx); 97 | for (int idx = 10; idx < 36; ++idx) 98 | symbols[idx] = (char) ('a' + idx - 10); 99 | } 100 | 101 | private final Random random = new Random(); 102 | 103 | private final char[] buf; 104 | 105 | public RandomString(int length) { 106 | if (length < 1) 107 | throw new IllegalArgumentException("length < 1: " + length); 108 | buf = new char[length]; 109 | } 110 | 111 | public String nextString() { 112 | for (int idx = 0; idx < buf.length; ++idx) 113 | buf[idx] = symbols[random.nextInt(symbols.length)]; 114 | return new String(buf); 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/inputs/eventlog/WindowsEventlogHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.inputs.eventlog; 18 | 19 | import org.graylog.collector.Message; 20 | import org.graylog.collector.MessageBuilder; 21 | import org.graylog.collector.buffer.Buffer; 22 | import org.hyperic.sigar.win32.EventLogNotification; 23 | import org.hyperic.sigar.win32.EventLogRecord; 24 | import org.joda.time.DateTime; 25 | import org.joda.time.DateTimeZone; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import static com.google.common.base.Strings.isNullOrEmpty; 30 | 31 | public class WindowsEventlogHandler implements EventLogNotification { 32 | private static final Logger LOG = LoggerFactory.getLogger(WindowsEventlogHandler.class); 33 | 34 | private final MessageBuilder messageBuilder; 35 | private final Buffer buffer; 36 | 37 | public WindowsEventlogHandler(MessageBuilder messageBuilder, Buffer buffer) { 38 | this.messageBuilder = messageBuilder; 39 | this.buffer = buffer; 40 | } 41 | 42 | @Override 43 | public boolean matches(EventLogRecord eventLogRecord) { 44 | return true; 45 | } 46 | 47 | @Override 48 | public void handleNotification(EventLogRecord record) { 49 | LOG.debug("EventLogRecord: {}", record); 50 | 51 | final MessageBuilder builder = messageBuilder 52 | .copy() 53 | .message(isNullOrEmpty(record.getMessage()) ? "empty" : record.getMessage().trim()) 54 | .timestamp(getDateTime(record.getTimeGenerated())) 55 | .level(getMessageLevel(record)); 56 | 57 | builder.addField("event_source", record.getSource()); 58 | builder.addField("event_category", record.getCategory()); 59 | builder.addField("event_category_string", record.getCategoryString()); 60 | builder.addField("event_computer_name", record.getComputerName()); 61 | builder.addField("event_id", record.getEventId()); 62 | builder.addField("event_type", record.getEventType()); 63 | builder.addField("event_type_string", record.getEventTypeString()); 64 | builder.addField("event_log_name", record.getLogName()); 65 | builder.addField("event_record_number", record.getRecordNumber()); 66 | builder.addField("event_time_generated", getDateTime(record.getTimeGenerated()).toString()); 67 | builder.addField("event_time_written", getDateTime(record.getTimeWritten()).toString()); 68 | builder.addField("event_user", isNullOrEmpty(record.getUser()) ? "" : record.getUser()); 69 | 70 | buffer.insert(builder.build()); 71 | } 72 | 73 | private DateTime getDateTime(long seconds) { 74 | return new DateTime(seconds * 1000, DateTimeZone.UTC); 75 | } 76 | 77 | /** 78 | * Returns the @{code Message.Level} for the given {@code EventLogRecord}. 79 | * 80 | * See: https://msdn.microsoft.com/en-us/library/windows/desktop/aa363646(v=vs.85).aspx 81 | * 82 | * @param record the eventlog record 83 | * @return the mapped message level 84 | */ 85 | private Message.Level getMessageLevel(EventLogRecord record) { 86 | switch (record.getEventType()) { 87 | case 1: // EVENTLOG_ERROR_TYPE 88 | return Message.Level.ERROR; 89 | case 2: // EVENTLOG_WARNING_TYPE 90 | return Message.Level.WARNING; 91 | case 4: // EVENTLOG_INFORMATION_TYPE 92 | return Message.Level.INFO; 93 | case 8: // EVENTLOG_AUDIT_SUCCESS 94 | return Message.Level.INFO; 95 | case 16: // EVENTLOG_AUDIT_FAILURE 96 | return Message.Level.ERROR; 97 | default: 98 | return Message.Level.INFO; 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/heartbeat/HeartbeatService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.heartbeat; 18 | 19 | import com.google.common.util.concurrent.AbstractScheduledService; 20 | import com.typesafe.config.Config; 21 | import org.graylog.collector.utils.CollectorId; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | import retrofit.RetrofitError; 25 | import retrofit.client.Response; 26 | 27 | import javax.inject.Inject; 28 | import java.util.concurrent.TimeUnit; 29 | 30 | public class HeartbeatService extends AbstractScheduledService { 31 | private static final Logger LOG = LoggerFactory.getLogger(HeartbeatService.class); 32 | private static final String heartbeatIntervalParameter = "heartbeat-interval"; 33 | private static final int defaultHeartbeatInterval = 5; 34 | private static final String enableRegistrationParameter = "enable-registration"; 35 | 36 | private final CollectorRegistrationService collectorRegistrationService; 37 | private final CollectorRegistrationRequest collectorRegistrationRequest; 38 | private final Config config; 39 | private final String collectorId; 40 | 41 | @Inject 42 | public HeartbeatService(CollectorRegistrationService collectorRegistrationService, 43 | CollectorRegistrationRequest collectorRegistrationRequest, 44 | Config config, 45 | CollectorId collectorId) { 46 | this.collectorRegistrationService = collectorRegistrationService; 47 | this.collectorRegistrationRequest = collectorRegistrationRequest; 48 | this.config = config; 49 | this.collectorId = collectorId.toString(); 50 | } 51 | 52 | @Override 53 | protected void runOneIteration() throws Exception { 54 | if (config.hasPath(enableRegistrationParameter) && !config.getBoolean(enableRegistrationParameter)) { 55 | return; 56 | } 57 | register(false); 58 | } 59 | 60 | private void register(final boolean legacy) { 61 | try { 62 | if (legacy) { 63 | collectorRegistrationService.legacyRegister(this.collectorId, this.collectorRegistrationRequest); 64 | } else { 65 | collectorRegistrationService.register(this.collectorId, this.collectorRegistrationRequest); 66 | } 67 | } catch (RetrofitError e) { 68 | final Response response = e.getResponse(); 69 | if (response != null) { 70 | if (!legacy && response.getStatus() == 404) { 71 | // Try again with the Graylog 1.x URL if we didn't try yet. 72 | register(true); 73 | } else { 74 | LOG.warn("Unable to send heartbeat to Graylog server, result was: {} - {}", response.getStatus(), response.getReason()); 75 | } 76 | } else { 77 | final String message; 78 | if (e.getCause() != null) { 79 | message = e.getCause().getClass().getSimpleName() + ": " + e.getCause().getMessage(); 80 | } else { 81 | message = e.getClass().getSimpleName() + ": " + e.getMessage(); 82 | } 83 | LOG.warn("Unable to send heartbeat to Graylog server: {}", message); 84 | } 85 | } 86 | } 87 | 88 | @Override 89 | protected Scheduler scheduler() { 90 | return Scheduler.newFixedRateSchedule(0, heartbeatInterval(config), TimeUnit.SECONDS); 91 | } 92 | 93 | private int heartbeatInterval(Config config) { 94 | if (config.hasPath(heartbeatIntervalParameter)) { 95 | return config.getInt(heartbeatIntervalParameter); 96 | } else { 97 | return defaultHeartbeatInterval; 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/inputs/file/FileInput.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.inputs.file; 18 | 19 | import com.google.inject.assistedinject.Assisted; 20 | import org.graylog.collector.MessageBuilder; 21 | import org.graylog.collector.annotations.CollectorHostName; 22 | import org.graylog.collector.buffer.Buffer; 23 | import org.graylog.collector.config.ConfigurationUtils; 24 | import org.graylog.collector.file.ChunkReader; 25 | import org.graylog.collector.file.FileObserver; 26 | import org.graylog.collector.file.FileReaderService; 27 | import org.graylog.collector.file.PathSet; 28 | import org.graylog.collector.inputs.InputService; 29 | 30 | import javax.inject.Inject; 31 | import java.util.Set; 32 | 33 | public class FileInput extends InputService { 34 | public enum InitialReadPosition { 35 | START, 36 | END 37 | } 38 | 39 | public interface Factory extends InputService.Factory { 40 | FileInput create(FileInputConfiguration configuration); 41 | } 42 | 43 | private final FileInputConfiguration configuration; 44 | private final Buffer buffer; 45 | private final FileObserver fileObserver; 46 | private final String collectorHostName; 47 | private FileReaderService readerService; 48 | 49 | @Inject 50 | public FileInput(@Assisted FileInputConfiguration inputConfiguration, Buffer buffer, FileObserver fileObserver, 51 | @CollectorHostName String collectorHostName) { 52 | this.configuration = inputConfiguration; 53 | this.buffer = buffer; 54 | this.fileObserver = fileObserver; 55 | this.collectorHostName = collectorHostName; 56 | } 57 | 58 | @Override 59 | public String getId() { 60 | return configuration.getId(); 61 | } 62 | 63 | @Override 64 | public Set getOutputs() { 65 | return configuration.getOutputs(); 66 | } 67 | 68 | @Override 69 | protected void doStart() { 70 | // TODO needs to be an absolute path because otherwise the FileObserver does weird things. Investigate what's wrong with it. 71 | final MessageBuilder messageBuilder = new MessageBuilder() 72 | .input(getId()) 73 | .outputs(getOutputs()) 74 | .source(collectorHostName) 75 | .fields(configuration.getMessageFields()); 76 | final PathSet pathSet = configuration.getPathSet(); 77 | readerService = new FileReaderService( 78 | pathSet, 79 | configuration.getCharset(), 80 | InitialReadPosition.END, 81 | this, 82 | messageBuilder, 83 | configuration.createContentSplitter(), 84 | buffer, 85 | configuration.getReaderBufferSize(), 86 | configuration.getReaderInterval(), 87 | fileObserver 88 | ); 89 | 90 | readerService.startAsync().awaitRunning(); 91 | notifyStarted(); 92 | } 93 | 94 | @Override 95 | protected void doStop() { 96 | readerService.stopAsync().awaitTerminated(); 97 | notifyStopped(); 98 | } 99 | 100 | @Override 101 | public void setReaderFinished(ChunkReader chunkReader) { 102 | // TODO Check if needed and for what it was used. 103 | } 104 | 105 | @Override 106 | public boolean equals(Object o) { 107 | if (this == o) return true; 108 | if (o == null || getClass() != o.getClass()) return false; 109 | 110 | FileInput fileInput = (FileInput) o; 111 | 112 | return configuration.equals(fileInput.configuration); 113 | 114 | } 115 | 116 | @Override 117 | public int hashCode() { 118 | return configuration.hashCode(); 119 | } 120 | 121 | @Override 122 | public String toString() { 123 | return ConfigurationUtils.toString(configuration, this); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/inputs/InputConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.inputs; 18 | 19 | import com.google.common.base.Joiner; 20 | import com.google.common.base.Splitter; 21 | import com.google.common.collect.Sets; 22 | import com.typesafe.config.Config; 23 | import com.typesafe.config.ConfigValue; 24 | import org.graylog.collector.MessageFields; 25 | import org.graylog.collector.config.Configuration; 26 | import org.hibernate.validator.constraints.NotBlank; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | import javax.validation.constraints.NotNull; 31 | import java.util.Collections; 32 | import java.util.HashMap; 33 | import java.util.Map; 34 | import java.util.Set; 35 | 36 | public abstract class InputConfiguration implements Configuration { 37 | private static final Logger log = LoggerFactory.getLogger(InputConfiguration.class); 38 | 39 | public interface Factory { 40 | C create(String id, Config config); 41 | } 42 | 43 | @NotBlank 44 | private final String id; 45 | 46 | @NotNull 47 | private Set outputs = Sets.newHashSet(); 48 | 49 | private final MessageFields messageFields = new MessageFields(); 50 | 51 | public InputConfiguration(String id, Config config) { 52 | this.id = id; 53 | 54 | if (config.hasPath("outputs")) { 55 | this.outputs = Sets.newHashSet(Splitter.on(",").omitEmptyStrings().trimResults().split(config.getString("outputs"))); 56 | } 57 | 58 | if (config.hasPath("message-fields")) { 59 | final Config messageFieldsConfig = config.getConfig("message-fields"); 60 | 61 | for (Map.Entry entry : messageFieldsConfig.entrySet()) { 62 | final String key = entry.getKey(); 63 | final ConfigValue value = entry.getValue(); 64 | 65 | switch (value.valueType()) { 66 | case NUMBER: 67 | this.messageFields.put(key, messageFieldsConfig.getNumber(key)); 68 | break; 69 | case BOOLEAN: 70 | this.messageFields.put(key, messageFieldsConfig.getBoolean(key)); 71 | break; 72 | case STRING: 73 | this.messageFields.put(key, messageFieldsConfig.getString(key)); 74 | break; 75 | default: 76 | log.warn("{}[{}] Message field value of type \"{}\" is not supported for key \"{}\" (value: {})", 77 | getClass().getSimpleName(), getId(), value.valueType(), key, value.toString()); 78 | break; 79 | } 80 | } 81 | } 82 | } 83 | 84 | public abstract InputService createInput(); 85 | 86 | @Override 87 | public String getId() { 88 | return id; 89 | } 90 | 91 | public Set getOutputs() { 92 | return outputs; 93 | } 94 | 95 | public MessageFields getMessageFields() { 96 | return messageFields; 97 | } 98 | 99 | @Override 100 | public Map toStringValues() { 101 | return Collections.unmodifiableMap(new HashMap() { 102 | { 103 | put("id", getId()); 104 | put("outputs", Joiner.on(",").join(getOutputs())); 105 | put("message-fields", getMessageFields().toString()); 106 | } 107 | }); 108 | } 109 | 110 | @Override 111 | public boolean equals(Object o) { 112 | if (this == o) return true; 113 | if (o == null || getClass() != o.getClass()) return false; 114 | 115 | InputConfiguration that = (InputConfiguration) o; 116 | 117 | return id.equals(that.id); 118 | 119 | } 120 | 121 | @Override 122 | public int hashCode() { 123 | return id.hashCode(); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/guice/CollectorModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.guice; 18 | 19 | import com.google.common.util.concurrent.Service; 20 | import com.google.inject.AbstractModule; 21 | import com.google.inject.TypeLiteral; 22 | import com.google.inject.assistedinject.FactoryModuleBuilder; 23 | import com.google.inject.multibindings.MapBinder; 24 | import com.google.inject.multibindings.Multibinder; 25 | import org.graylog.collector.buffer.BufferConsumer; 26 | import org.graylog.collector.config.Configuration; 27 | import org.graylog.collector.inputs.Input; 28 | import org.graylog.collector.inputs.InputConfiguration; 29 | import org.graylog.collector.outputs.Output; 30 | import org.graylog.collector.outputs.OutputConfiguration; 31 | 32 | public abstract class CollectorModule extends AbstractModule { 33 | private MapBinder> inputsMapBinder = null; 34 | private MapBinder> outputsMapBinder = null; 35 | 36 | private Multibinder services = null; 37 | private Multibinder bufferConsumers = null; 38 | 39 | public void registerService(Class serviceClass) { 40 | if (services == null) { 41 | services = Multibinder.newSetBinder(binder(), Service.class); 42 | } 43 | services.addBinding().to(serviceClass); 44 | } 45 | 46 | public void registerBufferConsumer(Class consumerClass) { 47 | if (bufferConsumers == null) { 48 | bufferConsumers = Multibinder.newSetBinder(binder(), BufferConsumer.class); 49 | } 50 | bufferConsumers.addBinding().to(consumerClass); 51 | } 52 | 53 | public void registerInput(String type, 54 | Class inputClass, 55 | Class> inputFactoryClass, 56 | Class inputConfigurationClass, 57 | Class> inputConfigurationFactoryClass) { 58 | if (inputsMapBinder == null) { 59 | this.inputsMapBinder = MapBinder.newMapBinder(binder(), 60 | TypeLiteral.get(String.class), 61 | new TypeLiteral>() { 62 | }); 63 | } 64 | 65 | install(new FactoryModuleBuilder().implement(Input.class, inputClass).build(inputFactoryClass)); 66 | install(new FactoryModuleBuilder().implement(Configuration.class, inputConfigurationClass).build(inputConfigurationFactoryClass)); 67 | 68 | inputsMapBinder.addBinding(type).to(inputConfigurationFactoryClass); 69 | } 70 | 71 | public void registerOutput(String type, 72 | Class outputClass, 73 | Class> outputFactoryClass, 74 | Class outputConfigurationClass, 75 | Class> outputConfigurationFactoryClass) { 76 | 77 | if (outputsMapBinder == null) { 78 | this.outputsMapBinder = MapBinder.newMapBinder(binder(), 79 | TypeLiteral.get(String.class), 80 | new TypeLiteral>() { 81 | }); 82 | } 83 | 84 | install(new FactoryModuleBuilder().implement(Output.class, outputClass).build(outputFactoryClass)); 85 | install(new FactoryModuleBuilder().implement(Configuration.class, outputConfigurationClass).build(outputConfigurationFactoryClass)); 86 | 87 | outputsMapBinder.addBinding(type).to(outputConfigurationFactoryClass); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/org/graylog/collector/inputs/file/FileInputConfigurationValidator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog. 3 | * 4 | * Graylog is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog. If not, see . 16 | */ 17 | package org.graylog.collector.inputs.file; 18 | 19 | import org.graylog.collector.file.PathSet; 20 | import org.hibernate.validator.constraintvalidation.HibernateConstraintValidatorContext; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import javax.validation.ConstraintValidator; 25 | import javax.validation.ConstraintValidatorContext; 26 | import java.nio.charset.Charset; 27 | import java.nio.charset.UnsupportedCharsetException; 28 | import java.util.regex.Pattern; 29 | import java.util.regex.PatternSyntaxException; 30 | 31 | public class FileInputConfigurationValidator implements ConstraintValidator { 32 | private static final Logger LOG = LoggerFactory.getLogger(FileInputConfigurationValidator.class); 33 | 34 | @Override 35 | public void initialize(ValidFileInputConfiguration constraintAnnotation) { 36 | 37 | } 38 | 39 | @Override 40 | public boolean isValid(FileInputConfiguration config, ConstraintValidatorContext context) { 41 | showWarnings(config); 42 | 43 | try { 44 | Charset.forName(config.getCharsetString()); 45 | } catch (UnsupportedCharsetException e) { 46 | setMessageTemplate(context, "{org.graylog.collector.inputs.file.ValidFileInputConfiguration.unsupportedCharset.message}", config.getCharsetString()); 47 | return false; 48 | } catch (IllegalArgumentException e) { 49 | setMessageTemplate(context, "{org.graylog.collector.inputs.file.ValidFileInputConfiguration.illegalCharset.message}", config.getCharsetString()); 50 | return false; 51 | } 52 | 53 | if (config.getReaderBufferSize() < 1) { 54 | setMessageTemplate(context, "{org.graylog.collector.inputs.file.ValidFileInputConfiguration.readerBufferSizeTooSmall.message}", String.valueOf(config.getReaderBufferSize())); 55 | return false; 56 | } 57 | if (config.getReaderInterval() < 1) { 58 | setMessageTemplate(context, "{org.graylog.collector.inputs.file.ValidFileInputConfiguration.readerIntervalTooSmall.message}", String.valueOf(config.getReaderInterval())); 59 | return false; 60 | } 61 | 62 | switch (config.getContentSplitter()) { 63 | case "PATTERN": 64 | if (config.getContentSplitterPattern() != null && !config.getContentSplitterPattern().isEmpty()) { 65 | try { 66 | Pattern.compile(config.getContentSplitterPattern(), Pattern.MULTILINE); 67 | return true; 68 | } catch (PatternSyntaxException ignored) { 69 | setMessageTemplate(context, "{org.graylog.collector.inputs.file.ValidFileInputConfiguration.invalidPattern.message}", config.getContentSplitterPattern()); 70 | return false; 71 | } 72 | } 73 | setMessageTemplate(context, "{org.graylog.collector.inputs.file.ValidFileInputConfiguration.missingPattern.message}", null); 74 | return false; 75 | default: 76 | return true; 77 | } 78 | } 79 | 80 | private void showWarnings(FileInputConfiguration config) { 81 | final PathSet pathSet = config.getPathSet(); 82 | 83 | if (pathSet == null) { 84 | return; 85 | } 86 | 87 | if (!pathSet.getRootPath().toFile().canRead()) { 88 | LOG.warn("Configured directory {} does not exist or is not accessible.", pathSet.getRootPath()); 89 | } 90 | } 91 | 92 | private void setMessageTemplate(ConstraintValidatorContext context, String messageTemplate, String value) { 93 | HibernateConstraintValidatorContext hibernateContext = context.unwrap(HibernateConstraintValidatorContext.class); 94 | hibernateContext.disableDefaultConstraintViolation(); 95 | hibernateContext.addExpressionVariable("value", value).buildConstraintViolationWithTemplate(messageTemplate).addConstraintViolation(); 96 | } 97 | } 98 | --------------------------------------------------------------------------------