├── .gitignore ├── README.md ├── build.gradle ├── docker-compose.yml ├── fluent-es ├── Dockerfile ├── fluent.conf └── plugins │ └── .gitkeep ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── src └── main ├── groovy └── fluentd │ ├── FluentdBootApplication.groovy │ └── MainRunner.groovy └── resources ├── application.properties └── logback.xml /.gitignore: -------------------------------------------------------------------------------- 1 | # IntelliJ project files 2 | .idea 3 | *.iml 4 | 5 | # gradle 6 | .gradle 7 | build 8 | 9 | fluent-data/ 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fluentd-boot 2 | 3 | Spring boot logs redirected to Elastic Search via fluentd. 4 | 5 | The necessary configuration for Elastic Search is in the docker-compose file. 6 | 7 | ## Usage 8 | 9 | Run fluentd + elastic search + kibana 10 | 11 | ``` 12 | docker-compose up -d 13 | ``` 14 | 15 | Then go to `http://${DOCKER_HOST}:5601` (ie, `http://localhost:5601` or your docker machine ip) to see the Kibana dashboard. 16 | 17 | Run the application with your IDE. If neither `FLUENTD_HOST` nor `DOCKER_HOST` variable are set, 18 | the logger will try to connect to fluentd on `localhost`. 19 | 20 | You can customize the `FLUENTD_HOST` and `FLUENTD_PORT` environment variables to point to your docker-machine IP. 21 | 22 | For instance, on my mac, I run the application with `FLUENTD_HOST=192.168.99.100`. 23 | 24 | ## Principle 25 | 26 | Uses a logback appender to redirect logs to fluentd. 27 | 28 | See `src/main/resources/logback.xml` for more information. -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | ext { 3 | springBootVersion = '1.3.1.RELEASE' 4 | } 5 | repositories { 6 | mavenCentral() 7 | } 8 | dependencies { 9 | classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") 10 | } 11 | } 12 | 13 | apply plugin: 'groovy' 14 | apply plugin: 'eclipse' 15 | apply plugin: 'idea' 16 | apply plugin: 'spring-boot' 17 | 18 | jar { 19 | baseName = 'fluentd-boot' 20 | version = '0.0.1-SNAPSHOT' 21 | } 22 | sourceCompatibility = 1.8 23 | targetCompatibility = 1.8 24 | 25 | repositories { 26 | mavenCentral() 27 | maven { url 'http://sndyuk.github.com/maven' } 28 | } 29 | 30 | 31 | dependencies { 32 | compile('org.springframework.boot:spring-boot-devtools') 33 | compile('org.springframework.retry:spring-retry') 34 | compile('org.springframework.boot:spring-boot-starter') 35 | compile('org.codehaus.groovy:groovy') 36 | compile 'org.fluentd:fluent-logger:0.3.2' 37 | compile 'com.sndyuk:logback-more-appenders:1.1.1' 38 | compile 'com.github.groovy-wslite:groovy-wslite:1.1.2' 39 | testCompile('org.springframework.boot:spring-boot-starter-test') 40 | } 41 | 42 | 43 | eclipse { 44 | classpath { 45 | containers.remove('org.eclipse.jdt.launching.JRE_CONTAINER') 46 | containers 'org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8' 47 | } 48 | } 49 | 50 | task wrapper(type: Wrapper) { 51 | gradleVersion = '2.9' 52 | } 53 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | es: 2 | image: elasticsearch:2 3 | # does not work on mac unless dir owner is elasticsearch 4 | # volumes: 5 | # - ${PWD}/es-data:/usr/share/elasticsearch/data 6 | # The following will store es data in your boot2docker vm 7 | # Use 'docker-machine ssh default' to remove it 8 | volumes: 9 | - /srv/docker/es:/usr/share/elasticsearch/data 10 | ports: 11 | - 9200:9200 12 | - 9300:9300 13 | 14 | kibana: 15 | image: kibana 16 | ports: 17 | - 5601:5601 18 | links: 19 | - es:elasticsearch 20 | 21 | fluentd: 22 | build: fluent-es/ 23 | ports: 24 | - 24224:24224 25 | links: 26 | - es:es 27 | -------------------------------------------------------------------------------- /fluent-es/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM fluent/fluentd:latest 2 | 3 | USER fluent 4 | WORKDIR /home/fluent 5 | ENV PATH /home/fluent/.gem/ruby/2.2.0/bin:$PATH 6 | RUN gem install fluent-plugin-elasticsearch 7 | 8 | EXPOSE 24284 9 | VOLUME /fluentd/log 10 | 11 | CMD fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT 12 | -------------------------------------------------------------------------------- /fluent-es/fluent.conf: -------------------------------------------------------------------------------- 1 | 2 | type forward 3 | port 24224 4 | bind 0.0.0.0 5 | 6 | 7 | 8 | 9 | type elasticsearch 10 | logstash_format true 11 | host "#{ENV['ES_PORT_9200_TCP_ADDR']}" # dynamically configured to use Docker's link feature 12 | port 9200 13 | flush_interval 5s 14 | 15 | -------------------------------------------------------------------------------- /fluent-es/plugins/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geowarin/fluentd-boot/ed84fca388d81ba98bf93ec6c2a219ea4831dbe6/fluent-es/plugins/.gitkeep -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geowarin/fluentd-boot/ed84fca388d81ba98bf93ec6c2a219ea4831dbe6/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Sun Jan 10 21:34:56 CET 2016 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-2.9-all.zip 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # Attempt to set APP_HOME 46 | # Resolve links: $0 may be a link 47 | PRG="$0" 48 | # Need this for relative symlinks. 49 | while [ -h "$PRG" ] ; do 50 | ls=`ls -ld "$PRG"` 51 | link=`expr "$ls" : '.*-> \(.*\)$'` 52 | if expr "$link" : '/.*' > /dev/null; then 53 | PRG="$link" 54 | else 55 | PRG=`dirname "$PRG"`"/$link" 56 | fi 57 | done 58 | SAVED="`pwd`" 59 | cd "`dirname \"$PRG\"`/" >&- 60 | APP_HOME="`pwd -P`" 61 | cd "$SAVED" >&- 62 | 63 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 64 | 65 | # Determine the Java command to use to start the JVM. 66 | if [ -n "$JAVA_HOME" ] ; then 67 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 68 | # IBM's JDK on AIX uses strange locations for the executables 69 | JAVACMD="$JAVA_HOME/jre/sh/java" 70 | else 71 | JAVACMD="$JAVA_HOME/bin/java" 72 | fi 73 | if [ ! -x "$JAVACMD" ] ; then 74 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 75 | 76 | Please set the JAVA_HOME variable in your environment to match the 77 | location of your Java installation." 78 | fi 79 | else 80 | JAVACMD="java" 81 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 82 | 83 | Please set the JAVA_HOME variable in your environment to match the 84 | location of your Java installation." 85 | fi 86 | 87 | # Increase the maximum file descriptors if we can. 88 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 89 | MAX_FD_LIMIT=`ulimit -H -n` 90 | if [ $? -eq 0 ] ; then 91 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 92 | MAX_FD="$MAX_FD_LIMIT" 93 | fi 94 | ulimit -n $MAX_FD 95 | if [ $? -ne 0 ] ; then 96 | warn "Could not set maximum file descriptor limit: $MAX_FD" 97 | fi 98 | else 99 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 100 | fi 101 | fi 102 | 103 | # For Darwin, add options to specify how the application appears in the dock 104 | if $darwin; then 105 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 106 | fi 107 | 108 | # For Cygwin, switch paths to Windows format before running java 109 | if $cygwin ; then 110 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 111 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 112 | JAVACMD=`cygpath --unix "$JAVACMD"` 113 | 114 | # We build the pattern for arguments to be converted via cygpath 115 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 116 | SEP="" 117 | for dir in $ROOTDIRSRAW ; do 118 | ROOTDIRS="$ROOTDIRS$SEP$dir" 119 | SEP="|" 120 | done 121 | OURCYGPATTERN="(^($ROOTDIRS))" 122 | # Add a user-defined pattern to the cygpath arguments 123 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 124 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 125 | fi 126 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 127 | i=0 128 | for arg in "$@" ; do 129 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 130 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 131 | 132 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 133 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 134 | else 135 | eval `echo args$i`="\"$arg\"" 136 | fi 137 | i=$((i+1)) 138 | done 139 | case $i in 140 | (0) set -- ;; 141 | (1) set -- "$args0" ;; 142 | (2) set -- "$args0" "$args1" ;; 143 | (3) set -- "$args0" "$args1" "$args2" ;; 144 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 145 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 146 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 147 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 148 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 149 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 150 | esac 151 | fi 152 | 153 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 154 | function splitJvmOpts() { 155 | JVM_OPTS=("$@") 156 | } 157 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 158 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 159 | 160 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 161 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /src/main/groovy/fluentd/FluentdBootApplication.groovy: -------------------------------------------------------------------------------- 1 | package fluentd 2 | 3 | import org.springframework.boot.SpringApplication 4 | import org.springframework.boot.autoconfigure.SpringBootApplication 5 | 6 | @SpringBootApplication 7 | class FluentdBootApplication { 8 | 9 | static void main(String[] args) { 10 | SpringApplication.run FluentdBootApplication, args 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/groovy/fluentd/MainRunner.groovy: -------------------------------------------------------------------------------- 1 | package fluentd 2 | 3 | import groovy.json.JsonSlurper 4 | import groovy.util.logging.Log4j 5 | import org.apache.log4j.Level 6 | import org.springframework.boot.CommandLineRunner 7 | import org.springframework.stereotype.Component 8 | import wslite.rest.RESTClient 9 | 10 | @Component 11 | @Log4j 12 | class MainRunner implements CommandLineRunner { 13 | 14 | private static Random random = new Random(); 15 | 16 | @Override 17 | void run(String... args) throws Exception { 18 | while (true) { 19 | log.log(randomLevel(), randomMessage()) 20 | sleep 3000 21 | } 22 | } 23 | 24 | private Level randomLevel() { 25 | switch (random.nextInt(3)) { 26 | case 0: 27 | return Level.DEBUG; 28 | case 1: 29 | return Level.INFO; 30 | case 2: 31 | return Level.ERROR; 32 | default: 33 | return Level.INFO; 34 | } 35 | } 36 | 37 | private String randomMessage() { 38 | def client = new RESTClient('http://api.icndb.com/jokes/random') 39 | def response = client.get() 40 | new JsonSlurper().parse(response.data).value.joke 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geowarin/fluentd-boot/ed84fca388d81ba98bf93ec6c2a219ea4831dbe6/src/main/resources/application.properties -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | dab 8 | 9 | ${FLUENTD_HOST} 10 | ${FLUENTD_PORT} 11 | 20 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | --------------------------------------------------------------------------------