├── .env ├── .gitignore ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── Dockerfile ├── LICENSE ├── Procfile ├── README.md ├── bin └── run ├── docker-compose.yml ├── mvnw ├── mvnw.bat ├── pom.xml └── src └── main ├── java └── com │ └── example │ ├── HerokuReplayApplication.java │ ├── KafkaConfig.java │ ├── MainController.java │ ├── Route.java │ └── consumers │ ├── AbstractLogConsumer.java │ ├── Metrics.java │ └── Replay.java └── resources ├── application.properties └── logback.xml /.env: -------------------------------------------------------------------------------- 1 | KAFKA_URL=kafka://kafka:9092 2 | REDIS_URL=redis://redis:6379 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | .idea 3 | *.iml 4 | .bash_history -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jkutner/heroku-metrics-spring/9f88501266f054ef1c7ab173988b857fc0bdee33/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM maven:3-jdk-8 2 | 3 | RUN apt-get update && apt-get install -y git-all 4 | 5 | RUN mkdir -p /app && useradd -d /home heroku 6 | USER heroku 7 | 8 | RUN mkdir -p /app 9 | ENV HOME /app 10 | WORKDIR /app 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Joe Kutner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: java $JAVA_OPTS -Dserver.port=$PORT -jar target/*.jar 2 | replay: sh bin/run com.example.consumers.Replay 3 | metrics: sh bin/run com.example.consumers.Metrics -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Heroku Replay 2 | 3 | This application funnels production log drains into Kafka, and then replays the requests represented in those logs 4 | against a staging application. 5 | 6 | ## Usage 7 | 8 | Locally with Docker, run: 9 | 10 | ``` 11 | $ docker-compose run shell 12 | ... 13 | heroku@03248675cfea:~$ mvn clean package 14 | ``` 15 | 16 | In other terminal run: 17 | 18 | ``` 19 | $ docker-compose up web 20 | ``` 21 | 22 | Again in the `shell` session run cURL to test it: 23 | 24 | ``` 25 | heroku@03248675cfea:~$ cat << EOF > logs.txt 26 | > 242 <158>1 2016-06-20T21:56:57.107495+00:00 host heroku router - at=info method=GET path="/" host=demodayex.herokuapp.com request_id=1850b395-c7aa-485c-aa04-7d0894b5f276 fwd="68.32.161.89" dyno=web.1 connect=0ms service=6ms status=200 bytes=1548 27 | > EOF 28 | heroku@03248675cfea:~$ curl -d @logs.txt web:8080/logs 29 | ``` 30 | 31 | ## Deploying 32 | 33 | To deploy on Heroku run: 34 | 35 | ``` 36 | $ heroku create mylogdrain 37 | $ heroku addons:create heroku-kafka 38 | $ heroku plugins:install heroku-kafka 39 | $ heroku kafka:wait 40 | $ heroku config:set REPLAY_HOST="http:://example.com" 41 | $ git push heroku master 42 | $ heroku ps:scale replay=1 43 | $ heroku drains:add https://user:pass@mylogdrain.herokuapp.com/logs -a myapp 44 | ``` -------------------------------------------------------------------------------- /bin/run: -------------------------------------------------------------------------------- 1 | cd $(mktemp -d) 2 | unzip -qq /app/target/*.jar 3 | java -cp BOOT-INF/classes/:BOOT-INF/lib/* "$@" -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | redis: 4 | image: redis:3.0-alpine 5 | zookeeper: 6 | image: hone/kafka:2.11 7 | expose: 8 | - "2181" 9 | command: /opt/kafka_2.11-0.9.0.1/bin/zookeeper-server-start.sh /opt/kafka_2.11-0.9.0.1/config/zookeeper.properties 10 | kafka: 11 | image: hone/kafka:2.11 12 | links: 13 | - zookeeper 14 | expose: 15 | - "9092" 16 | command: /home/kafka/bin/kafka-env-start.sh 17 | web: 18 | build: . 19 | command: "java -jar target/heroku-replay-spring-1.0.jar" 20 | volumes: 21 | - .:/app/ 22 | ports: 23 | - "8080:8080" 24 | depends_on: 25 | - redis 26 | - kafka 27 | env_file: .env 28 | shell: 29 | build: . 30 | volumes: 31 | - .:/app/ 32 | - ~/.m2/:/home/.m2/ 33 | command: bash 34 | depends_on: 35 | - web 36 | env_file: .env 37 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # 58 | # Look for the Apple JDKs first to preserve the existing behaviour, and then look 59 | # for the new JDKs provided by Oracle. 60 | # 61 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then 62 | # 63 | # Apple JDKs 64 | # 65 | export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home 66 | fi 67 | 68 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then 69 | # 70 | # Apple JDKs 71 | # 72 | export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 73 | fi 74 | 75 | if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then 76 | # 77 | # Oracle JDKs 78 | # 79 | export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 80 | fi 81 | 82 | if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then 83 | # 84 | # Apple JDKs 85 | # 86 | export JAVA_HOME=`/usr/libexec/java_home` 87 | fi 88 | ;; 89 | esac 90 | 91 | if [ -z "$JAVA_HOME" ] ; then 92 | if [ -r /etc/gentoo-release ] ; then 93 | JAVA_HOME=`java-config --jre-home` 94 | fi 95 | fi 96 | 97 | if [ -z "$M2_HOME" ] ; then 98 | ## resolve links - $0 may be a link to maven's home 99 | PRG="$0" 100 | 101 | # need this for relative symlinks 102 | while [ -h "$PRG" ] ; do 103 | ls=`ls -ld "$PRG"` 104 | link=`expr "$ls" : '.*-> \(.*\)$'` 105 | if expr "$link" : '/.*' > /dev/null; then 106 | PRG="$link" 107 | else 108 | PRG="`dirname "$PRG"`/$link" 109 | fi 110 | done 111 | 112 | saveddir=`pwd` 113 | 114 | M2_HOME=`dirname "$PRG"`/.. 115 | 116 | # make it fully qualified 117 | M2_HOME=`cd "$M2_HOME" && pwd` 118 | 119 | cd "$saveddir" 120 | # echo Using m2 at $M2_HOME 121 | fi 122 | 123 | # For Cygwin, ensure paths are in UNIX format before anything is touched 124 | if $cygwin ; then 125 | [ -n "$M2_HOME" ] && 126 | M2_HOME=`cygpath --unix "$M2_HOME"` 127 | [ -n "$JAVA_HOME" ] && 128 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 129 | [ -n "$CLASSPATH" ] && 130 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 131 | fi 132 | 133 | # For Migwn, ensure paths are in UNIX format before anything is touched 134 | if $mingw ; then 135 | [ -n "$M2_HOME" ] && 136 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 137 | [ -n "$JAVA_HOME" ] && 138 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 139 | # TODO classpath? 140 | fi 141 | 142 | if [ -z "$JAVA_HOME" ]; then 143 | javaExecutable="`which javac`" 144 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 145 | # readlink(1) is not available as standard on Solaris 10. 146 | readLink=`which readlink` 147 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 148 | if $darwin ; then 149 | javaHome="`dirname \"$javaExecutable\"`" 150 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 151 | else 152 | javaExecutable="`readlink -f \"$javaExecutable\"`" 153 | fi 154 | javaHome="`dirname \"$javaExecutable\"`" 155 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 156 | JAVA_HOME="$javaHome" 157 | export JAVA_HOME 158 | fi 159 | fi 160 | fi 161 | 162 | if [ -z "$JAVACMD" ] ; then 163 | if [ -n "$JAVA_HOME" ] ; then 164 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 165 | # IBM's JDK on AIX uses strange locations for the executables 166 | JAVACMD="$JAVA_HOME/jre/sh/java" 167 | else 168 | JAVACMD="$JAVA_HOME/bin/java" 169 | fi 170 | else 171 | JAVACMD="`which java`" 172 | fi 173 | fi 174 | 175 | if [ ! -x "$JAVACMD" ] ; then 176 | echo "Error: JAVA_HOME is not defined correctly." >&2 177 | echo " We cannot execute $JAVACMD" >&2 178 | exit 1 179 | fi 180 | 181 | if [ -z "$JAVA_HOME" ] ; then 182 | echo "Warning: JAVA_HOME environment variable is not set." 183 | fi 184 | 185 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 186 | 187 | # For Cygwin, switch paths to Windows format before running java 188 | if $cygwin; then 189 | [ -n "$M2_HOME" ] && 190 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 191 | [ -n "$JAVA_HOME" ] && 192 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 193 | [ -n "$CLASSPATH" ] && 194 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 195 | fi 196 | 197 | # traverses directory structure from process work directory to filesystem root 198 | # first directory with .mvn subdirectory is considered project base directory 199 | find_maven_basedir() { 200 | local basedir=$(pwd) 201 | local wdir=$(pwd) 202 | while [ "$wdir" != '/' ] ; do 203 | wdir=$(cd "$wdir/.."; pwd) 204 | if [ -d "$wdir"/.mvn ] ; then 205 | basedir=$wdir 206 | break 207 | fi 208 | done 209 | echo "${basedir}" 210 | } 211 | 212 | # concatenates all lines of a file 213 | concat_lines() { 214 | if [ -f "$1" ]; then 215 | echo "$(tr -s '\n' ' ' < "$1")" 216 | fi 217 | } 218 | 219 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} 220 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 221 | 222 | # Provide a "standardized" way to retrieve the CLI args that will 223 | # work with both Windows and non-Windows executions. 224 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" 225 | export MAVEN_CMD_LINE_ARGS 226 | 227 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 228 | 229 | exec "$JAVACMD" \ 230 | $MAVEN_OPTS \ 231 | -classpath "./.mvn/wrapper/maven-wrapper.jar" \ 232 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 233 | ${WRAPPER_LAUNCHER} "$@" 234 | 235 | -------------------------------------------------------------------------------- /mvnw.bat: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto chkMHome 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | :chkMHome 80 | if not "%M2_HOME%"=="" goto valMHome 81 | 82 | SET "M2_HOME=%~dp0.." 83 | if not "%M2_HOME%"=="" goto valMHome 84 | 85 | echo. 86 | echo Error: M2_HOME not found in your environment. >&2 87 | echo Please set the M2_HOME variable in your environment to match the >&2 88 | echo location of the Maven installation. >&2 89 | echo. 90 | goto error 91 | 92 | :valMHome 93 | 94 | :stripMHome 95 | if not "_%M2_HOME:~-1%"=="_\" goto checkMCmd 96 | set "M2_HOME=%M2_HOME:~0,-1%" 97 | goto stripMHome 98 | 99 | :checkMCmd 100 | if exist "%M2_HOME%\bin\mvn.cmd" goto init 101 | 102 | echo. 103 | echo Error: M2_HOME is set to an invalid directory. >&2 104 | echo M2_HOME = "%M2_HOME%" >&2 105 | echo Please set the M2_HOME variable in your environment to match the >&2 106 | echo location of the Maven installation >&2 107 | echo. 108 | goto error 109 | @REM ==== END VALIDATION ==== 110 | 111 | :init 112 | 113 | set MAVEN_CMD_LINE_ARGS=%* 114 | 115 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 116 | @REM Fallback to current working directory if not found. 117 | 118 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 119 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 120 | 121 | set EXEC_DIR=%CD% 122 | set WDIR=%EXEC_DIR% 123 | :findBaseDir 124 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 125 | cd .. 126 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 127 | set WDIR=%CD% 128 | goto findBaseDir 129 | 130 | :baseDirFound 131 | set MAVEN_PROJECTBASEDIR=%WDIR% 132 | cd "%EXEC_DIR%" 133 | goto endDetectBaseDir 134 | 135 | :baseDirNotFound 136 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 137 | cd "%EXEC_DIR%" 138 | 139 | :endDetectBaseDir 140 | 141 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 142 | 143 | @setlocal EnableExtensions EnableDelayedExpansion 144 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 145 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 146 | 147 | :endReadAdditionalConfig 148 | 149 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 150 | 151 | for %%i in ("%M2_HOME%"\boot\plexus-classworlds-*) do set CLASSWORLDS_JAR="%%i" 152 | 153 | set WRAPPER_JAR="".\.mvn\wrapper\maven-wrapper.jar"" 154 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 155 | 156 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.home=%M2_HOME%" "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CMD_LINE_ARGS% 157 | if ERRORLEVEL 1 goto error 158 | goto end 159 | 160 | :error 161 | set ERROR_CODE=1 162 | 163 | :end 164 | @endlocal & set ERROR_CODE=%ERROR_CODE% 165 | 166 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 167 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 168 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 169 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 170 | :skipRcPost 171 | 172 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 173 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 174 | 175 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 176 | 177 | exit /B %ERROR_CODE% 178 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | com.example 7 | 1.0 8 | heroku-replay-spring 9 | 10 | 11 | org.springframework.boot 12 | spring-boot-starter-parent 13 | 1.4.0.M3 14 | 15 | 16 | 17 | 18 | repo.spring.io.milestone 19 | Spring Framework Maven Milestone Repository 20 | https://repo.spring.io/libs-milestone 21 | 22 | 23 | 24 | 25 | 26 | repo.spring.io.milestone 27 | Spring Framework Maven Milestone Repository 28 | https://repo.spring.io/libs-milestone 29 | 30 | 31 | 32 | 33 | 34 | org.springframework.boot 35 | spring-boot-starter-web 36 | 37 | 38 | org.springframework.boot 39 | spring-boot-starter-integration 40 | 41 | 42 | org.springframework.boot 43 | spring-boot-starter-actuator 44 | 45 | 46 | org.springframework.integration 47 | spring-integration-syslog 48 | 4.3.0.RELEASE 49 | 50 | 51 | org.springframework.integration 52 | spring-integration-kafka 53 | 2.0.0.M1 54 | 55 | 56 | com.github.jkutner 57 | env-keystore 58 | 0.1.2 59 | 60 | 61 | org.apache.kafka 62 | kafka-clients 63 | 0.9.0.1 64 | 65 | 66 | redis.clients 67 | jedis 68 | 2.8.0 69 | 70 | 71 | 72 | 73 | 74 | 75 | org.apache.maven.plugins 76 | maven-compiler-plugin 77 | 2.5.1 78 | 79 | 1.8 80 | 1.8 81 | 82 | 83 | 84 | org.springframework.boot 85 | spring-boot-maven-plugin 86 | 87 | true 88 | com.example.HerokuReplayApplication 89 | 90 | 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /src/main/java/com/example/HerokuReplayApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.expression.common.LiteralExpression; 7 | import org.springframework.integration.annotation.ServiceActivator; 8 | import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; 9 | import org.springframework.kafka.core.DefaultKafkaProducerFactory; 10 | import org.springframework.kafka.core.KafkaTemplate; 11 | import org.springframework.kafka.core.ProducerFactory; 12 | import org.springframework.messaging.MessageHandler; 13 | 14 | @SpringBootApplication 15 | public class HerokuReplayApplication { 16 | 17 | @ServiceActivator(inputChannel = "toKafka") 18 | @Bean 19 | public MessageHandler kafkaHandler() throws Exception { 20 | KafkaProducerMessageHandler handler = 21 | new KafkaProducerMessageHandler<>(kafkaTemplate()); 22 | handler.setTopicExpression(new LiteralExpression(KafkaConfig.getTopic())); 23 | handler.setMessageKeyExpression(new LiteralExpression(KafkaConfig.getMessageKey())); 24 | return handler; 25 | } 26 | 27 | @Bean 28 | public KafkaTemplate kafkaTemplate() { 29 | return new KafkaTemplate<>(producerFactory()); 30 | } 31 | 32 | @Bean 33 | public ProducerFactory producerFactory() { 34 | return new DefaultKafkaProducerFactory<>(KafkaConfig.producerDefaults()); 35 | } 36 | 37 | public static void main(String[] args) throws Exception { 38 | SpringApplication.run(HerokuReplayApplication.class, args); 39 | } 40 | 41 | } -------------------------------------------------------------------------------- /src/main/java/com/example/KafkaConfig.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import com.github.jkutner.EnvKeyStore; 4 | import org.apache.kafka.clients.CommonClientConfigs; 5 | import org.apache.kafka.clients.consumer.ConsumerConfig; 6 | import org.apache.kafka.clients.producer.ProducerConfig; 7 | import org.apache.kafka.common.config.SslConfigs; 8 | import org.apache.kafka.common.serialization.StringDeserializer; 9 | import org.apache.kafka.common.serialization.StringSerializer; 10 | 11 | import java.net.URI; 12 | import java.net.URISyntaxException; 13 | import java.util.*; 14 | 15 | import static java.lang.String.format; 16 | 17 | public class KafkaConfig { 18 | 19 | public static String getTopic() { 20 | return "logs"; //getenv("KAFKA_TOPIC"); 21 | } 22 | 23 | public static String getMessageKey() { 24 | return "logs.key"; //getenv("KAFKA_MESSAGE_KEY"); 25 | } 26 | 27 | public static Map consumerDefaults() { 28 | Map properties = defaultKafkaProps(); 29 | properties.put(ConsumerConfig.GROUP_ID_CONFIG, "test"); 30 | properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); 31 | properties.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000"); 32 | properties.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000"); 33 | properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 34 | properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); 35 | return properties; 36 | } 37 | 38 | public static Map producerDefaults() { 39 | Map properties = defaultKafkaProps(); 40 | properties.put(ProducerConfig.ACKS_CONFIG, "all"); 41 | properties.put(ProducerConfig.RETRIES_CONFIG, 0); 42 | properties.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384); 43 | properties.put(ProducerConfig.LINGER_MS_CONFIG, 1); 44 | properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432); 45 | properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 46 | properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 47 | return properties; 48 | } 49 | 50 | private static Map defaultKafkaProps() { 51 | Map properties = new HashMap<>(); 52 | List hostPorts = new ArrayList<>(); 53 | 54 | String kafkaUrl = getenv("KAFKA_URL"); 55 | for (String url : kafkaUrl.split(",")) { 56 | try { 57 | URI uri = new URI(url); 58 | hostPorts.add(String.format("%s:%d", uri.getHost(), uri.getPort())); 59 | 60 | switch (uri.getScheme()) { 61 | case "kafka": 62 | properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "PLAINTEXT"); 63 | break; 64 | case "kafka+ssl": 65 | properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); 66 | 67 | try { 68 | EnvKeyStore envTrustStore = EnvKeyStore.createWithRandomPassword("KAFKA_TRUSTED_CERT"); 69 | EnvKeyStore envKeyStore = EnvKeyStore.createWithRandomPassword("KAFKA_CLIENT_CERT_KEY", "KAFKA_CLIENT_CERT"); 70 | 71 | properties.put(SslConfigs.SSL_TRUSTSTORE_TYPE_CONFIG, envTrustStore.type()); 72 | properties.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, envTrustStore.storeTemp().getAbsolutePath()); 73 | properties.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, envTrustStore.password()); 74 | properties.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, envKeyStore.type()); 75 | properties.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, envKeyStore.storeTemp().getAbsolutePath()); 76 | properties.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, envKeyStore.password()); 77 | } catch (Exception e) { 78 | throw new RuntimeException("There was a problem creating the Kafka key stores", e); 79 | } 80 | break; 81 | default: 82 | throw new IllegalArgumentException(format("unknown scheme; %s", uri.getScheme())); 83 | } 84 | } catch (URISyntaxException e) { 85 | throw new RuntimeException(e); 86 | } 87 | } 88 | 89 | properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, String.join(",", hostPorts)); 90 | return properties; 91 | } 92 | 93 | private static String getenv(String var) { 94 | String val = System.getenv(var); 95 | if (val == null) throw new IllegalArgumentException("Env var $" + var + " is not set!"); 96 | return val; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/com/example/MainController.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.springframework.beans.factory.BeanFactory; 5 | import org.springframework.beans.factory.BeanFactoryAware; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.context.annotation.Description; 8 | import org.springframework.integration.syslog.inbound.RFC6587SyslogDeserializer; 9 | import org.springframework.messaging.MessageChannel; 10 | import org.springframework.messaging.support.GenericMessage; 11 | import org.springframework.stereotype.Controller; 12 | import org.springframework.web.bind.annotation.RequestBody; 13 | import org.springframework.web.bind.annotation.RequestMapping; 14 | import org.springframework.web.bind.annotation.RequestMethod; 15 | import org.springframework.web.bind.annotation.ResponseBody; 16 | 17 | import java.io.ByteArrayInputStream; 18 | import java.io.IOException; 19 | import java.io.InputStream; 20 | import java.util.Collections; 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | 24 | @Controller 25 | @Description("A controller for handling requests for hello messages") 26 | public class MainController implements BeanFactoryAware { 27 | 28 | private BeanFactory context; 29 | 30 | @Override 31 | public void setBeanFactory(BeanFactory factory) { 32 | this.context = factory; 33 | } 34 | 35 | @RequestMapping(value = "/", method = RequestMethod.GET) 36 | @ResponseBody 37 | public String hello() { 38 | return "Hello from Spring!"; 39 | } 40 | 41 | @RequestMapping(value = "/logs", method = RequestMethod.POST) 42 | @ResponseBody 43 | public String logs(@RequestBody String body) throws IOException { 44 | 45 | // "application/logplex-1" does not conform to RFC5424. 46 | // It leaves out STRUCTURED-DATA but does not replace it with 47 | // a NILVALUE. To workaround this, we inject empty STRUCTURED-DATA. 48 | String[] parts = body.split("router - "); 49 | String log = parts[0] + "router - [] " + (parts.length > 1 ? parts[1] : ""); 50 | 51 | RFC6587SyslogDeserializer parser = new RFC6587SyslogDeserializer(); 52 | InputStream is = new ByteArrayInputStream(log.getBytes()); 53 | Map messages = parser.deserialize(is); 54 | ObjectMapper mapper = new ObjectMapper(); 55 | 56 | MessageChannel toKafka = context.getBean("toKafka", MessageChannel.class); 57 | String json = mapper.writeValueAsString(messages); 58 | toKafka.send(new GenericMessage<>(json)); 59 | 60 | return "ok"; 61 | } 62 | 63 | } -------------------------------------------------------------------------------- /src/main/java/com/example/Route.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.regex.Matcher; 6 | import java.util.regex.Pattern; 7 | 8 | public class Route { 9 | 10 | private static final Pattern MSG_PATTERN = Pattern.compile("(\\w+)=\"*((?<=\")[^\"]+(?=\")|([^\\s]+))\"*"); 11 | 12 | private String timestamp; 13 | 14 | private Map message; 15 | 16 | private String rawMessage; 17 | 18 | public Route(Map log) { 19 | this.timestamp = log.get("syslog_TIMESTAMP").toString(); 20 | 21 | this.message = new HashMap<>(); 22 | 23 | rawMessage = log.get("syslog_MESSAGE").toString(); 24 | Matcher m = MSG_PATTERN.matcher(rawMessage); 25 | 26 | while(m.find()) { 27 | message.put(m.group(1), m.group(2)); 28 | } 29 | } 30 | 31 | public String get(String key) { 32 | return message.get(key); 33 | } 34 | 35 | public String timestamp() { 36 | return this.timestamp; 37 | } 38 | 39 | public String toString() { 40 | return rawMessage; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/example/consumers/AbstractLogConsumer.java: -------------------------------------------------------------------------------- 1 | package com.example.consumers; 2 | 3 | import com.example.KafkaConfig; 4 | import com.example.Route; 5 | import com.fasterxml.jackson.core.type.TypeReference; 6 | import com.fasterxml.jackson.databind.ObjectMapper; 7 | import org.apache.kafka.clients.consumer.ConsumerRecord; 8 | import org.apache.kafka.clients.consumer.ConsumerRecords; 9 | import org.apache.kafka.clients.consumer.KafkaConsumer; 10 | 11 | import java.io.IOException; 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | import java.util.Properties; 15 | import java.util.concurrent.CountDownLatch; 16 | import java.util.concurrent.ExecutorService; 17 | import java.util.concurrent.Executors; 18 | import java.util.concurrent.atomic.AtomicBoolean; 19 | 20 | import static java.util.Collections.singletonList; 21 | 22 | public abstract class AbstractLogConsumer { 23 | 24 | public abstract void receive(Route route); 25 | 26 | private ExecutorService executor; 27 | 28 | private KafkaConsumer consumer; 29 | 30 | private final AtomicBoolean running = new AtomicBoolean(); 31 | 32 | private CountDownLatch stopLatch; 33 | 34 | private TypeReference> typeRef = new TypeReference>() {}; 35 | 36 | private ObjectMapper mapper = new ObjectMapper(); 37 | 38 | public void start() { 39 | Runtime.getRuntime().addShutdownHook(new Thread() { 40 | public void run() { 41 | stopLoop(); 42 | } 43 | }); 44 | 45 | running.set(true); 46 | executor = Executors.newSingleThreadExecutor(); 47 | executor.submit(this::loop); 48 | stopLatch = new CountDownLatch(1); 49 | } 50 | 51 | public void stopLoop() { 52 | running.set(false); 53 | try { 54 | stopLatch.await(); 55 | } catch (InterruptedException e) { 56 | e.printStackTrace(); 57 | } finally { 58 | executor.shutdown(); 59 | } 60 | } 61 | 62 | private void loop() { 63 | System.out.println("starting consumer..."); 64 | Properties properties = new Properties(); 65 | properties.putAll(KafkaConfig.consumerDefaults()); 66 | 67 | consumer = new KafkaConsumer<>(properties); 68 | consumer.subscribe(singletonList(KafkaConfig.getTopic())); 69 | 70 | while (running.get()) { 71 | ConsumerRecords records = consumer.poll(100); 72 | for (ConsumerRecord record : records) { 73 | try { 74 | Map recordMap = mapper.readValue(record.value(), typeRef); 75 | Route route = new Route(recordMap); 76 | receive(route); 77 | } catch (IOException e) { 78 | System.out.println("Error parsing record: " + record.value()); 79 | e.printStackTrace(); 80 | } 81 | } 82 | } 83 | 84 | System.out.println("closing consumer..."); 85 | consumer.close(); 86 | stopLatch.countDown(); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/com/example/consumers/Metrics.java: -------------------------------------------------------------------------------- 1 | package com.example.consumers; 2 | 3 | import com.example.Route; 4 | import redis.clients.jedis.Jedis; 5 | import redis.clients.jedis.JedisPool; 6 | 7 | import java.net.URI; 8 | import java.net.URISyntaxException; 9 | import java.nio.charset.StandardCharsets; 10 | import java.security.MessageDigest; 11 | import java.util.Arrays; 12 | 13 | public class Metrics extends AbstractLogConsumer { 14 | 15 | private JedisPool pool; 16 | 17 | public static void main(String[] args) throws URISyntaxException { 18 | new Metrics().start(); 19 | } 20 | 21 | public Metrics() throws URISyntaxException { 22 | if(System.getenv("REDIS_URL") == null) { 23 | throw new IllegalArgumentException("No REDIS_URL is set!"); 24 | } 25 | URI redisUri = new URI(System.getenv("REDIS_URL")); 26 | 27 | pool = new JedisPool(redisUri); 28 | } 29 | 30 | @Override 31 | public void stopLoop() { 32 | pool.destroy(); 33 | super.stopLoop(); 34 | } 35 | 36 | @Override 37 | public void receive(Route route) { 38 | String path = route.get("path"); 39 | 40 | try { 41 | MessageDigest digest = MessageDigest.getInstance("SHA-256"); 42 | byte[] hash = digest.digest(path.getBytes(StandardCharsets.UTF_8)); 43 | String pathDigest = new String(hash); 44 | 45 | try (Jedis jedis = pool.getResource()) { 46 | System.out.println("Updating Redis: " + route.toString()); 47 | jedis.hset("routes", path, pathDigest); 48 | 49 | for (String metric : Arrays.asList("service", "connect")) { 50 | Integer value = Integer.valueOf(route.get(metric).replace("ms", "")); 51 | String key = pathDigest + "::" + metric; 52 | 53 | jedis.hincrBy(key, "sum", value); 54 | jedis.hincrBy(key, "count", 1); 55 | 56 | Integer sum = Integer.valueOf(jedis.hget(key, "sum")); 57 | Float count = Float.valueOf(jedis.hget(key, "count")); 58 | Float avg = sum / count; 59 | jedis.hset(key, "average", String.valueOf(avg)); 60 | } 61 | 62 | jedis.hincrBy(pathDigest + "::statuses", route.get("status"), 1); 63 | } 64 | } catch (Exception e) { 65 | System.out.print("Error updating Redis!"); 66 | e.printStackTrace(); 67 | } 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/example/consumers/Replay.java: -------------------------------------------------------------------------------- 1 | package com.example.consumers; 2 | 3 | import com.example.Route; 4 | 5 | import javax.net.ssl.HttpsURLConnection; 6 | import java.net.ProtocolException; 7 | import java.net.URL; 8 | import java.util.Map; 9 | 10 | public class Replay extends AbstractLogConsumer { 11 | 12 | public static void main(String[] args) { 13 | new Replay().start(); 14 | } 15 | 16 | @Override 17 | public void receive(Route route) { 18 | if ("GET".equals(route.get("method"))) { 19 | String path = route.get("path"); 20 | 21 | if (null == System.getenv("REPLAY_HOST")) { 22 | System.out.println("Simulating request: " + path); 23 | } else { 24 | try { 25 | URL url = new URL(new URL(System.getenv("REPLAY_HOST")), path); 26 | HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); 27 | con.setRequestMethod("GET"); 28 | int responseCode = con.getResponseCode(); 29 | System.out.println("Sent request (" + responseCode + "): " + path); 30 | } catch (Exception e) { 31 | e.printStackTrace(); 32 | } 33 | } 34 | } 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | endpoints.shutdown.enabled: true 2 | server.tomcat.basedir: target/tomcat 3 | server.tomcat.access_log_enabled: true 4 | server.tomcat.access_log_pattern: %h %t "%r" %s %b 5 | security.require_ssl: false 6 | service.name: heroku-replay-spring 7 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | --------------------------------------------------------------------------------