├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ └── build.yaml ├── .gitignore ├── app ├── .dockerignore ├── .gitignore ├── .mvn │ └── wrapper │ │ ├── maven-wrapper.jar │ │ └── maven-wrapper.properties ├── Dockerfile ├── mvnw ├── mvnw.cmd ├── otel.properties ├── pom.xml ├── readme.md └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── app │ │ │ ├── AppApplication.java │ │ │ ├── Peanuts.java │ │ │ ├── PeanutsRepository.java │ │ │ └── PeanutsService.java │ └── resources │ │ ├── application-test.yaml │ │ └── application.yaml │ └── test │ └── java │ └── com │ └── example │ └── app │ └── AppApplicationTests.java ├── docker-compose.yaml ├── etc ├── dashboards.yaml ├── dashboards │ └── spring-boot-observability.json ├── grafana │ └── datasource.yml ├── prometheus │ └── prometheus.yml └── tempo.yml ├── images ├── dashboard.png ├── logs-to-traces.png ├── loki-derive-filed.png ├── metrics-to-traces.png ├── observability-arch.drawio ├── observability-arch.jpg ├── observability-correlations.jpeg ├── prometheus-exemplars.png ├── span-data-http.png ├── span-data-postgresql.png ├── span-data-redis.png ├── tempo-trace-to-logs.png └── traces-to-logs.png ├── k6-script.js ├── pull_arm_image.sh ├── readme.md ├── request-script.sh └── trace.sh /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | ko_fi: blueswen 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: maven 4 | directory: "/app" 5 | schedule: 6 | interval: "weekly" 7 | time: '10:00' 8 | timezone: Asia/Taipei 9 | open-pull-requests-limit: 10 10 | - package-ecosystem: "github-actions" 11 | directory: "/" 12 | schedule: 13 | interval: "weekly" 14 | time: '10:00' 15 | timezone: Asia/Taipei 16 | open-pull-requests-limit: 10 17 | - package-ecosystem: "docker" 18 | directory: "/app" 19 | schedule: 20 | interval: "weekly" 21 | time: '10:00' 22 | timezone: Asia/Taipei 23 | open-pull-requests-limit: 10 24 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: Build and Push Docker Images 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - 'app/**' 9 | - '.github/workflows/build.yaml' 10 | workflow_dispatch: 11 | 12 | jobs: 13 | build-and-push: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Check out code 17 | uses: actions/checkout@v4 18 | 19 | - name: Set up Docker Buildx 20 | uses: docker/setup-buildx-action@v3 21 | 22 | - name: Login to GitHub Container Registry 23 | uses: docker/login-action@v3 24 | with: 25 | registry: ghcr.io 26 | username: ${{ github.actor }} 27 | password: ${{ secrets.REGISTRY_TOKEN }} 28 | 29 | - name: Build and push Spring Boot image 30 | uses: docker/build-push-action@v6 31 | with: 32 | context: ./app 33 | push: true 34 | tags: ghcr.io/${{ github.repository }}/app:latest 35 | platforms: linux/amd64,linux/arm64 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | logs 3 | .idea 4 | *.iml -------------------------------------------------------------------------------- /app/.dockerignore: -------------------------------------------------------------------------------- 1 | otel.properties 2 | -------------------------------------------------------------------------------- /app/.gitignore: -------------------------------------------------------------------------------- 1 | opentelemetry-javaagent.jar 2 | HELP.md 3 | target/ 4 | !.mvn/wrapper/maven-wrapper.jar 5 | !**/src/main/**/target/ 6 | !**/src/test/**/target/ 7 | 8 | ### STS ### 9 | .apt_generated 10 | .classpath 11 | .factorypath 12 | .project 13 | .settings 14 | .springBeans 15 | .sts4-cache 16 | 17 | ### IntelliJ IDEA ### 18 | .idea 19 | *.iws 20 | *.iml 21 | *.ipr 22 | logs 23 | 24 | ### NetBeans ### 25 | /nbproject/private/ 26 | /nbbuild/ 27 | /dist/ 28 | /nbdist/ 29 | /.nb-gradle/ 30 | build/ 31 | !**/src/main/**/build/ 32 | !**/src/test/**/build/ 33 | 34 | ### VS Code ### 35 | .vscode/ 36 | -------------------------------------------------------------------------------- /app/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blueswen/spring-boot-observability/c73e743034d8ce032eab63664565c3e75e3feeb3/app/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /app/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.6/apache-maven-3.8.6-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar 3 | -------------------------------------------------------------------------------- /app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM curlimages/curl:8.13.0 AS download 2 | ARG OTEL_AGENT_VERSION="2.16.0" 3 | RUN curl --silent --fail --insecure -L "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v${OTEL_AGENT_VERSION}/opentelemetry-javaagent.jar" \ 4 | -o "$HOME/opentelemetry-javaagent.jar" 5 | 6 | FROM maven:3-amazoncorretto-24-alpine AS build 7 | WORKDIR /build 8 | COPY . . 9 | RUN mvn clean package -DskipTests --quiet 10 | 11 | FROM amazoncorretto:24-alpine3.21 12 | COPY --from=build /build/target/*.jar /app.jar 13 | COPY --from=download /home/curl_user/opentelemetry-javaagent.jar /opentelemetry-javaagent.jar 14 | ENTRYPOINT ["java", \ 15 | "-javaagent:/opentelemetry-javaagent.jar", \ 16 | "-jar", "/app.jar" \ 17 | ] 18 | -------------------------------------------------------------------------------- /app/mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /usr/local/etc/mavenrc ] ; then 40 | . /usr/local/etc/mavenrc 41 | fi 42 | 43 | if [ -f /etc/mavenrc ] ; then 44 | . /etc/mavenrc 45 | fi 46 | 47 | if [ -f "$HOME/.mavenrc" ] ; then 48 | . "$HOME/.mavenrc" 49 | fi 50 | 51 | fi 52 | 53 | # OS specific support. $var _must_ be set to either true or false. 54 | cygwin=false; 55 | darwin=false; 56 | mingw=false 57 | case "`uname`" in 58 | CYGWIN*) cygwin=true ;; 59 | MINGW*) mingw=true;; 60 | Darwin*) darwin=true 61 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 62 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 63 | if [ -z "$JAVA_HOME" ]; then 64 | if [ -x "/usr/libexec/java_home" ]; then 65 | export JAVA_HOME="`/usr/libexec/java_home`" 66 | else 67 | export JAVA_HOME="/Library/Java/Home" 68 | fi 69 | fi 70 | ;; 71 | esac 72 | 73 | if [ -z "$JAVA_HOME" ] ; then 74 | if [ -r /etc/gentoo-release ] ; then 75 | JAVA_HOME=`java-config --jre-home` 76 | fi 77 | fi 78 | 79 | if [ -z "$M2_HOME" ] ; then 80 | ## resolve links - $0 may be a link to maven's home 81 | PRG="$0" 82 | 83 | # need this for relative symlinks 84 | while [ -h "$PRG" ] ; do 85 | ls=`ls -ld "$PRG"` 86 | link=`expr "$ls" : '.*-> \(.*\)$'` 87 | if expr "$link" : '/.*' > /dev/null; then 88 | PRG="$link" 89 | else 90 | PRG="`dirname "$PRG"`/$link" 91 | fi 92 | done 93 | 94 | saveddir=`pwd` 95 | 96 | M2_HOME=`dirname "$PRG"`/.. 97 | 98 | # make it fully qualified 99 | M2_HOME=`cd "$M2_HOME" && pwd` 100 | 101 | cd "$saveddir" 102 | # echo Using m2 at $M2_HOME 103 | fi 104 | 105 | # For Cygwin, ensure paths are in UNIX format before anything is touched 106 | if $cygwin ; then 107 | [ -n "$M2_HOME" ] && 108 | M2_HOME=`cygpath --unix "$M2_HOME"` 109 | [ -n "$JAVA_HOME" ] && 110 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 111 | [ -n "$CLASSPATH" ] && 112 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 113 | fi 114 | 115 | # For Mingw, ensure paths are in UNIX format before anything is touched 116 | if $mingw ; then 117 | [ -n "$M2_HOME" ] && 118 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 119 | [ -n "$JAVA_HOME" ] && 120 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 121 | fi 122 | 123 | if [ -z "$JAVA_HOME" ]; then 124 | javaExecutable="`which javac`" 125 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 126 | # readlink(1) is not available as standard on Solaris 10. 127 | readLink=`which readlink` 128 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 129 | if $darwin ; then 130 | javaHome="`dirname \"$javaExecutable\"`" 131 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 132 | else 133 | javaExecutable="`readlink -f \"$javaExecutable\"`" 134 | fi 135 | javaHome="`dirname \"$javaExecutable\"`" 136 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 137 | JAVA_HOME="$javaHome" 138 | export JAVA_HOME 139 | fi 140 | fi 141 | fi 142 | 143 | if [ -z "$JAVACMD" ] ; then 144 | if [ -n "$JAVA_HOME" ] ; then 145 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 146 | # IBM's JDK on AIX uses strange locations for the executables 147 | JAVACMD="$JAVA_HOME/jre/sh/java" 148 | else 149 | JAVACMD="$JAVA_HOME/bin/java" 150 | fi 151 | else 152 | JAVACMD="`\\unset -f command; \\command -v java`" 153 | fi 154 | fi 155 | 156 | if [ ! -x "$JAVACMD" ] ; then 157 | echo "Error: JAVA_HOME is not defined correctly." >&2 158 | echo " We cannot execute $JAVACMD" >&2 159 | exit 1 160 | fi 161 | 162 | if [ -z "$JAVA_HOME" ] ; then 163 | echo "Warning: JAVA_HOME environment variable is not set." 164 | fi 165 | 166 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 167 | 168 | # traverses directory structure from process work directory to filesystem root 169 | # first directory with .mvn subdirectory is considered project base directory 170 | find_maven_basedir() { 171 | 172 | if [ -z "$1" ] 173 | then 174 | echo "Path not specified to find_maven_basedir" 175 | return 1 176 | fi 177 | 178 | basedir="$1" 179 | wdir="$1" 180 | while [ "$wdir" != '/' ] ; do 181 | if [ -d "$wdir"/.mvn ] ; then 182 | basedir=$wdir 183 | break 184 | fi 185 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 186 | if [ -d "${wdir}" ]; then 187 | wdir=`cd "$wdir/.."; pwd` 188 | fi 189 | # end of workaround 190 | done 191 | echo "${basedir}" 192 | } 193 | 194 | # concatenates all lines of a file 195 | concat_lines() { 196 | if [ -f "$1" ]; then 197 | echo "$(tr -s '\n' ' ' < "$1")" 198 | fi 199 | } 200 | 201 | BASE_DIR=`find_maven_basedir "$(pwd)"` 202 | if [ -z "$BASE_DIR" ]; then 203 | exit 1; 204 | fi 205 | 206 | ########################################################################################## 207 | # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 208 | # This allows using the maven wrapper in projects that prohibit checking in binary data. 209 | ########################################################################################## 210 | if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then 211 | if [ "$MVNW_VERBOSE" = true ]; then 212 | echo "Found .mvn/wrapper/maven-wrapper.jar" 213 | fi 214 | else 215 | if [ "$MVNW_VERBOSE" = true ]; then 216 | echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." 217 | fi 218 | if [ -n "$MVNW_REPOURL" ]; then 219 | jarUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" 220 | else 221 | jarUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" 222 | fi 223 | while IFS="=" read key value; do 224 | case "$key" in (wrapperUrl) jarUrl="$value"; break ;; 225 | esac 226 | done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" 227 | if [ "$MVNW_VERBOSE" = true ]; then 228 | echo "Downloading from: $jarUrl" 229 | fi 230 | wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" 231 | if $cygwin; then 232 | wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` 233 | fi 234 | 235 | if command -v wget > /dev/null; then 236 | if [ "$MVNW_VERBOSE" = true ]; then 237 | echo "Found wget ... using wget" 238 | fi 239 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 240 | wget "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" 241 | else 242 | wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" 243 | fi 244 | elif command -v curl > /dev/null; then 245 | if [ "$MVNW_VERBOSE" = true ]; then 246 | echo "Found curl ... using curl" 247 | fi 248 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 249 | curl -o "$wrapperJarPath" "$jarUrl" -f 250 | else 251 | curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f 252 | fi 253 | 254 | else 255 | if [ "$MVNW_VERBOSE" = true ]; then 256 | echo "Falling back to using Java to download" 257 | fi 258 | javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" 259 | # For Cygwin, switch paths to Windows format before running javac 260 | if $cygwin; then 261 | javaClass=`cygpath --path --windows "$javaClass"` 262 | fi 263 | if [ -e "$javaClass" ]; then 264 | if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 265 | if [ "$MVNW_VERBOSE" = true ]; then 266 | echo " - Compiling MavenWrapperDownloader.java ..." 267 | fi 268 | # Compiling the Java class 269 | ("$JAVA_HOME/bin/javac" "$javaClass") 270 | fi 271 | if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 272 | # Running the downloader 273 | if [ "$MVNW_VERBOSE" = true ]; then 274 | echo " - Running MavenWrapperDownloader.java ..." 275 | fi 276 | ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") 277 | fi 278 | fi 279 | fi 280 | fi 281 | ########################################################################################## 282 | # End of extension 283 | ########################################################################################## 284 | 285 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 286 | if [ "$MVNW_VERBOSE" = true ]; then 287 | echo $MAVEN_PROJECTBASEDIR 288 | fi 289 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 290 | 291 | # For Cygwin, switch paths to Windows format before running java 292 | if $cygwin; then 293 | [ -n "$M2_HOME" ] && 294 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 295 | [ -n "$JAVA_HOME" ] && 296 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 297 | [ -n "$CLASSPATH" ] && 298 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 299 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 300 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 301 | fi 302 | 303 | # Provide a "standardized" way to retrieve the CLI args that will 304 | # work with both Windows and non-Windows executions. 305 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" 306 | export MAVEN_CMD_LINE_ARGS 307 | 308 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 309 | 310 | exec "$JAVACMD" \ 311 | $MAVEN_OPTS \ 312 | $MAVEN_DEBUG_OPTS \ 313 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 314 | "-Dmaven.home=${M2_HOME}" \ 315 | "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 316 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 317 | -------------------------------------------------------------------------------- /app/mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM https://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* 50 | if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" 124 | 125 | FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( 126 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 127 | ) 128 | 129 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 130 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 131 | if exist %WRAPPER_JAR% ( 132 | if "%MVNW_VERBOSE%" == "true" ( 133 | echo Found %WRAPPER_JAR% 134 | ) 135 | ) else ( 136 | if not "%MVNW_REPOURL%" == "" ( 137 | SET DOWNLOAD_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar" 138 | ) 139 | if "%MVNW_VERBOSE%" == "true" ( 140 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 141 | echo Downloading from: %DOWNLOAD_URL% 142 | ) 143 | 144 | powershell -Command "&{"^ 145 | "$webclient = new-object System.Net.WebClient;"^ 146 | "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ 147 | "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ 148 | "}"^ 149 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ 150 | "}" 151 | if "%MVNW_VERBOSE%" == "true" ( 152 | echo Finished downloading %WRAPPER_JAR% 153 | ) 154 | ) 155 | @REM End of extension 156 | 157 | @REM Provide a "standardized" way to retrieve the CLI args that will 158 | @REM work with both Windows and non-Windows executions. 159 | set MAVEN_CMD_LINE_ARGS=%* 160 | 161 | %MAVEN_JAVA_EXE% ^ 162 | %JVM_CONFIG_MAVEN_PROPS% ^ 163 | %MAVEN_OPTS% ^ 164 | %MAVEN_DEBUG_OPTS% ^ 165 | -classpath %WRAPPER_JAR% ^ 166 | "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ 167 | %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 168 | if ERRORLEVEL 1 goto error 169 | goto end 170 | 171 | :error 172 | set ERROR_CODE=1 173 | 174 | :end 175 | @endlocal & set ERROR_CODE=%ERROR_CODE% 176 | 177 | if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost 178 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 179 | if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" 180 | if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" 181 | :skipRcPost 182 | 183 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 184 | if "%MAVEN_BATCH_PAUSE%"=="on" pause 185 | 186 | if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% 187 | 188 | cmd /C exit /B %ERROR_CODE% 189 | -------------------------------------------------------------------------------- /app/otel.properties: -------------------------------------------------------------------------------- 1 | otel.exporter.otlp.endpoint=http://localhost:4317 2 | otel.exporter.otlp.protocol=grpc 3 | otel.service.name=app 4 | otel.metrics.exporter=none 5 | otel.logs.exporter=none 6 | -------------------------------------------------------------------------------- /app/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 3.5.0 9 | 10 | 11 | com.example 12 | app 13 | 0.0.1 14 | app 15 | Demo project for Spring Boot Observability 16 | 17 | 24 18 | 19 | 20 | 21 | 22 | 23 | io.opentelemetry 24 | opentelemetry-bom 25 | 1.50.0 26 | pom 27 | import 28 | 29 | 30 | 31 | 32 | 33 | 34 | org.springframework.boot 35 | spring-boot-starter-web 36 | 37 | 38 | org.springframework.boot 39 | spring-boot-devtools 40 | runtime 41 | true 42 | 43 | 44 | org.springframework.boot 45 | spring-boot-starter-test 46 | test 47 | 48 | 49 | org.springframework.boot 50 | spring-boot-starter-actuator 51 | 52 | 53 | org.springframework.boot 54 | spring-boot-starter-data-jpa 55 | 56 | 57 | org.springframework.boot 58 | spring-boot-starter-cache 59 | 60 | 61 | org.springframework.boot 62 | spring-boot-starter-data-redis 63 | 64 | 65 | 66 | 67 | io.micrometer 68 | micrometer-registry-prometheus 69 | 70 | 71 | io.micrometer 72 | micrometer-tracing-bridge-otel 73 | 74 | 75 | 76 | 77 | io.opentelemetry 78 | opentelemetry-api 79 | 80 | 81 | 82 | 83 | org.postgresql 84 | postgresql 85 | 86 | 87 | 88 | 89 | org.apache.httpcomponents 90 | httpclient 91 | 4.5.14 92 | 93 | 94 | org.apache.httpcomponents 95 | httpcore 96 | 97 | 98 | org.apache.httpcomponents 99 | fluent-hc 100 | 4.5.14 101 | 102 | 103 | 104 | 105 | org.springdoc 106 | springdoc-openapi-starter-webmvc-ui 107 | 2.8.8 108 | 109 | 110 | 111 | 112 | 113 | 114 | org.springframework.boot 115 | spring-boot-maven-plugin 116 | 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /app/readme.md: -------------------------------------------------------------------------------- 1 | # Spring Boot Observability Demo Application 2 | 3 | ## Development Setup 4 | 5 | 1. Download OpenTelemetry Agent Jar, check the latest version at [OpenTelemetry Java Instrumentation Releases](https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/) 6 | 7 | ```bash 8 | OTEL_AGENT_VERSION=2.16.0 wget "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v${OTEL_AGENT_VERSION}/opentelemetry-javaagent.jar" 9 | ``` 10 | 11 | 2. Start dependency services using Docker Compose 12 | 13 | ```bash 14 | docker compose -f ../docker-compose.yaml up -d grafana tempo postgres redis 15 | ``` 16 | 17 | 3. Build the Spring Boot application 18 | 19 | ```bash 20 | ./mvnw clean package 21 | ``` 22 | 23 | 4. Run the Spring Boot application with OpenTelemetry agent 24 | 25 | ```bash 26 | java -javaagent:opentelemetry-javaagent.jar \ 27 | -Dotel.javaagent.configuration-file=./app/otel.properties \ 28 | -jar target/app-0.0.1.jar \ 29 | --spring.profiles.active=test 30 | ``` 31 | 5. Access application and Grafana: 32 | 33 | - Application: 34 | - Grafana: (admin/admin) 35 | 36 | 6. Tear down services 37 | 38 | ```bash 39 | docker compose -f ../docker-compose.yaml down 40 | ``` 41 | -------------------------------------------------------------------------------- /app/src/main/java/com/example/app/AppApplication.java: -------------------------------------------------------------------------------- 1 | package com.example.app; 2 | 3 | import jakarta.servlet.http.HttpServletResponse; 4 | import org.apache.http.client.fluent.Request; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.boot.SpringApplication; 9 | import org.springframework.boot.autoconfigure.SpringBootApplication; 10 | import org.springframework.cache.annotation.EnableCaching; 11 | import org.springframework.http.HttpHeaders; 12 | import org.springframework.web.bind.annotation.GetMapping; 13 | import org.springframework.web.bind.annotation.PathVariable; 14 | import org.springframework.web.bind.annotation.PostMapping; 15 | import org.springframework.web.bind.annotation.RequestBody; 16 | import org.springframework.web.bind.annotation.RequestHeader; 17 | import org.springframework.web.bind.annotation.RequestParam; 18 | import org.springframework.web.bind.annotation.RestController; 19 | 20 | import java.io.IOException; 21 | import java.util.Arrays; 22 | import java.util.List; 23 | import java.util.Random; 24 | 25 | @RestController 26 | @EnableCaching 27 | @SpringBootApplication 28 | public class AppApplication { 29 | @Autowired 30 | private PeanutsService service; 31 | 32 | Logger logger = LoggerFactory.getLogger(AppApplication.class); 33 | 34 | public static void main(String[] args) { 35 | SpringApplication.run(AppApplication.class, args); 36 | } 37 | 38 | @GetMapping("/") 39 | public String root(@RequestParam(value = "name", defaultValue = "World") String name, @RequestHeader HttpHeaders headers) { 40 | logger.error(headers.toString()); 41 | logger.error(String.format("Hello %s!!", name)); 42 | logger.debug("Debugging log"); 43 | logger.info("Info log"); 44 | logger.warn("Hey, This is a warning!"); 45 | logger.error("Oops! We have an Error. OK"); 46 | return String.format("Hello %s!!", name); 47 | } 48 | 49 | @GetMapping("/io_task") 50 | public String io_task() throws InterruptedException { 51 | Thread.sleep(1000); 52 | logger.info("io_task"); 53 | return "io_task"; 54 | } 55 | 56 | @GetMapping("/cpu_task") 57 | public String cpu_task() { 58 | for (int i = 0; i < 100; i++) { 59 | int tmp = i * i * i; 60 | } 61 | logger.info("cpu_task"); 62 | return "cpu_task"; 63 | } 64 | 65 | @GetMapping("/random_sleep") 66 | public String random_sleep() throws InterruptedException { 67 | Thread.sleep((int) (Math.random() / 5 * 10000)); 68 | logger.info("random_sleep"); 69 | return "random_sleep"; 70 | } 71 | 72 | @GetMapping("/random_status") 73 | public String random_status(HttpServletResponse response) throws InterruptedException { 74 | List givenList = Arrays.asList(200, 200, 300, 400, 500); 75 | Random rand = new Random(); 76 | int randomElement = givenList.get(rand.nextInt(givenList.size())); 77 | response.setStatus(randomElement); 78 | logger.info("random_status"); 79 | return "random_status"; 80 | } 81 | 82 | @GetMapping("/chain") 83 | public String chain() throws InterruptedException, IOException { 84 | String TARGET_ONE_HOST = System.getenv().getOrDefault("TARGET_ONE_HOST", "localhost"); 85 | String TARGET_TWO_HOST = System.getenv().getOrDefault("TARGET_TWO_HOST", "localhost"); 86 | logger.debug("chain is starting"); 87 | Request.Get("http://localhost:8080/") 88 | .execute().returnContent(); 89 | Request.Get(String.format("http://%s:8080/io_task", TARGET_ONE_HOST)) 90 | .execute().returnContent(); 91 | Request.Get(String.format("http://%s:8080/cpu_task", TARGET_TWO_HOST)) 92 | .execute().returnContent(); 93 | logger.debug("chain is finished"); 94 | return "chain"; 95 | } 96 | 97 | @GetMapping("/error_test") 98 | public String error_test() throws Exception { 99 | throw new Exception("Error test"); 100 | } 101 | 102 | @GetMapping("/peanuts/{id}") 103 | public Peanuts getPeanutsById(@PathVariable Long id) { 104 | logger.info("Get Peanuts Character by id"); 105 | return service.getPeanutsById(id); 106 | } 107 | 108 | @PostMapping("/peanuts") 109 | public Peanuts savePeanuts(@RequestBody Peanuts peanuts) { 110 | logger.info("Create Peanuts Character"); 111 | return service.savePeanuts(peanuts); 112 | } 113 | 114 | } 115 | 116 | -------------------------------------------------------------------------------- /app/src/main/java/com/example/app/Peanuts.java: -------------------------------------------------------------------------------- 1 | package com.example.app; 2 | 3 | import io.swagger.v3.oas.annotations.media.Schema; 4 | import jakarta.persistence.Entity; 5 | import jakarta.persistence.GeneratedValue; 6 | import jakarta.persistence.GenerationType; 7 | import jakarta.persistence.Id; 8 | 9 | import java.io.Serializable; 10 | 11 | @Entity 12 | public class Peanuts implements Serializable { 13 | private static final long serialVersionUID = 1L; 14 | 15 | @Id 16 | @GeneratedValue(strategy = GenerationType.IDENTITY) 17 | @Schema(accessMode = Schema.AccessMode.READ_ONLY) 18 | private Long id; 19 | 20 | @Schema(example = "Snoopy") 21 | private String name; 22 | 23 | @Schema(example = "Charlie Brown's pet beagle") 24 | private String description; 25 | 26 | // Id 27 | public Long getId() { 28 | return id; 29 | } 30 | 31 | // Name 32 | public String getName() { 33 | return name; 34 | } 35 | public void setName(String name) { 36 | this.name = name; 37 | } 38 | 39 | // Description 40 | public String getDescription() { 41 | return description; 42 | } 43 | public void setDescription(String description) { 44 | this.description = description; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /app/src/main/java/com/example/app/PeanutsRepository.java: -------------------------------------------------------------------------------- 1 | package com.example.app; 2 | 3 | import org.springframework.data.jpa.repository.JpaRepository; 4 | 5 | public interface PeanutsRepository extends JpaRepository { 6 | } 7 | -------------------------------------------------------------------------------- /app/src/main/java/com/example/app/PeanutsService.java: -------------------------------------------------------------------------------- 1 | package com.example.app; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.cache.annotation.CachePut; 5 | import org.springframework.cache.annotation.Cacheable; 6 | import org.springframework.stereotype.Service; 7 | import org.springframework.transaction.annotation.Transactional; 8 | 9 | @Service 10 | public class PeanutsService { 11 | @Autowired 12 | private PeanutsRepository repository; 13 | 14 | @Cacheable(value = "peanuts", key = "#id") 15 | public Peanuts getPeanutsById(Long id) { 16 | return repository.findById(id).orElse(null); 17 | } 18 | 19 | @CachePut(value = "peanuts", key = "#peanuts.id") 20 | public Peanuts savePeanuts(Peanuts peanuts) { 21 | return repository.save(peanuts); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /app/src/main/resources/application-test.yaml: -------------------------------------------------------------------------------- 1 | spring: 2 | datasource: 3 | url: jdbc:postgresql://localhost:5432/postgres 4 | username: postgres 5 | password: postgres 6 | jpa: 7 | hibernate: 8 | ddl-auto: update 9 | cache: 10 | type: redis 11 | # https://stackoverflow.com/a/79180709 12 | data: 13 | redis: 14 | host: localhost 15 | port: 6379 16 | -------------------------------------------------------------------------------- /app/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | management: 2 | endpoints: 3 | web: 4 | exposure: 5 | include: prometheus 6 | metrics: 7 | tags: 8 | application: app 9 | distribution: 10 | percentiles-histogram: 11 | http: 12 | server: 13 | requests: 'true' 14 | logging: 15 | pattern: 16 | level: "trace_id=%mdc{trace_id} span_id=%mdc{span_id} trace_flags=%mdc{trace_flags} %p" 17 | level: 18 | com.example.app: DEBUG 19 | 20 | spring: 21 | datasource: 22 | url: jdbc:postgresql://postgres:5432/postgres 23 | username: postgres 24 | password: postgres 25 | jpa: 26 | hibernate: 27 | ddl-auto: update 28 | cache: 29 | type: redis 30 | # https://stackoverflow.com/a/79180709 31 | data: 32 | redis: 33 | host: redis 34 | port: 6379 35 | -------------------------------------------------------------------------------- /app/src/test/java/com/example/app/AppApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.example.app; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | import org.springframework.test.context.ActiveProfiles; 6 | 7 | @SpringBootTest 8 | @ActiveProfiles("test") 9 | class AppApplicationTests { 10 | 11 | @Test 12 | void contextLoads() { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | x-logging: &default-logging 2 | driver: loki 3 | options: 4 | loki-url: 'http://localhost:3100/api/prom/push' 5 | loki-pipeline-stages: | 6 | - multiline: 7 | firstline: '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}' 8 | max_wait_time: 3s 9 | - regex: 10 | expression: '^(?P\d{4}-\d{2}-\d{2})T(?P