├── .gitignore ├── eureka-service ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── EurekaServiceApplication.java │ └── resources │ │ └── bootstrap.properties │ └── test │ └── java │ └── com │ └── example │ └── EurekaServiceApplicationTests.java ├── images ├── feign.png ├── template.png ├── zipkin-logo.jpg ├── zipkin-ui-and-resttemplate-detail.png ├── zipkin-ui-and-resttemplate.png └── zipkin-ui.png ├── message-client ├── manifest.yml ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── MessageClientApplication.java │ └── resources │ │ ├── application-cloud.properties │ │ └── application.properties │ └── test │ └── java │ └── com │ └── example │ └── ZipkinClientApplicationTests.java ├── message-service ├── manifest.yml ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── MessageServiceApplication.java │ └── resources │ │ ├── application-cloud.properties │ │ └── application.properties │ └── test │ └── java │ └── com │ └── example │ └── ZipkinClientApplicationTests.java ├── pom.xml ├── sleuth-basic ├── mvnw ├── mvnw.cmd ├── pom.xml ├── sleuth-basic.iml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── DemoApplication.java │ └── resources │ │ └── application.properties │ └── test │ └── java │ └── com │ └── example │ └── DemoApplicationTests.java ├── sleuth.md ├── tracing.iml ├── zipkin-query-service ├── manifest.yml ├── pom.xml ├── src │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── example │ │ │ │ └── ZipkinQueryServiceApplication.java │ │ └── resources │ │ │ ├── application-cloud.properties │ │ │ └── application.properties │ └── test │ │ └── java │ │ └── com │ │ └── example │ │ └── ZipkinQueryApplicationTests.java └── zipkin-query-service.iml └── zipkin-web ├── lib └── zipkin-web-all.jar ├── manifest.yml └── zipkin-web.sh /.gitignore: -------------------------------------------------------------------------------- 1 | ### Maven template 2 | target/ 3 | pom.xml.tag 4 | pom.xml.releaseBackup 5 | pom.xml.versionsBackup 6 | pom.xml.next 7 | release.properties 8 | dependency-reduced-pom.xml 9 | buildNumber.properties 10 | .mvn/timing.properties 11 | ### JetBrains template 12 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio 13 | 14 | *.iml 15 | 16 | ## Directory-based project format: 17 | .idea/ 18 | # if you remove the above rule, at least ignore the following: 19 | 20 | # User-specific stuff: 21 | # .idea/workspace.xml 22 | # .idea/tasks.xml 23 | # .idea/dictionaries 24 | 25 | # Sensitive or high-churn files: 26 | # .idea/dataSources.ids 27 | # .idea/dataSources.xml 28 | # .idea/sqlDataSources.xml 29 | # .idea/dynamic.xml 30 | # .idea/uiDesigner.xml 31 | 32 | # Gradle: 33 | # .idea/gradle.xml 34 | # .idea/libraries 35 | 36 | # Mongo Explorer plugin: 37 | # .idea/mongoSettings.xml 38 | 39 | ## File-based project format: 40 | *.ipr 41 | *.iws 42 | 43 | ## Plugin-specific files: 44 | 45 | # IntelliJ 46 | /out/ 47 | 48 | # mpeltonen/sbt-idea plugin 49 | .idea_modules/ 50 | 51 | # JIRA plugin 52 | atlassian-ide-plugin.xml 53 | 54 | # Crashlytics plugin (for Android Studio and IntelliJ) 55 | com_crashlytics_export_strings.xml 56 | crashlytics.properties 57 | crashlytics-build.properties 58 | 59 | ### Eclipse template 60 | *.pydevproject 61 | .metadata 62 | .gradle 63 | bin/ 64 | tmp/ 65 | *.tmp 66 | *.bak 67 | *.swp 68 | *~.nib 69 | local.properties 70 | .settings/ 71 | .loadpath 72 | 73 | # Eclipse Core 74 | .project 75 | 76 | # External tool builders 77 | .externalToolBuilders/ 78 | 79 | # Locally stored "Eclipse launch configurations" 80 | *.launch 81 | 82 | # CDT-specific 83 | .cproject 84 | 85 | # JDT-specific (Eclipse Java Development Tools) 86 | .classpath 87 | 88 | # Java annotation processor (APT) 89 | .factorypath 90 | 91 | # PDT-specific 92 | .buildpath 93 | 94 | # sbteclipse plugin 95 | .target 96 | 97 | # TeXlipse plugin 98 | .texlipse 99 | ### Java template 100 | *.class 101 | 102 | # Mobile Tools for Java (J2ME) 103 | .mtj.tmp/ 104 | 105 | # Package Files # 106 | *.jar 107 | *.war 108 | *.ear 109 | 110 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 111 | hs_err_pid* 112 | 113 | # Created by .ignore support plugin (hsz.mobi) 114 | -------------------------------------------------------------------------------- /eureka-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 7 | io.spring 8 | tracing 9 | 1.0.0.BUILD-SNAPSHOT 10 | 11 | eureka-service 12 | 13 | 14 | org.springframework.cloud 15 | spring-cloud-starter-eureka-server 16 | 17 | 18 | 19 | 20 | 21 | org.springframework.boot 22 | spring-boot-maven-plugin 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /eureka-service/src/main/java/com/example/EurekaServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.netflix.eureka.server.EnableEurekaServer; 6 | 7 | @EnableEurekaServer 8 | @SpringBootApplication 9 | public class EurekaServiceApplication { 10 | 11 | public static void main(String[] args) { 12 | SpringApplication.run(EurekaServiceApplication.class, args); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /eureka-service/src/main/resources/bootstrap.properties: -------------------------------------------------------------------------------- 1 | spring.cloud.config.uri=http://localhost:8888 2 | spring.application.name=eureka-service 3 | 4 | server.port=8761 5 | eureka.client.register-with-eureka=false 6 | eureka.client.fetch-registry=false -------------------------------------------------------------------------------- /eureka-service/src/test/java/com/example/EurekaServiceApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.SpringApplicationConfiguration; 6 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 7 | 8 | @RunWith(SpringJUnit4ClassRunner.class) 9 | @SpringApplicationConfiguration(classes = EurekaServiceApplication.class) 10 | public class EurekaServiceApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /images/feign.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joshlong-attic/sleuth-blog/3c353ce8a610e0e1c734c7902432ab6d3a1babae/images/feign.png -------------------------------------------------------------------------------- /images/template.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joshlong-attic/sleuth-blog/3c353ce8a610e0e1c734c7902432ab6d3a1babae/images/template.png -------------------------------------------------------------------------------- /images/zipkin-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joshlong-attic/sleuth-blog/3c353ce8a610e0e1c734c7902432ab6d3a1babae/images/zipkin-logo.jpg -------------------------------------------------------------------------------- /images/zipkin-ui-and-resttemplate-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joshlong-attic/sleuth-blog/3c353ce8a610e0e1c734c7902432ab6d3a1babae/images/zipkin-ui-and-resttemplate-detail.png -------------------------------------------------------------------------------- /images/zipkin-ui-and-resttemplate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joshlong-attic/sleuth-blog/3c353ce8a610e0e1c734c7902432ab6d3a1babae/images/zipkin-ui-and-resttemplate.png -------------------------------------------------------------------------------- /images/zipkin-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joshlong-attic/sleuth-blog/3c353ce8a610e0e1c734c7902432ab6d3a1babae/images/zipkin-ui.png -------------------------------------------------------------------------------- /message-client/manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | applications: 3 | - name: zipkin-client-a 4 | memory: 512M 5 | instances: 1 6 | host: zipkin-client-a-${random-word} 7 | path: target/zipkin-client-a.jar 8 | services: 9 | - cnj-rabbitmq 10 | - zipkin-client-b 11 | env: 12 | SPRING_PROFILES_ACTIVE: cloud 13 | DEBUG: "true" 14 | -------------------------------------------------------------------------------- /message-client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | io.spring 7 | tracing 8 | 1.0.0.BUILD-SNAPSHOT 9 | 10 | message-client 11 | 12 | 13 | org.springframework.cloud 14 | spring-cloud-starter-eureka 15 | 16 | 17 | org.springframework.cloud 18 | spring-cloud-starter-stream-rabbit 19 | 20 | 21 | org.springframework.boot 22 | spring-boot-starter-aop 23 | 24 | 25 | org.springframework.cloud 26 | spring-cloud-sleuth-stream 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-web 31 | 32 | 33 | org.springframework.cloud 34 | spring-cloud-starter-feign 35 | 36 | 37 | 38 | 39 | 40 | org.springframework.boot 41 | spring-boot-maven-plugin 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /message-client/src/main/java/com/example/MessageClientApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.autoconfigure.SpringBootApplication; 8 | import org.springframework.cloud.client.discovery.EnableDiscoveryClient; 9 | import org.springframework.cloud.netflix.feign.EnableFeignClients; 10 | import org.springframework.cloud.netflix.feign.FeignClient; 11 | import org.springframework.cloud.sleuth.Sampler; 12 | import org.springframework.cloud.stream.annotation.EnableBinding; 13 | import org.springframework.cloud.stream.messaging.Sink; 14 | import org.springframework.context.annotation.Bean; 15 | import org.springframework.context.annotation.EnableAspectJAutoProxy; 16 | import org.springframework.core.ParameterizedTypeReference; 17 | import org.springframework.http.HttpMethod; 18 | import org.springframework.http.MediaType; 19 | import org.springframework.http.ResponseEntity; 20 | import org.springframework.integration.annotation.*; 21 | import org.springframework.web.bind.annotation.RequestMapping; 22 | import org.springframework.web.bind.annotation.RequestMethod; 23 | import org.springframework.web.bind.annotation.RestController; 24 | import org.springframework.web.client.RestTemplate; 25 | 26 | import java.util.Map; 27 | 28 | @EnableDiscoveryClient 29 | @EnableFeignClients 30 | @EnableAspectJAutoProxy(proxyTargetClass = true) 31 | @EnableBinding(Sink.class) 32 | @IntegrationComponentScan 33 | @SpringBootApplication 34 | public class MessageClientApplication { 35 | 36 | /** 37 | * the service with which we're communicating 38 | */ 39 | public static final String ZIPKIN_CLIENT_B = "message-service"; 40 | 41 | @Bean 42 | Sampler sampler() { 43 | return span -> true; 44 | } 45 | 46 | public static void main(String[] args) { 47 | SpringApplication.run(MessageClientApplication.class, args); 48 | } 49 | } 50 | 51 | @MessageEndpoint 52 | class MessageProcessor { 53 | 54 | private Log log = LogFactory.getLog(getClass()); 55 | 56 | @ServiceActivator(inputChannel = Sink.INPUT) 57 | public void onMessage(String msg) { 58 | this.log.info("received message: '" + msg + "'."); 59 | } 60 | } 61 | 62 | @FeignClient(serviceId = MessageClientApplication.ZIPKIN_CLIENT_B) 63 | interface RestMessageReader { 64 | 65 | @RequestMapping( 66 | method = RequestMethod.GET, 67 | value = "/", 68 | consumes = MediaType.APPLICATION_JSON_VALUE) 69 | Map readMessage(); 70 | } 71 | 72 | @RestController 73 | @RequestMapping("/message") 74 | class MessageClientRestController { 75 | 76 | @Autowired 77 | private RestTemplate restTemplate; 78 | 79 | @Autowired 80 | private RestMessageReader restReader; 81 | 82 | 83 | @RequestMapping("/template") 84 | ResponseEntity> template() { 85 | 86 | String url = "http://" + MessageClientApplication.ZIPKIN_CLIENT_B; 87 | 88 | ParameterizedTypeReference> ptr = 89 | new ParameterizedTypeReference>() { 90 | }; 91 | 92 | ResponseEntity> responseEntity = 93 | this.restTemplate.exchange(url, HttpMethod.GET, null, ptr); 94 | 95 | return ResponseEntity 96 | .ok() 97 | .contentType(responseEntity.getHeaders().getContentType()) 98 | .body(responseEntity.getBody()); 99 | } 100 | 101 | @RequestMapping("/feign") 102 | Map feign() { 103 | return this.restReader.readMessage(); 104 | } 105 | } 106 | 107 | 108 | -------------------------------------------------------------------------------- /message-client/src/main/resources/application-cloud.properties: -------------------------------------------------------------------------------- 1 | spring.rabbitmq.addresses=${vcap.services.cnj-rabbitmq.credentials.uri} 2 | -------------------------------------------------------------------------------- /message-client/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=8082 2 | 3 | spring.application.name=message-client 4 | 5 | 6 | logging.level.com.netflix.eureka=OFF 7 | logging.level.com.netflix.discovery=OFF 8 | spring.cloud.stream.bindings.input=messages -------------------------------------------------------------------------------- /message-client/src/test/java/com/example/ZipkinClientApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.SpringApplicationConfiguration; 6 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 7 | 8 | @RunWith(SpringJUnit4ClassRunner.class) 9 | @SpringApplicationConfiguration(classes = MessageClientApplication.class) 10 | public class ZipkinClientApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /message-service/manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | applications: 3 | - name: zipkin-client-b 4 | memory: 512M 5 | instances: 1 6 | host: zipkin-client-b-${random-word} 7 | path: target/zipkin-client-b.jar 8 | services: 9 | - cnj-rabbitmq 10 | env: 11 | SPRING_PROFILES_ACTIVE: cloud 12 | DEBUG: "true" -------------------------------------------------------------------------------- /message-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 4.0.0 6 | 7 | 8 | io.spring 9 | tracing 10 | 1.0.0.BUILD-SNAPSHOT 11 | 12 | 13 | message-service 14 | 15 | 16 | 17 | org.springframework.cloud 18 | spring-cloud-starter-stream-rabbit 19 | 20 | 21 | org.springframework.boot 22 | spring-boot-starter-aop 23 | 24 | 25 | org.springframework.cloud 26 | spring-cloud-sleuth-stream 27 | 28 | 29 | org.springframework.cloud 30 | spring-cloud-starter-eureka 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-web 35 | 36 | 37 | 38 | 39 | 40 | org.springframework.boot 41 | spring-boot-maven-plugin 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /message-service/src/main/java/com/example/MessageServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.cloud.client.discovery.EnableDiscoveryClient; 7 | import org.springframework.cloud.sleuth.Sampler; 8 | import org.springframework.cloud.stream.annotation.EnableBinding; 9 | import org.springframework.cloud.stream.messaging.Source; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.context.annotation.EnableAspectJAutoProxy; 12 | import org.springframework.integration.annotation.*; 13 | import org.springframework.web.bind.annotation.RequestMapping; 14 | import org.springframework.web.bind.annotation.RestController; 15 | 16 | import javax.servlet.http.HttpServletRequest; 17 | import java.util.Arrays; 18 | import java.util.HashMap; 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | @EnableDiscoveryClient 23 | @EnableAspectJAutoProxy(proxyTargetClass = true) 24 | @IntegrationComponentScan 25 | @SpringBootApplication 26 | @EnableBinding(Source.class) 27 | public class MessageServiceApplication { 28 | 29 | @Bean 30 | Sampler sampler() { 31 | return span -> true; 32 | } 33 | 34 | public static void main(String[] args) { 35 | SpringApplication.run(MessageServiceApplication.class, args); 36 | } 37 | } 38 | 39 | 40 | @MessagingGateway 41 | interface ReplySender { 42 | 43 | @Gateway(requestChannel = Source.OUTPUT) 44 | void sendMessage(String msg); 45 | } 46 | 47 | 48 | @RestController 49 | class MessageServiceRestController { 50 | 51 | @Autowired 52 | ReplySender replySender; 53 | 54 | @RequestMapping("/") 55 | Map message(HttpServletRequest httpRequest) { 56 | 57 | List headers = Arrays.asList("x-span-id", 58 | "x-span-name", "x-trace-id"); 59 | 60 | String key = "message"; 61 | 62 | Map response = new HashMap<>(); 63 | 64 | String value = "Hi, from a REST endpoint: " + System.currentTimeMillis(); 65 | 66 | response.put(key, value); 67 | 68 | headers 69 | .stream() 70 | .filter(h -> httpRequest.getHeader(h) != null) 71 | .forEach(h -> response.put(h, httpRequest.getHeader(h))); 72 | 73 | this.replySender.sendMessage(value); 74 | 75 | return response; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /message-service/src/main/resources/application-cloud.properties: -------------------------------------------------------------------------------- 1 | spring.rabbitmq.addresses=${vcap.services.cnj-rabbitmq.credentials.uri} 2 | -------------------------------------------------------------------------------- /message-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=8081 2 | 3 | spring.application.name=message-service 4 | 5 | logging.level.com.netflix.eureka=OFF 6 | logging.level.com.netflix.discovery=OFF 7 | 8 | spring.cloud.stream.bindings.output=messages -------------------------------------------------------------------------------- /message-service/src/test/java/com/example/ZipkinClientApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.SpringApplicationConfiguration; 6 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 7 | 8 | @RunWith(SpringJUnit4ClassRunner.class) 9 | @SpringApplicationConfiguration(classes = MessageServiceApplication.class) 10 | public class ZipkinClientApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 1.3.2.RELEASE 9 | 10 | 11 | 1.0.0.BUILD-SNAPSHOT 12 | io.spring 13 | tracing 14 | pom 15 | 16 | sleuth-basic 17 | message-client 18 | message-service 19 | zipkin-query-service 20 | eureka-service 21 | 22 | 23 | 1.8 24 | UTF-8 25 | 26 | 27 | ${project.artifactId} 28 | 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-starter-test 33 | test 34 | 35 | 36 | 37 | 38 | 39 | org.springframework.cloud 40 | spring-cloud-dependencies 41 | Brixton.BUILD-SNAPSHOT 42 | pom 43 | import 44 | 45 | 46 | 47 | 48 | 49 | spring-snapshots 50 | Spring Snapshots 51 | https://repo.spring.io/snapshot 52 | 53 | true 54 | 55 | 56 | 57 | spring-milestones 58 | Spring Milestones 59 | https://repo.spring.io/milestone 60 | 61 | false 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /sleuth-basic/mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # 58 | # Look for the Apple JDKs first to preserve the existing behaviour, and then look 59 | # for the new JDKs provided by Oracle. 60 | # 61 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then 62 | # 63 | # Apple JDKs 64 | # 65 | export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home 66 | fi 67 | 68 | if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then 69 | # 70 | # Apple JDKs 71 | # 72 | export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 73 | fi 74 | 75 | if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then 76 | # 77 | # Oracle JDKs 78 | # 79 | export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home 80 | fi 81 | 82 | if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then 83 | # 84 | # Apple JDKs 85 | # 86 | export JAVA_HOME=`/usr/libexec/java_home` 87 | fi 88 | ;; 89 | esac 90 | 91 | if [ -z "$JAVA_HOME" ] ; then 92 | if [ -r /etc/gentoo-release ] ; then 93 | JAVA_HOME=`java-config --jre-home` 94 | fi 95 | fi 96 | 97 | if [ -z "$M2_HOME" ] ; then 98 | ## resolve links - $0 may be a link to maven's home 99 | PRG="$0" 100 | 101 | # need this for relative symlinks 102 | while [ -h "$PRG" ] ; do 103 | ls=`ls -ld "$PRG"` 104 | link=`expr "$ls" : '.*-> \(.*\)$'` 105 | if expr "$link" : '/.*' > /dev/null; then 106 | PRG="$link" 107 | else 108 | PRG="`dirname "$PRG"`/$link" 109 | fi 110 | done 111 | 112 | saveddir=`pwd` 113 | 114 | M2_HOME=`dirname "$PRG"`/.. 115 | 116 | # make it fully qualified 117 | M2_HOME=`cd "$M2_HOME" && pwd` 118 | 119 | cd "$saveddir" 120 | # echo Using m2 at $M2_HOME 121 | fi 122 | 123 | # For Cygwin, ensure paths are in UNIX format before anything is touched 124 | if $cygwin ; then 125 | [ -n "$M2_HOME" ] && 126 | M2_HOME=`cygpath --unix "$M2_HOME"` 127 | [ -n "$JAVA_HOME" ] && 128 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 129 | [ -n "$CLASSPATH" ] && 130 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 131 | fi 132 | 133 | # For Migwn, ensure paths are in UNIX format before anything is touched 134 | if $mingw ; then 135 | [ -n "$M2_HOME" ] && 136 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 137 | [ -n "$JAVA_HOME" ] && 138 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 139 | # TODO classpath? 140 | fi 141 | 142 | if [ -z "$JAVA_HOME" ]; then 143 | javaExecutable="`which javac`" 144 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 145 | # readlink(1) is not available as standard on Solaris 10. 146 | readLink=`which readlink` 147 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 148 | if $darwin ; then 149 | javaHome="`dirname \"$javaExecutable\"`" 150 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 151 | else 152 | javaExecutable="`readlink -f \"$javaExecutable\"`" 153 | fi 154 | javaHome="`dirname \"$javaExecutable\"`" 155 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 156 | JAVA_HOME="$javaHome" 157 | export JAVA_HOME 158 | fi 159 | fi 160 | fi 161 | 162 | if [ -z "$JAVACMD" ] ; then 163 | if [ -n "$JAVA_HOME" ] ; then 164 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 165 | # IBM's JDK on AIX uses strange locations for the executables 166 | JAVACMD="$JAVA_HOME/jre/sh/java" 167 | else 168 | JAVACMD="$JAVA_HOME/bin/java" 169 | fi 170 | else 171 | JAVACMD="`which java`" 172 | fi 173 | fi 174 | 175 | if [ ! -x "$JAVACMD" ] ; then 176 | echo "Error: JAVA_HOME is not defined correctly." >&2 177 | echo " We cannot execute $JAVACMD" >&2 178 | exit 1 179 | fi 180 | 181 | if [ -z "$JAVA_HOME" ] ; then 182 | echo "Warning: JAVA_HOME environment variable is not set." 183 | fi 184 | 185 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 186 | 187 | # For Cygwin, switch paths to Windows format before running java 188 | if $cygwin; then 189 | [ -n "$M2_HOME" ] && 190 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 191 | [ -n "$JAVA_HOME" ] && 192 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 193 | [ -n "$CLASSPATH" ] && 194 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 195 | fi 196 | 197 | # traverses directory structure from process work directory to filesystem root 198 | # first directory with .mvn subdirectory is considered project base directory 199 | find_maven_basedir() { 200 | local basedir=$(pwd) 201 | local wdir=$(pwd) 202 | while [ "$wdir" != '/' ] ; do 203 | if [ -d "$wdir"/.mvn ] ; then 204 | basedir=$wdir 205 | break 206 | fi 207 | wdir=$(cd "$wdir/.."; pwd) 208 | done 209 | echo "${basedir}" 210 | } 211 | 212 | # concatenates all lines of a file 213 | concat_lines() { 214 | if [ -f "$1" ]; then 215 | echo "$(tr -s '\n' ' ' < "$1")" 216 | fi 217 | } 218 | 219 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} 220 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 221 | 222 | # Provide a "standardized" way to retrieve the CLI args that will 223 | # work with both Windows and non-Windows executions. 224 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" 225 | export MAVEN_CMD_LINE_ARGS 226 | 227 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 228 | 229 | exec "$JAVACMD" \ 230 | $MAVEN_OPTS \ 231 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 232 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 233 | ${WRAPPER_LAUNCHER} "$@" 234 | -------------------------------------------------------------------------------- /sleuth-basic/mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto init 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | @REM ==== END VALIDATION ==== 80 | 81 | :init 82 | 83 | set MAVEN_CMD_LINE_ARGS=%* 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | 121 | set WRAPPER_JAR="".\.mvn\wrapper\maven-wrapper.jar"" 122 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 123 | 124 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CMD_LINE_ARGS% 125 | if ERRORLEVEL 1 goto error 126 | goto end 127 | 128 | :error 129 | set ERROR_CODE=1 130 | 131 | :end 132 | @endlocal & set ERROR_CODE=%ERROR_CODE% 133 | 134 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 135 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 136 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 137 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 138 | :skipRcPost 139 | 140 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 141 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 142 | 143 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 144 | 145 | exit /B %ERROR_CODE% -------------------------------------------------------------------------------- /sleuth-basic/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | io.spring 7 | tracing 8 | 1.0.0.BUILD-SNAPSHOT 9 | 10 | sleuth-basic 11 | 12 | UTF-8 13 | 1.8 14 | 15 | 16 | 17 | org.springframework.cloud 18 | spring-cloud-starter-sleuth 19 | 20 | 21 | org.apache.commons 22 | commons-lang3 23 | 3.4 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-starter-web 28 | 29 | 30 | 31 | 32 | 33 | 34 | org.springframework.boot 35 | spring-boot-maven-plugin 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /sleuth-basic/sleuth-basic.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /sleuth-basic/src/main/java/com/example/DemoApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.autoconfigure.SpringBootApplication; 8 | import org.springframework.cloud.sleuth.Sampler; 9 | import org.springframework.cloud.sleuth.Span; 10 | import org.springframework.cloud.sleuth.SpanAccessor; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.http.MediaType; 13 | import org.springframework.http.ResponseEntity; 14 | import org.springframework.web.bind.annotation.PathVariable; 15 | import org.springframework.web.bind.annotation.RequestMapping; 16 | import org.springframework.web.bind.annotation.RequestMethod; 17 | import org.springframework.web.bind.annotation.RestController; 18 | import org.springframework.web.client.RestTemplate; 19 | 20 | import java.net.URI; 21 | 22 | @SpringBootApplication 23 | public class DemoApplication { 24 | 25 | public static void main(String[] args) { 26 | SpringApplication.run(DemoApplication.class, args); 27 | } 28 | } 29 | 30 | @RestController 31 | class SleuthMessageRestController { 32 | 33 | private final RestTemplate restTemplate; 34 | private final SpanAccessor spanAccessor; 35 | 36 | private Log log = LogFactory.getLog(getClass()); 37 | 38 | @Autowired 39 | SleuthMessageRestController(RestTemplate restTemplate, 40 | SpanAccessor spanAccessor) { 41 | this.restTemplate = restTemplate; 42 | this.spanAccessor = spanAccessor; 43 | } 44 | 45 | @RequestMapping(method = RequestMethod.GET, value = "/{domainDotCom}") 46 | ResponseEntity call(@PathVariable String domainDotCom) { 47 | try { 48 | URI uri = URI.create("http://" + domainDotCom + ".com"); 49 | ResponseEntity response = this.restTemplate.getForEntity(uri, String.class); 50 | String body = response.getBody(); 51 | MediaType mediaType = response.getHeaders().getContentType(); 52 | return ResponseEntity.ok() 53 | .contentType(mediaType) 54 | .body(body); 55 | } finally { 56 | debug(); 57 | } 58 | } 59 | 60 | private void debug() { 61 | Span span = this.spanAccessor.getCurrentSpan(); 62 | this.log.info(String.format("traceId: %s, spanId: %s", 63 | span.getTraceId(), span.getSpanId())); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /sleuth-basic/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.sleuth.sampler.percentage=0.1 -------------------------------------------------------------------------------- /sleuth-basic/src/test/java/com/example/DemoApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.SpringApplicationConfiguration; 6 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 7 | 8 | @RunWith(SpringJUnit4ClassRunner.class) 9 | @SpringApplicationConfiguration(classes = DemoApplication.class) 10 | public class DemoApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /sleuth.md: -------------------------------------------------------------------------------- 1 | > I was inspired by OpenZipkin contributor and Spring Cloud Sleuth and Zipkin contributor [Adrian Cole](https://twitter.com/adrianfcole)'s [epic talk introducing Zipkin](https://www.youtube.com/watch?v=f9J1Av8rwCE) to write this particular post. There's so much epic stuff in that talk, so once you're done reading this, go watch that! 2 | 3 | Advances in technology and cloud computing have made it easier to stand up and deploy services with ease. Cloud computing enables us to automate away the pain (from days or weeks (gasp!) to minutes!) associated with standing up new services. This increase in velocity in turn enables us to be more agile, to think about smaller batches of independently deployable services. The proliferation of new services complicates reasoning about system-wide and request-specific performance characteristics. 4 | 5 | When all of an application's functionality lives in a _monolith_ - what we call applications written as one, large, unbroken deployable like a `.war` or `.ear` - it's much easier to reason about where things have gone wrong. Is there a memory leak? It's in the monolith. Is a component not handling requests correctly? It's in the monolith. Messages getting dropped? Also, probably in the monolith. Distribution changes everything. 6 | 7 | Systems behave differently under load and at scale. The specification of a system's behavior often diverges from the actual behavior of the system, and the actual behavior may itself vary in different contexts. It is important to contextualize requests as they transit through a system. It's also important to be able to talk about the nature of a specific request and to be able to understand that specific request's behavior relative to the general behavior of similar requests in the past minute, hour, day (or whatever!) other useful interval provides a statistically significant sampling. Context helps us establish whether a request was abnormal and whether it merits attention. You can't trace bugs in a system until you've established a baseline for what _normal_ is. How long is is _long_? For some systems it might be microseconds, for others it might be seconds or minutes! 8 | 9 | In this post, we'll look at how Spring Cloud Sleuth, which supports distributed tracing, can help us establish this context and helps us better understand a system's actual behavior, not just its specified behavior. 10 | 11 | ## Finding Clues with Spring Cloud Sleuth 12 | 13 | Tracing is simple, in theory. As a request flows from one component to another in a system, through ingress and egress points, **tracers** add logic where possible to perpetuate a unique **trace ID** that's generated when the first request is made. As a request arrives at a component along its journey, a new **span ID** is assigned for that component and added to the trace. A trace represents the whole journey of a request, and a span is each individual hop along the way, each request. Spans may contain **tags**, or metadata, that can be used to later contextualize the request. Spans typically contain common tags like start timestamps and stop timestamp, though it's easy to associate semantically relevant tags like an a business entity ID with a span. 14 | 15 | [Spring Cloud Sleuth](http://cloud.spring.io/spring-cloud-sleuth/) (`org.springframework.cloud`:`spring-cloud-starter-sleuth`), once added to the CLASSPATH, automatically instruments common communication channels: 16 | 17 | - requests over messaging technologies like [Apache Kafka](https://spring.io/blog/2015/04/15/using-apache-kafka-for-integration-and-data-processing-pipelines-with-spring) or RabbitMQ (or any other [Spring Cloud Stream](http://cloud.spring.io/spring-cloud-stream/) binder 18 | - HTTP headers received at Spring MVC controllers 19 | - requests that pass through a Netflix Zuul microproxy 20 | - requests made with the `RestTemplate`, etc. 21 | 22 | Spring Cloud Sleuth sets up useful log formatting for you that prints the trace ID and the span ID. Assuming you're running Spring Cloud Sleuth-enabled code in a microservice whose `spring.application.name` is `my-service-id`, you will see something like this in the logging for your microservice: 23 | 24 | ``` 25 | 2016-02-11 17:12:45.404 INFO [my-service-id,73b62c0f90d11e06,73b62c0f90d11e06,false] 85184 --- [nio-8080-exec-1] com.example.MySimpleComponentMakingARequest : ... 26 | ``` 27 | 28 | In that example, `my-service-id` is the `spring.application.name`, `73b62c0f90d11e06` is the trace ID and `73b62c0f90d11e06` is the span ID. This information is very useful. You can publish your logs to log analysis and manipulation tools like Elasticsearch and Splunk. There are various ways to get that data there. Logstash, for example, is a log publisher that will write to ElasticSearch. Cloud Foundry automatically aggregates logs from all instances of a service into a unified log [through a tool called the Loggregator](https://github.com/cloudfoundry/loggregator) that can then be forwarded to any [Syslog](https://en.wikipedia.org/wiki/Syslog)-compatible service, including tools like [Splunk](http://www.splunk.com/) or [PaperTrail](https://papertrailapp.com/). Whatever approach you take, you can do interesting queries if you have all the logs, and the trace information, in a single place available for query and analysis. 29 | 30 | Spring Cloud Sleuth also makes this information available to any Spring Cloud Sleuth-aware Spring application by simply injecting the `SpanAccessor`. You can also use this to instrument your own components that aren't already instrumented by Spring Cloud so that they can perpetuate trace information. Naturally, each tracer is going to be different, but Spring Cloud Sleuth's code itself (e.g.: [`TraceFeignClientAutoConfiguration`](https://github.com/spring-cloud/spring-cloud-sleuth/blob/master/spring-cloud-sleuth-core/src/main/java/org/springframework/cloud/sleuth/instrument/web/client/TraceFeignClientAutoConfiguration.java)) is suggestive of how a typical tracer works: 31 | 32 | ```java 33 | ... 34 | @Autowired 35 | private SpanAccessor spanAccessor; 36 | 37 | ... 38 | Span span = this.spanAccessor.getCurrentSpan(); 39 | ... 40 | template.header(Span.TRACE_ID_NAME, Span.toHex(span.getTraceId())); 41 | setHeader(template, Span.SPAN_NAME_NAME, span.getName().toString()); 42 | setHeader(template, Span.SPAN_ID_NAME, Span.toHex(span.getSpanId())); 43 | ... 44 | ``` 45 | 46 | 47 | 48 | ## How Much Data is Enough? 49 | 50 | Which requests should be traced? Ideally, you'll want enough data to see trends reflective of live, operational traffic. You don't want to overwhelm your logging and analysis infrastructure, though. Some organizations may only need request for every thousand requests, or every ten, or every million! By default, the threshold is 10%, or .1, though you may override it by specifying a sampling percentage: 51 | 52 | ```properties 53 | spring.sleuth.sampler.percentage = 0.2 54 | ``` 55 | 56 | Alternatively, you may register your own `Sampler` bean definition and programmatically make the decision which requests should be sampled. You can make more intelligent choices about which things to trace, for example, by ignoring successful requests, perhaps checking whether some component is in an error state, or really anything else. The `Span` given as an argument represents the span for the current in-flight request in the larger trace. You can do interesting and request-type specific types of sampling if you'd like. You might decide to only sample requests that have a 500 HTTP status code, for example. The following `Sampler`, for example, would trace roughly half of all requests: 57 | 58 | ```java 59 | @Bean 60 | Sampler customSampler() { 61 | return new Sampler() { 62 | @Override 63 | public boolean isSampled(Span span) { 64 | return Math.random() > .5 ; 65 | } 66 | }; 67 | } 68 | ``` 69 | 70 | Make sure to set realistic expectations for your application and infrastructure. It may well be that the usage patterns for your applications require something more sensitive or less sensitive to detect trends and patterns. This is meant to be operational data; most organizations don't warehouse this data more than a few days or, at the upper bound, a week. 71 | 72 | ## Seeing the Big Picture with OpenZipkin 73 | 74 | 75 | 76 | Data collection is a start but the goal is to _understand_ the data, not just collect it. In order to appreciate the big picture, we need to get beyond individual events. 77 | 78 | For this we'll use [the OpenZipkin project](http://zipkin.io/). OpenZipkin is the fully open-source version of Zipkin, a project that originated at Twitter in 2010, and is based [on the Google Dapper papers](http://research.google.com/pubs/pub36356.html). 79 | 80 | > Previously, the open-source version of Zipkin evolved at a different pace than the version used internally at Twitter. OpenZipkin represents the synchronization of those efforts: [OpenZipkin](http://github.com/openzipkin) _is_ Zipkin and when we refer to Zipkin in this post, we're referring to the version reflected in OpenZipkin. 81 | 82 | Zipkin provides a REST API that clients talk to directly. Zipkin even supports a Spring Boot-based implementation of this REST API. Using that is as simple as using Zipkin's `@EnableZipkinServer` directly. The Zipkin Server delegates writes to the persistence tier via a `SpanStore`. Presently, there is support for using MySQL or an in-memory `SpanStore` out-of-the-box. As an alternative to REST, we can _also_ publish messages to the Zipkin server over a Spring Cloud Stream binder like RabbitMQ or Apache Kafka. We'll use this option, and `org.springframework.cloud`:`spring-cloud-sleuth-zipkin-stream`'s `@EnableZipkinStreamServer`, to adapt incoming Spring Cloud Stream-based Sleuth `Span`s into Zipkin's `Span`s and then persist them using the `SpanStore`. You may use whatever Spring Cloud Stream binding you like, but in this case we'll use Spring Cloud Stream RabbitMQ (`org.springframework.cloud`:`spring-cloud-starter-stream-rabbitmq`). 83 | 84 | 85 | ```java 86 | package com.example; 87 | 88 | import org.springframework.boot.SpringApplication; 89 | import org.springframework.boot.autoconfigure.SpringBootApplication; 90 | import org.springframework.cloud.sleuth.zipkin.stream.EnableZipkinStreamServer; 91 | 92 | @EnableZipkinStreamServer 93 | @SpringBootApplication 94 | public class ZipkinQueryServiceApplication { 95 | 96 | public static void main(String[] args) { 97 | SpringApplication.run(ZipkinQueryServiceApplication.class, args); 98 | } 99 | } 100 | ``` 101 | 102 | Specify a port in `application.properties` to pin the Zipkin server to a well-known port so that the UI may later talk to it: 103 | 104 | ``` 105 | server.port=9411 106 | ``` 107 | 108 | My [example code](https://github.com/joshlong/sleuth-blog) uses the [Spring Cloud Netflix-powered Eureka service registry in `eureka-servie`](https://github.com/joshlong/sleuth-blog/tree/master/eureka-service) for service registration and discovery, so start that up next. 109 | 110 | Our microservices ([`message-client`](https://github.com/joshlong/sleuth-blog/tree/master/message-client) and [`message-service`](https://github.com/joshlong/sleuth-blog/tree/master/message-service)) are typical Spring Cloud microservices. I only added `org.springframework.cloud`:`spring-cloud-sleuth-stream` and the appropriate Spring Cloud Stream binder to have their Sleuth traces published, out-of-band, to Zipkin for analysis. 111 | 112 | The Zipkin web UI makes it easy to analyze and query Zipkin data. You can run the Zipkin Web [build in my example](https://github.com/joshlong/sleuth-blog/blob/master/zipkin-web/zipkin-web.sh) or just grab the [latest from the Zipkin project's builds](http://oss.jfrog.org/artifactory/oss-snapshot-local/io/zipkin/zipkin-web/1.28.1-SNAPSHOT/) and then run it: 113 | 114 | ```bash 115 | java -jar lib/zipkin-web-all.jar -zipkin.web.port=:9412 -zipkin.web.rootUrl=/ -zipkin.web.query.dest=localhost:9411 116 | ``` 117 | 118 | When the `message-service` receives a request it sends a reply message _back_ to the `message-client` over a Spring Cloud Stream binder which the client then accepts and logs with a Spring Integration messaging endpoint. This is a contrived call sequence to demonstrate some of Spring Cloud Sleuth's power. 119 | 120 | Bring up the UI and then find all the recent traces. You can sort by most recent, longest, etc., for finer-grained control over which results you see. 121 | 122 | 123 | 124 | When I click on one of the traces that comes back, I get a UI that looks like this: 125 | 126 | 127 | 128 | Each individual span also carries with it information (**_tag_**s) about the particular request with which its associated. You can view this detail by clicking on an individual span: 129 | 130 | 131 | 132 | ## The OpenTracing Initiative 133 | 134 | For Spring-based workloads, distributed tracing couldn't be easier! However, tracing, by its very nature, is a cross-cutting concern for all services no matter which technology stack they're implemented in. [The OpenTracing initiative](https://github.com/opentracing) is an effort to standardize the vocabulary and concepts of modern tracing for multiple languages and platforms. The OpenTracing API has support from multiple _very_ large organizations as its lead one of the original authors on the original Google Dapper paper. The effort defines language bindings; there are already implementations for JavaScript, Python, Go, etc. We will keep Spring Cloud Sleuth conceptually compatible with this effort and will track it. It is expected, but not implied, that the bindings will as often as not have Zipkin as their backend. 135 | 136 | ## Next Steps 137 | 138 | This blog was meant to skim the surface of the concepts and supporting technologies in distributed tracing. We've looked at Spring Cloud Sleuth and how Spring Cloud Sleuth works with Zipkin. Zipkin itself has an interesting supporting ecosystem. If you really want to understand what distributed tracing tools like Spring Cloud Sleuth, Zipkin, Apache's HTrace and others are modeled after, check [out the original Google Dapper paper](http://research.google.com/pubs/pub36356.html). You should check out [Adrian Cole's introduction to the larger Zipkin ecosystem](https://www.youtube.com/watch?v=f9J1Av8rwCE). Naturally, the code for this blog is also online](https://github.com/joshlong/sleuth-blog). Finally, look no further than [the Spring Initializr](http://start.spring.io) and add the Spring Cloud Sleuth Stream and Zipkin Stream Server to your Maven builds to get started. 139 | -------------------------------------------------------------------------------- /tracing.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /zipkin-query-service/manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | applications: 3 | - name: zipkin-query-service 4 | memory: 512M 5 | instances: 1 6 | host: zipkin-query-service-${random-word} 7 | path: target/zipkin-query-service.jar 8 | services: 9 | - cnj-rabbitmq 10 | - cnj-mysql 11 | env: 12 | SPRING_PROFILES_ACTIVE: cloud 13 | DEBUG: "true" 14 | debug: "true" -------------------------------------------------------------------------------- /zipkin-query-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | io.spring 7 | tracing 8 | 1.0.0.BUILD-SNAPSHOT 9 | 10 | zipkin-query-service 11 | 12 | 13 | 14 | org.springframework.cloud 15 | spring-cloud-sleuth-zipkin-stream 16 | 17 | 18 | org.springframework.cloud 19 | spring-cloud-starter-stream-rabbit 20 | 21 | 29 | 30 | 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-maven-plugin 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /zipkin-query-service/src/main/java/com/example/ZipkinQueryServiceApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.sleuth.zipkin.stream.EnableZipkinStreamServer; 6 | 7 | @EnableZipkinStreamServer 8 | @SpringBootApplication 9 | public class ZipkinQueryServiceApplication { 10 | 11 | 12 | public static void main(String[] args) { 13 | SpringApplication.run(ZipkinQueryServiceApplication.class, args); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /zipkin-query-service/src/main/resources/application-cloud.properties: -------------------------------------------------------------------------------- 1 | spring.rabbitmq.addresses=${vcap.services.cnj-rabbitmq.credentials.uri} 2 | -------------------------------------------------------------------------------- /zipkin-query-service/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9411 2 | #zipkin.store.type=mem 3 | #spring.datasource.initialize=false 4 | -------------------------------------------------------------------------------- /zipkin-query-service/src/test/java/com/example/ZipkinQueryApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.SpringApplicationConfiguration; 6 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 7 | 8 | @RunWith(SpringJUnit4ClassRunner.class) 9 | @SpringApplicationConfiguration(classes = ZipkinQueryServiceApplication.class) 10 | public class ZipkinQueryApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /zipkin-query-service/zipkin-query-service.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | -------------------------------------------------------------------------------- /zipkin-web/lib/zipkin-web-all.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joshlong-attic/sleuth-blog/3c353ce8a610e0e1c734c7902432ab6d3a1babae/zipkin-web/lib/zipkin-web-all.jar -------------------------------------------------------------------------------- /zipkin-web/manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | applications: 3 | - name: zipkin-web 4 | memory: 1024M 5 | host: zipkin-web-${random-word} 6 | path: lib/zipkin-web-all.jar -------------------------------------------------------------------------------- /zipkin-web/zipkin-web.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | java -jar lib/zipkin-web-all.jar -zipkin.web.port=:9412 -zipkin.web.rootUrl=/ -zipkin.web.query.dest=localhost:9411 --------------------------------------------------------------------------------