├── .github ├── CONTRIBUTING.md └── ISSUE_TEMPLATE.md ├── .gitignore ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── .settings.xml ├── LICENSE ├── NOTICE.md ├── README.md ├── RELEASE.md ├── adjuster ├── finagle │ ├── README.md │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── zipkin │ │ │ └── sparkstreaming │ │ │ └── adjuster │ │ │ └── finagle │ │ │ ├── FinagleAdjuster.java │ │ │ └── FinagleIssue343Adjuster.java │ │ └── test │ │ └── java │ │ └── zipkin │ │ └── sparkstreaming │ │ └── adjuster │ │ └── finagle │ │ ├── FinagleAdjusterTest.java │ │ └── FinagleIssue343AdjusterTest.java └── pom.xml ├── autoconfigure ├── adjuster-finagle │ ├── README.md │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── zipkin │ │ │ │ └── sparkstreaming │ │ │ │ └── autoconfigure │ │ │ │ └── adjuster │ │ │ │ └── finagle │ │ │ │ ├── ZipkinFinagleAdjusterAutoConfiguration.java │ │ │ │ └── ZipkinFinagleAdjusterProperties.java │ │ └── resources │ │ │ └── META-INF │ │ │ └── spring.factories │ │ └── test │ │ └── java │ │ └── zipkin │ │ └── sparkstreaming │ │ └── adjuster │ │ └── finagle │ │ └── ZipkinFinagleAdjusterAutoConfigurationTest.java ├── consumer-storage │ ├── README.md │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── zipkin │ │ │ │ └── sparkstreaming │ │ │ │ └── autoconfigure │ │ │ │ └── consumer │ │ │ │ └── storage │ │ │ │ └── ZipkinStorageConsumerAutoConfiguration.java │ │ └── resources │ │ │ └── META-INF │ │ │ └── spring.factories │ │ └── test │ │ └── java │ │ └── zipkin │ │ └── sparkstreaming │ │ └── autoconfigure │ │ └── consumer │ │ └── storage │ │ └── ZipkinStorageConsumerAutoConfigurationTest.java ├── pom.xml └── stream-kafka │ ├── README.md │ ├── pom.xml │ └── src │ ├── main │ ├── java │ │ └── zipkin │ │ │ └── sparkstreaming │ │ │ └── autoconfigure │ │ │ └── stream │ │ │ └── kafka │ │ │ ├── ZipkinKafkaStreamFactoryAutoConfiguration.java │ │ │ └── ZipkinKafkaStreamFactoryProperties.java │ └── resources │ │ └── META-INF │ │ └── spring.factories │ └── test │ └── java │ └── zipkin │ └── sparkstreaming │ ├── autoconfigure │ └── stream │ │ └── kafka │ │ └── ZipkinKafkaStreamFactoryPropertiesTest.java │ └── stream │ └── kafka │ └── ZipkinKafkaStreamFactoryAutoConfigurationTest.java ├── build-support ├── go-offline.sh ├── pom-no-crossmodule-dependencies.xsl ├── publish-snapshot.sh ├── publish-stable.sh └── trigger-publish.sh ├── circle.yml ├── consumer ├── pom.xml └── storage │ ├── README.md │ ├── pom.xml │ └── src │ ├── main │ └── java │ │ └── zipkin │ │ └── sparkstreaming │ │ └── consumer │ │ └── storage │ │ └── StorageConsumer.java │ └── test │ └── java │ └── zipkin │ └── sparkstreaming │ └── consumer │ └── storage │ └── StorageConsumerTest.java ├── mvnw ├── mvnw.cmd ├── pom.xml ├── sparkstreaming-job ├── README.md ├── pom.xml └── src │ ├── main │ ├── java │ │ └── zipkin │ │ │ └── sparkstreaming │ │ │ └── job │ │ │ ├── ZipkinSparkStreamingConfiguration.java │ │ │ ├── ZipkinSparkStreamingJob.java │ │ │ └── ZipkinSparkStreamingProperties.java │ └── resources │ │ └── log4j.properties │ └── test │ └── java │ └── zipkin │ └── sparkstreaming │ └── ZipkinSparkStreamingJobAutoConfigurationTest.java ├── sparkstreaming ├── pom.xml └── src │ ├── main │ └── java │ │ └── zipkin │ │ └── sparkstreaming │ │ ├── AdjustAndConsumeSpansSharingTraceId.java │ │ ├── Adjuster.java │ │ ├── Consumer.java │ │ ├── LogInitializer.java │ │ ├── ReadSpans.java │ │ ├── SparkStreamingJob.java │ │ └── StreamFactory.java │ └── test │ └── java │ └── zipkin │ └── sparkstreaming │ └── AdjusterTest.java ├── src └── etc │ └── header.txt └── stream ├── kafka ├── README.md ├── pom.xml └── src │ ├── main │ └── java │ │ └── zipkin │ │ └── sparkstreaming │ │ └── stream │ │ └── kafka │ │ ├── BootstrapServers.java │ │ ├── KafkaStreamFactory.java │ │ └── ZookeeperBootstrapServers.java │ └── test │ └── java │ └── zipkin │ └── sparkstreaming │ └── stream │ └── kafka │ ├── KafkaStreamFactoryTest.java │ └── ZookeeperBootstrapServersTest.java └── pom.xml /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Zipkin 2 | 3 | If you would like to contribute code you can do so through GitHub by forking the repository and sending a pull request (on a branch other than `master` or `gh-pages`). 4 | 5 | When submitting code, please apply [Square Code Style](https://github.com/square/java-code-styles). 6 | * If the settings import correctly, CodeStyle/Java will be named Square and use 2 space tab and indent, with 4 space continuation indent. 7 | 8 | ## License 9 | 10 | By contributing your code, you agree to license your contribution under the terms of the APLv2: https://github.com/openzipkin/zipkin/blob/master/LICENSE 11 | 12 | All files are released with the Apache 2.0 license. 13 | 14 | If you are adding a new file it should have a header like below. This can be automatically added by running `./mvnw com.mycila:license-maven-plugin:format`. 15 | 16 | ``` 17 | /** 18 | * Copyright 2017 The OpenZipkin Authors 19 | * 20 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 21 | * in compliance with the License. You may obtain a copy of the License at 22 | * 23 | * http://www.apache.org/licenses/LICENSE-2.0 24 | * 25 | * Unless required by applicable law or agreed to in writing, software distributed under the License 26 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 27 | * or implied. See the License for the specific language governing permissions and limitations under 28 | * the License. 29 | */ 30 | ``` 31 | 32 | ## Logging 33 | 34 | Spark logging is managed via log4J configuration. [LogInitializer](./sparkstreaming/src/main/java/zipkin/dependencies/LogInitializer.java) 35 | adds configuration during bootstrap for the "zipkin" category and 36 | propagates it to Spark executors in a dependency free manner. 37 | 38 | Even though Spark uses log4J underneath, declare loggers using the SLF4J 39 | api, notable as static final field. SLF4J loggers are serializable, so 40 | do not require special handling when part of a spark task. If you need 41 | to test logging, encapsulate the static field in an instance method 42 | `log()` and override it during tests. -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | What kind of issue is this? 2 | 3 | - [ ] Question. This issue tracker is not the place for questions. If you want to ask how to do 4 | something, or to understand why something isn't working the way you expect it to, use Gitter 5 | or Stack Overflow. https://gitter.im/openzipkin/zipkin https://stackoverflow.com/questions/tagged/zipkin 6 | 7 | - [ ] Bug report. If you’ve found a bug, spend the time to write a failing test. Bugs with tests 8 | get fixed and stay fixed. If you have a solution in mind, skip raising an issue and open a 9 | pull request instead. 10 | 11 | - [ ] Feature Request. First, look at existing issues to see if the feature has been requested 12 | before. If you don't find anything, tell us what problem you’re trying to solve. Often a 13 | solution already exists! Don’t send pull requests to implement new features without first 14 | getting our support. Sometimes we leave features out on purpose to keep the project small. 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .classpath 3 | .project 4 | .settings/ 5 | target/ 6 | *.iml 7 | .idea 8 | *.ipr 9 | *.iws 10 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openzipkin-attic/zipkin-sparkstreaming/ae3f73f858557218f0611667c1d9c2b1c7da2a31/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip -------------------------------------------------------------------------------- /.settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 21 | 22 | 23 | sonatype 24 | ${env.SONATYPE_USER} 25 | ${env.SONATYPE_PASSWORD} 26 | 27 | 28 | bintray 29 | ${env.BINTRAY_USER} 30 | ${env.BINTRAY_KEY} 31 | 32 | 33 | jfrog-snapshots 34 | ${env.BINTRAY_USER} 35 | ${env.BINTRAY_KEY} 36 | 37 | 38 | github.com 39 | ${env.GH_USER} 40 | ${env.GH_TOKEN} 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NOTICE.md: -------------------------------------------------------------------------------- 1 | # Pintrace 2 | 3 | The initial version of this code came from a project called Pintrace by Suman Karumuri, Naoman Abbas and Phoebe Tse at Pinterest. For more information about this project, please see the blog post at https://engineering.pinterest.com/blog/distributed-tracing-pinterest-new-open-source-tools 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Gitter chat](http://img.shields.io/badge/gitter-join%20chat%20%E2%86%92-brightgreen.svg)](https://gitter.im/openzipkin/zipkin) [![Build Status](https://circleci.com/gh/openzipkin/zipkin-azure.svg?style=svg)](https://circleci.com/gh/openzipkin/zipkin-sparkstreaming) [![Download](https://api.bintray.com/packages/openzipkin/maven/zipkin-sparkstreaming/images/download.svg) ](https://bintray.com/openzipkin/maven/zipkin-sparkstreaming/_latestVersion) 2 | 3 | # zipkin-sparkstreaming 4 | This is a streaming alternative to Zipkin's collector. 5 | 6 | Zipkin's collector receives span messages reported by applications, or 7 | via Kafka. It does very little besides storing them for later query, and 8 | there are limited options for downsampling or otherwise. 9 | 10 | This project provides a more flexible pipeline, including the ability to 11 | * receive spans from other sources, like files 12 | * perform dynamic sampling, like retain only latent or error traces 13 | * process data in real-time, like reporting or alternate visualization tools 14 | * adjust data, like scrubbing private data or normalizing service names 15 | 16 | ## Status 17 | Many features are incomplete. Please join us to help complete them. 18 | 19 | ## Usage 20 | 21 | The quickest way to get started is to fetch the [latest released job](https://search.maven.org/remote_content?g=io.zipkin.sparkstreaming&a=zipkin-sparkstreaming-job&v=LATEST) as a self-contained executable jar. Note that the Zipkin Spark Streaming Job requires minimum JRE 7. For example: 22 | 23 | ### Download the latest job 24 | The following downloads the latest version using wget: 25 | 26 | ```bash 27 | wget -O zipkin-sparkstreaming-job.jar 'https://search.maven.org/remote_content?g=io.zipkin.sparkstreaming&a=zipkin-sparkstreaming-job&v=LATEST' 28 | ``` 29 | 30 | ### Run the job 31 | You can either run the job in local or cluster mode. Here's an example of each: 32 | 33 | ```bash 34 | # run local 35 | java -jar zipkin-sparkstreaming-job.jar \ 36 | --zipkin.log-level=debug \ 37 | --zipkin.storage.type=elasticsearch \ 38 | --zipkin.storage.elasticsearch.hosts=http://127.0.0.1:9200 \ 39 | --zipkin.sparkstreaming.stream.kafka.bootstrap-servers=127.0.0.1:9092 40 | # run in a cluster 41 | java -jar zipkin-sparkstreaming-job.jar \ 42 | --zipkin.log-level=debug \ 43 | --zipkin.storage.type=elasticsearch \ 44 | --zipkin.storage.elasticsearch.hosts=http://127.0.0.1:9200 \ 45 | --zipkin.sparkstreaming.stream.kafka.bootstrap-servers=127.0.0.1:9092 \ 46 | --zipkin.sparkstreaming.master=spark://127.0.0.1:7077 47 | ``` 48 | 49 | ## Key Components 50 | 51 | The image below shows the internal architecture of zipkin spark streaming job. StreamFactory is a extensible interface that ingests data from Kafka or any other transport. The filtering step filters spans based on criteria like service name([#33](https://github.com/openzipkin/zipkin-sparkstreaming/issues/33)). The aggregation phase groups the spans by time or trace ID. The adjuster phase is useful for making adjustments to spans that belong to the same trace. For example, the FinagleAdjuster fixes known bugs in the old finagle zipkin tracer. The final consumer stage persists the data to a storage system like ElasticSearch service. 52 | 53 | 54 | ┌────────────────────────────┐ 55 | │ Kafka │ 56 | └────────────────────────────┘ 57 | ┌────────────────┼────────────────┐ 58 | │ ▼ │ 59 | │ ┌────────────────────────────┐ │ 60 | │ │ StreamFactory │ │ 61 | │ └────────────────────────────┘ │ 62 | │ │ │ 63 | │ ▼ │ 64 | │ ┌────────────────────────────┐ │ 65 | │ │ Filtering │ │ 66 | │ └────────────────────────────┘ │ 67 | │ │ │ 68 | │ ▼ │ 69 | │ ┌────────────────────────────┐ │ 70 | │ │ Aggregation │ │ 71 | │ └────────────────────────────┘ │ 72 | │ │ │ 73 | │ ▼ │ 74 | │ ┌────────────────────────────┐ │ 75 | │ │ Adjuster │ │ 76 | │ └────────────────────────────┘ │ 77 | │ │ │ 78 | │ ▼ │ 79 | │ ┌────────────────────────────┐ │ 80 | │ │ Consumer │ │ 81 | │ └────────────────────────────┘ │ 82 | └─────────────────┼───────────────┘ 83 | ▼ 84 | ┌──────────────────────────┐ 85 | │ Storage (ES, Cassandra) │ 86 | └──────────────────────────┘ 87 | 88 | ### Stream 89 | A stream is a source of json or thrift encoded span messages. 90 | 91 | For example, a message stream could be a Kafka topic named "zipkin" 92 | 93 | Stream | Description 94 | --- | --- 95 | [Kafka](./stream/kafka) | Ingests spans from a Kafka topic. 96 | 97 | ### Adjuster 98 | An adjuster conditionally changes spans sharing the same trace ID. 99 | 100 | You can make adjusters to fixup data reported by instrumentation, or to 101 | scrub private data. This [example](https://github.com/openzipkin/zipkin-sparkstreaming-example) shows how to add a custom adjuster to the spark job. 102 | 103 | Below is the list of prepackaged adjusters. 104 | 105 | Adjuster | Description 106 | --- | --- 107 | [Finagle](./adjuster/finagle) | Fixes up spans reported by [Finagle](https://github.com/twitter/finagle/tree/develop/finagle-zipkin). 108 | 109 | ### Consumer 110 | A consumer is an end-recipient of potentially adjusted spans sharing the 111 | same trace ID. 112 | 113 | This could be a Zipkin storage component, like Elasticsearch, or another 114 | sink, such as a streaming visualization tool. 115 | 116 | Consumer | Description 117 | --- | --- 118 | [Storage](./consumer/storage) | Writes spans to a Zipkin Storage Component 119 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Zipkin Spark Streaming Release Process 2 | 3 | This repo uses semantic versions. Please keep this in mind when choosing version numbers. 4 | 5 | 1. **Alert others you are releasing** 6 | 7 | There should be no commits made to master while the release is in progress (about 10 minutes). Before you start 8 | a release, alert others on [gitter](https://gitter.im/openzipkin/zipkin) so that they don't accidentally merge 9 | anything. If they do, and the build fails because of that, you'll have to recreate the release tag described below. 10 | 11 | 1. **Push a git tag** 12 | 13 | The tag should be of the format `release-N.M.L`, for example `release-3.7.1`. 14 | 15 | 1. **Wait for CircleCI** 16 | 17 | This part is controlled by [`build-support/publish-stable.sh`](build-support/publish-stable.sh). It creates a bunch of new commits, bumps 18 | the version, publishes artifacts, and syncs to Maven Central. https://circleci.com/gh/openzipkin/zipkin-sparkstreaming 19 | 20 | ## Credentials 21 | 22 | Credentials of various kind are needed for the release process to work. If you notice something 23 | failing due to unauthorized, re-save them as [environment variables](https://circleci.com/gh/openzipkin/zipkin-sparkstreaming/edit#env-vars). 24 | 25 | ## First release of the year 26 | 27 | The license plugin verifies license headers of files include a copyright notice indicating the years a file was affected. 28 | This information is taken from git history. There's a once-a-year problem with files that include version numbers (pom.xml). 29 | When a release tag is made, it increments version numbers, then commits them to git. On the first release of the year, 30 | further commands will fail due to the version increments invalidating the copyright statement. The way to sort this out is 31 | the following: 32 | 33 | Before you do the first release of the year, move the SNAPSHOT version back and forth from whatever the current is. 34 | In-between, re-apply the licenses. 35 | ```bash 36 | $ ./mvnw versions:set -DnewVersion=1.3.3-SNAPSHOT -DgenerateBackupPoms=false 37 | $ ./mvnw com.mycila:license-maven-plugin:format 38 | $ ./mvnw versions:set -DnewVersion=1.3.2-SNAPSHOT -DgenerateBackupPoms=false 39 | $ git commit -am"Adjusts copyright headers for this year" 40 | ``` 41 | 42 | -------------------------------------------------------------------------------- /adjuster/finagle/README.md: -------------------------------------------------------------------------------- 1 | # adjuster-finagle 2 | 3 | ## FinagleAdjuster 4 | This fixes up spans reported by [Finagle](https://github.com/twitter/finagle/tree/develop/finagle-zipkin). 5 | 6 | Adjustment code should not be added without a tracking issue either in 7 | [Finagle](https://github.com/twitter/finagle/issues) or [Zipkin Finagle](https://github.com/openzipkin/zipkin-finagle/issues). 8 | 9 | ## Usage 10 | 11 | While the `FinagleAdjuster` can be used directly through the provided 12 | builder interface, most users will likely find more value in the Spring 13 | Boot autoconfiguraton module. Additional information for using the 14 | module can be found [here](../../autoconfigure/adjuster-finagle). 15 | -------------------------------------------------------------------------------- /adjuster/finagle/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | io.zipkin.sparkstreaming 20 | zipkin-sparkstreaming-adjuster-parent 21 | 0.3.10-SNAPSHOT 22 | 23 | 4.0.0 24 | 25 | zipkin-sparkstreaming-adjuster-finagle 26 | Zipkin Spark Streaming Adjuster: Finagle 27 | 28 | 29 | ${project.basedir}/../.. 30 | 31 | 32 | 33 | 34 | com.google.auto.value 35 | auto-value 36 | provided 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /adjuster/finagle/src/main/java/zipkin/sparkstreaming/adjuster/finagle/FinagleAdjuster.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.adjuster.finagle; 15 | 16 | import com.google.auto.value.AutoValue; 17 | import zipkin.BinaryAnnotation; 18 | import zipkin.Span; 19 | import zipkin.internal.ApplyTimestampAndDuration; 20 | import zipkin.sparkstreaming.Adjuster; 21 | 22 | /** 23 | * Contains adjustments that pertain to Finagle tracing. Detection is based on the binary 24 | * annotations named ".*finagle.version.*". 25 | */ 26 | @AutoValue 27 | public abstract class FinagleAdjuster extends Adjuster { 28 | 29 | public static Builder newBuilder() { 30 | return new AutoValue_FinagleAdjuster.Builder() 31 | .applyTimestampAndDuration(true); 32 | } 33 | 34 | abstract boolean applyTimestampAndDuration(); 35 | 36 | @AutoValue.Builder 37 | public interface Builder { 38 | /** 39 | * As of Finagle 6.41.0, tracing is always RPC in nature, but timestamp and duration are not 40 | * added. This backfills timestamps. Default true 41 | * 42 | *

The current fix is to use zipkin-finagle to report spans. 43 | * See https://github.com/openzipkin/zipkin-finagle/issues/10 44 | */ 45 | Builder applyTimestampAndDuration(boolean applyTimestampAndDuration); 46 | 47 | FinagleAdjuster build(); 48 | } 49 | 50 | @Override protected boolean shouldAdjust(Span span) { 51 | for (BinaryAnnotation b : span.binaryAnnotations) { 52 | if (b.key.indexOf("finagle.version") != -1) return true; 53 | } 54 | return false; 55 | } 56 | 57 | @Override protected Span adjust(Span span) { 58 | if (applyTimestampAndDuration()) { 59 | return ApplyTimestampAndDuration.apply(span); 60 | } 61 | return span; 62 | } 63 | 64 | FinagleAdjuster() { 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /adjuster/finagle/src/main/java/zipkin/sparkstreaming/adjuster/finagle/FinagleIssue343Adjuster.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.adjuster.finagle; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | import zipkin.Annotation; 19 | import zipkin.BinaryAnnotation; 20 | import zipkin.Span; 21 | import zipkin.internal.Util; 22 | import zipkin.sparkstreaming.Adjuster; 23 | 24 | /** 25 | * This adjuster handles a bug in finagle memcached library where the duration of a span ends 26 | * up being more than the actual value because a span is submitted twice. 27 | * 28 | * The fix is to drops finagle.flush annotation for memcache spans. We look for "Hit" or "Miss" 29 | * binary annotations to make sure the span is coming from memcache. 30 | * 31 | * For more details see 32 | * https://github.com/twitter/finagle/issues/343 33 | * 34 | */ 35 | public final class FinagleIssue343Adjuster extends Adjuster{ 36 | 37 | FinagleIssue343Adjuster() {} 38 | 39 | public static FinagleIssue343Adjuster create() { 40 | return new FinagleIssue343Adjuster(); 41 | } 42 | 43 | @Override protected boolean shouldAdjust(Span span) { 44 | if (containsFinagleFlushAnnotation(span) && containsHitOrMissBinaryAnnotation(span)) { 45 | return true; 46 | } 47 | return false; 48 | } 49 | 50 | private boolean containsHitOrMissBinaryAnnotation(Span span) { 51 | for (BinaryAnnotation b : span.binaryAnnotations) { 52 | String value = new String(b.value, Util.UTF_8); 53 | if (value.equals("Hit") || value.equals("Miss")) { 54 | return true; 55 | } 56 | } 57 | return false; 58 | } 59 | 60 | private boolean containsFinagleFlushAnnotation(Span span) { 61 | for (Annotation a : span.annotations) { 62 | if (a.value.equals("finagle.flush")) { 63 | return true; 64 | } 65 | } 66 | return false; 67 | } 68 | 69 | @Override protected Span adjust(Span span) { 70 | List annotations = new ArrayList<>(); 71 | for (Annotation a : span.annotations) { 72 | if (!a.value.equals("finagle.flush")) { 73 | annotations.add(a); 74 | } 75 | } 76 | return span.toBuilder().annotations(annotations).build(); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /adjuster/finagle/src/test/java/zipkin/sparkstreaming/adjuster/finagle/FinagleAdjusterTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.adjuster.finagle; 15 | 16 | import org.junit.Test; 17 | import zipkin.Annotation; 18 | import zipkin.BinaryAnnotation; 19 | import zipkin.Constants; 20 | import zipkin.Endpoint; 21 | import zipkin.Span; 22 | import zipkin.TestObjects; 23 | import zipkin.internal.ApplyTimestampAndDuration; 24 | 25 | import static java.util.Arrays.asList; 26 | import static org.assertj.core.api.Assertions.assertThat; 27 | 28 | public class FinagleAdjusterTest { 29 | FinagleAdjuster adjuster = FinagleAdjuster.newBuilder().build(); 30 | 31 | Endpoint localEndpoint = 32 | Endpoint.builder().serviceName("my-host").ipv4(127 << 24 | 1).port(9411).build(); 33 | Endpoint localEndpoint0 = localEndpoint.toBuilder().port(null).build(); 34 | // finagle often sets to the client endpoint to the same as the local endpoint 35 | Endpoint remoteEndpoint = localEndpoint.toBuilder().port(63840).build(); 36 | 37 | Span serverSpan = Span.builder() 38 | .traceId(-6054243957716233329L) 39 | .name("my-span") 40 | .id(-3615651937927048332L) 41 | .parentId(-6054243957716233329L) 42 | .addAnnotation(Annotation.create(1442493420635000L, Constants.SERVER_RECV, localEndpoint)) 43 | .addAnnotation(Annotation.create(1442493422680000L, Constants.SERVER_SEND, localEndpoint)) 44 | .addBinaryAnnotation(BinaryAnnotation.create("srv/finagle.version", "6.28.0", localEndpoint0)) 45 | .addBinaryAnnotation(BinaryAnnotation.address(Constants.SERVER_ADDR, localEndpoint)) 46 | .addBinaryAnnotation(BinaryAnnotation.address(Constants.CLIENT_ADDR, remoteEndpoint)) 47 | .build(); 48 | 49 | /** Default is to apply timestamp and duration */ 50 | @Test 51 | public void adjustsFinagleSpans() throws Exception { 52 | Iterable adjusted = adjuster.adjust(asList(serverSpan)); 53 | assertThat(adjusted).containsExactly(ApplyTimestampAndDuration.apply(serverSpan)); 54 | } 55 | 56 | @Test 57 | public void applyTimestampAndDuration_disabled() throws Exception { 58 | adjuster = FinagleAdjuster.newBuilder().applyTimestampAndDuration(false).build(); 59 | Iterable adjusted = adjuster.adjust(asList(serverSpan)); 60 | assertThat(adjusted).containsExactly(serverSpan); 61 | } 62 | 63 | @Test 64 | public void doesntAdjustNonFinagleSpans() throws Exception { 65 | Iterable adjusted = adjuster.adjust(TestObjects.TRACE); 66 | assertThat(adjusted).containsExactlyElementsOf(TestObjects.TRACE); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /adjuster/finagle/src/test/java/zipkin/sparkstreaming/adjuster/finagle/FinagleIssue343AdjusterTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.adjuster.finagle; 15 | 16 | import java.util.ArrayList; 17 | import org.junit.Test; 18 | import zipkin.Annotation; 19 | import zipkin.BinaryAnnotation; 20 | import zipkin.Constants; 21 | import zipkin.Endpoint; 22 | import zipkin.Span; 23 | import zipkin.TestObjects; 24 | 25 | import static java.util.Arrays.asList; 26 | import static org.assertj.core.api.Assertions.assertThat; 27 | 28 | public class FinagleIssue343AdjusterTest { 29 | FinagleIssue343Adjuster adjuster = FinagleIssue343Adjuster.create(); 30 | 31 | Endpoint localEndpoint = 32 | Endpoint.builder().serviceName("my-host").ipv4(127 << 24 | 1).port(9411).build(); 33 | Endpoint localEndpoint0 = localEndpoint.toBuilder().port(null).build(); 34 | // finagle often sets to the client endpoint to the same as the local endpoint 35 | Endpoint remoteEndpoint = localEndpoint.toBuilder().port(63840).build(); 36 | 37 | Span serverSpan = Span.builder() 38 | .traceId(-6054243957716233329L) 39 | .name("my-span") 40 | .id(-3615651937927048332L) 41 | .parentId(-6054243957716233329L) 42 | .addAnnotation(Annotation.create(1442493420635000L, Constants.SERVER_RECV, localEndpoint)) 43 | .addAnnotation(Annotation.create(1442493422680000L, Constants.SERVER_SEND, localEndpoint)) 44 | .addBinaryAnnotation(BinaryAnnotation.create("key", "Hit", localEndpoint0)) 45 | .build(); 46 | 47 | Span serverSpanWithFinagleFlush = serverSpan.toBuilder() 48 | .addAnnotation(Annotation.create(1442493500000000L, "finagle.flush", localEndpoint)) 49 | .build(); 50 | 51 | Span serverSpanWithFinagleFlushWithoutHitBinaryAnnotation = 52 | serverSpanWithFinagleFlush.toBuilder() 53 | .binaryAnnotations(new ArrayList<>()) 54 | .build(); 55 | 56 | @Test 57 | public void adjustSpanWithFinagleFlush() { 58 | Iterable adjusted = adjuster.adjust(asList(serverSpanWithFinagleFlush)); 59 | assertThat(adjusted).containsExactly(serverSpan); 60 | } 61 | 62 | @Test 63 | public void hitOrMissBinaryAnnotationMissing() { 64 | Iterable adjusted = 65 | adjuster.adjust(asList(serverSpanWithFinagleFlushWithoutHitBinaryAnnotation)); 66 | assertThat(adjusted).containsExactly(serverSpanWithFinagleFlushWithoutHitBinaryAnnotation); 67 | } 68 | 69 | @Test 70 | public void doesntAdjustNonFinagleSpans() throws Exception { 71 | Iterable adjusted = adjuster.adjust(TestObjects.TRACE); 72 | assertThat(adjusted).containsExactlyElementsOf(TestObjects.TRACE); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /adjuster/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 4.0.0 19 | 20 | 21 | io.zipkin.sparkstreaming 22 | zipkin-sparkstreaming-parent 23 | 0.3.10-SNAPSHOT 24 | 25 | 26 | zipkin-sparkstreaming-adjuster-parent 27 | Zipkin Spark Streaming: Adjusters 28 | pom 29 | 30 | 31 | ${project.basedir}/.. 32 | 33 | 34 | 35 | finagle 36 | 37 | 38 | 39 | 40 | ${project.groupId} 41 | zipkin-sparkstreaming 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /autoconfigure/adjuster-finagle/README.md: -------------------------------------------------------------------------------- 1 | # autoconfigure-adjuster-finagle 2 | 3 | ## ZipkinFinagleAdjusterAutoConfiguration 4 | 5 | This is a Spring Boot [AutoConfiguration](http://docs.spring.io/spring-boot/docs/current/reference/html/using-boot-auto-configuration.html) 6 | module built into the [Spark Streaming Job](../../sparkstreaming-job) 7 | which fixes up spans reported by Finagle applications. Internally, this 8 | module wraps the [FinagleAdjuster](../../adjuster/finagle) to expose 9 | configuration options via properties. 10 | 11 | ## Usage 12 | 13 | In order to connect, you minimally need to set 14 | `zipkin.sparkstreaming.adjuster.finagle.enabled` to true. 15 | 16 | Ex. to enable Finagle adjustment 17 | 18 | ```bash 19 | java -jar zipkin-sparkstreaming-job.jar \ 20 | --zipkin.sparkstreaming.adjuster.finagle.enabled=true \ 21 | --zipkin.storage.type=elasticsearch \ 22 | --zipkin.storage.elasticsearch.hosts=http://127.0.0.1:9200 \ 23 | --zipkin.sparkstreaming.stream.kafka.bootstrap-servers=127.0.0.1:9092 24 | ``` 25 | 26 | ### Configuration 27 | 28 | Configuration properties can be set via commandline parameters, system 29 | properties or any other alternative [supported by Spring Boot](https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html). 30 | 31 | Here are the relevant setting and a short description. Properties all 32 | have a prefix of "zipkin.sparkstreaming.adjuster.finagle" 33 | 34 | Property | Default | Description | Fix 35 | --- | --- | --- | --- 36 | apply-timestamp-and-duration | true | Backfill span.timestamp and duration based on annotations. | [Use zipkin-finagle](https://github.com/openzipkin/zipkin-finagle/issues/10) 37 | adjust-issue343 | false | Drops "finagle.flush" annotation, to rectify [finagle memcached bug](https://github.com/twitter/finagle/issues/343). | Use finagle version 6.36.0 or higher 38 | -------------------------------------------------------------------------------- /autoconfigure/adjuster-finagle/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | io.zipkin.sparkstreaming 20 | zipkin-sparkstreaming-autoconfigure-parent 21 | 0.3.10-SNAPSHOT 22 | 23 | 4.0.0 24 | zipkin-sparkstreaming-autoconfigure-adjuster-finagle 25 | Zipkin Spark Streaming Auto Configure: Finagle Adjuster 26 | 27 | 28 | ${project.basedir}/../.. 29 | 30 | 31 | 32 | 33 | io.zipkin.sparkstreaming 34 | zipkin-sparkstreaming-adjuster-finagle 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /autoconfigure/adjuster-finagle/src/main/java/zipkin/sparkstreaming/autoconfigure/adjuster/finagle/ZipkinFinagleAdjusterAutoConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.autoconfigure.adjuster.finagle; 15 | 16 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 17 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 18 | import org.springframework.context.annotation.Bean; 19 | import org.springframework.context.annotation.Configuration; 20 | import zipkin.sparkstreaming.Adjuster; 21 | import zipkin.sparkstreaming.adjuster.finagle.FinagleIssue343Adjuster; 22 | 23 | @Configuration 24 | @EnableConfigurationProperties(ZipkinFinagleAdjusterProperties.class) 25 | @ConditionalOnProperty( 26 | value = "zipkin.sparkstreaming.adjuster.finagle.enabled", 27 | havingValue = "true" 28 | ) 29 | public class ZipkinFinagleAdjusterAutoConfiguration { 30 | 31 | @Bean 32 | Adjuster finagleAdjuster(ZipkinFinagleAdjusterProperties properties) { 33 | return properties.toBuilder().build(); 34 | } 35 | 36 | @Bean 37 | @ConditionalOnProperty( 38 | value = "zipkin.sparkstreaming.adjuster.finagle.adjust-issue343", 39 | havingValue = "true" 40 | ) 41 | Adjuster finagleIssue343Adjuster() { 42 | return FinagleIssue343Adjuster.create(); 43 | } 44 | 45 | } 46 | 47 | -------------------------------------------------------------------------------- /autoconfigure/adjuster-finagle/src/main/java/zipkin/sparkstreaming/autoconfigure/adjuster/finagle/ZipkinFinagleAdjusterProperties.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.autoconfigure.adjuster.finagle; 15 | 16 | import org.springframework.boot.context.properties.ConfigurationProperties; 17 | import zipkin.sparkstreaming.adjuster.finagle.FinagleAdjuster; 18 | 19 | @ConfigurationProperties("zipkin.sparkstreaming.adjuster.finagle") 20 | public class ZipkinFinagleAdjusterProperties { 21 | private boolean applyTimestampAndDuration = true; 22 | 23 | public boolean isApplyTimestampAndDuration() { 24 | return applyTimestampAndDuration; 25 | } 26 | 27 | public void setApplyTimestampAndDuration(boolean applyTimestampAndDuration) { 28 | this.applyTimestampAndDuration = applyTimestampAndDuration; 29 | } 30 | 31 | FinagleAdjuster.Builder toBuilder() { 32 | return FinagleAdjuster.newBuilder() 33 | .applyTimestampAndDuration(applyTimestampAndDuration); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /autoconfigure/adjuster-finagle/src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- 1 | org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ 2 | zipkin.sparkstreaming.autoconfigure.adjuster.finagle.ZipkinFinagleAdjusterAutoConfiguration 3 | -------------------------------------------------------------------------------- /autoconfigure/adjuster-finagle/src/test/java/zipkin/sparkstreaming/adjuster/finagle/ZipkinFinagleAdjusterAutoConfigurationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.adjuster.finagle; 15 | 16 | import org.junit.After; 17 | import org.junit.Rule; 18 | import org.junit.Test; 19 | import org.junit.rules.ExpectedException; 20 | import org.springframework.beans.factory.NoSuchBeanDefinitionException; 21 | import org.springframework.boot.autoconfigure.context.PropertyPlaceholderAutoConfiguration; 22 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 23 | import zipkin.sparkstreaming.autoconfigure.adjuster.finagle.ZipkinFinagleAdjusterAutoConfiguration; 24 | import zipkin.sparkstreaming.autoconfigure.adjuster.finagle.ZipkinFinagleAdjusterProperties; 25 | 26 | import static org.assertj.core.api.Assertions.assertThat; 27 | import static org.springframework.boot.test.util.EnvironmentTestUtils.addEnvironment; 28 | 29 | public class ZipkinFinagleAdjusterAutoConfigurationTest { 30 | 31 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 32 | @Rule 33 | public ExpectedException thrown = ExpectedException.none(); 34 | 35 | @After 36 | public void close() { 37 | if (context != null) context.close(); 38 | } 39 | 40 | @Test 41 | public void doesntProvideAdjusterWhenDisabled() { 42 | context.register(PropertyPlaceholderAutoConfiguration.class, 43 | ZipkinFinagleAdjusterProperties.class, 44 | ZipkinFinagleAdjusterAutoConfiguration.class); 45 | context.refresh(); 46 | 47 | thrown.expect(NoSuchBeanDefinitionException.class); 48 | context.getBean(FinagleAdjuster.class); 49 | } 50 | 51 | @Test 52 | public void providesAdjusterWhenEnabled() { 53 | addEnvironment(context, 54 | "zipkin.sparkstreaming.adjuster.finagle.enabled:" + true); 55 | context.register(PropertyPlaceholderAutoConfiguration.class, 56 | ZipkinFinagleAdjusterAutoConfiguration.class); 57 | context.refresh(); 58 | 59 | FinagleAdjuster adjuster = context.getBean(FinagleAdjuster.class); 60 | assertThat(adjuster.applyTimestampAndDuration()).isTrue(); 61 | } 62 | 63 | @Test 64 | public void disableTimestampAndDuration() { 65 | addEnvironment(context, 66 | "zipkin.sparkstreaming.adjuster.finagle.enabled:" + true, 67 | "zipkin.sparkstreaming.adjuster.finagle.apply-timestamp-and-duration:" + false); 68 | context.register(PropertyPlaceholderAutoConfiguration.class, 69 | ZipkinFinagleAdjusterAutoConfiguration.class); 70 | context.refresh(); 71 | 72 | FinagleAdjuster adjuster = context.getBean(FinagleAdjuster.class); 73 | assertThat(adjuster.applyTimestampAndDuration()).isFalse(); 74 | } 75 | 76 | @Test 77 | public void doesntProvideIssue343AdjusterWhenFinagleDisabled() { 78 | context.register(PropertyPlaceholderAutoConfiguration.class, 79 | ZipkinFinagleAdjusterAutoConfiguration.class); 80 | context.refresh(); 81 | 82 | thrown.expect(NoSuchBeanDefinitionException.class); 83 | context.getBean(FinagleIssue343Adjuster.class); 84 | } 85 | 86 | @Test 87 | public void doesntProvidesIssue343AdjusterWhenFinagleEnabledAndIssue343Disabled() { 88 | addEnvironment(context, 89 | "zipkin.sparkstreaming.adjuster.finagle.enabled:" + true); 90 | context.register(PropertyPlaceholderAutoConfiguration.class, 91 | ZipkinFinagleAdjusterAutoConfiguration.class); 92 | context.refresh(); 93 | 94 | thrown.expect(NoSuchBeanDefinitionException.class); 95 | context.getBean(FinagleIssue343Adjuster.class); 96 | } 97 | 98 | @Test 99 | public void providesIssue343Adjuster() { 100 | addEnvironment(context, 101 | "zipkin.sparkstreaming.adjuster.finagle.enabled:" + true, 102 | "zipkin.sparkstreaming.adjuster.finagle.adjust-issue343:" + true); 103 | context.register(PropertyPlaceholderAutoConfiguration.class, 104 | ZipkinFinagleAdjusterAutoConfiguration.class); 105 | context.refresh(); 106 | 107 | FinagleIssue343Adjuster adjuster = context.getBean(FinagleIssue343Adjuster.class); 108 | assertThat(adjuster).isNotNull(); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /autoconfigure/consumer-storage/README.md: -------------------------------------------------------------------------------- 1 | # autoconfigure-consumer-storage 2 | 3 | ## ZipkinStorageConsumerAutoConfiguration 4 | 5 | This is a Spring Boot [AutoConfiguration](http://docs.spring.io/spring-boot/docs/current/reference/html/using-boot-auto-configuration.html) 6 | module built into the [Spark Streaming Job](../../sparkstreaming-job) 7 | which writes spans to a Zipkin `StorageComponent`. Internally, this 8 | module wraps the [StorageConsumer](../../consumer/storage) to expose 9 | configuration options via properties. 10 | 11 | ## Usage 12 | 13 | In order to connect, you minimally need to set `zipkin.storage.type` to 14 | a configured backend. You also need to set relevant properties for that. 15 | 16 | Ex. 17 | ```bash 18 | java -jar zipkin-sparkstreaming-job.jar \ 19 | --zipkin.storage.type=elasticsearch \ 20 | --zipkin.storage.elasticsearch.hosts=http://127.0.0.1:9200 \ 21 | ... 22 | ``` 23 | 24 | ### Configuration 25 | 26 | Configuration properties can be set via commandline parameters, system 27 | properties or any other alternative [supported by Spring Boot](https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html). 28 | 29 | Besides setting storage properties, you can also override the following. 30 | All of the below have a prefix of "zipkin.sparkstreaming.consumer.storage" 31 | 32 | Property | Default |Description 33 | --- | --- | --- 34 | fail-fast | true | check storage before submitting the job. 35 | 36 | ## More Examples 37 | 38 | Ex. to connect to a local Elasticsearch service: 39 | 40 | ```bash 41 | java -jar zipkin-sparkstreaming-job.jar \ 42 | --zipkin.storage.type=elasticsearch \ 43 | --zipkin.storage.elasticsearch.hosts=http://127.0.0.1:9200 \ 44 | --zipkin.sparkstreaming.stream.kafka.bootstrap-servers=127.0.0.1:9092 45 | ``` 46 | 47 | Ex. to connect to a local MySQL service: 48 | 49 | ```bash 50 | java -jar zipkin-sparkstreaming-job.jar \ 51 | --zipkin.storage.type=mysql \ 52 | --zipkin.storage.mysql.host=127.0.0.1 \ 53 | --zipkin.storage.mysql.username=root \ 54 | --zipkin.sparkstreaming.stream.kafka.bootstrap-servers=127.0.0.1:9092 55 | ``` 56 | 57 | Ex. to connect to a local Cassandra service: 58 | 59 | ```bash 60 | java -jar zipkin-sparkstreaming-job.jar \ 61 | --zipkin.storage.type=cassandra \ 62 | --zipkin.storage.cassandra.contact-points=127.0.0.1 \ 63 | --zipkin.sparkstreaming.stream.kafka.bootstrap-servers=127.0.0.1:9092 64 | ``` 65 | -------------------------------------------------------------------------------- /autoconfigure/consumer-storage/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | io.zipkin.sparkstreaming 20 | zipkin-sparkstreaming-autoconfigure-parent 21 | 0.3.10-SNAPSHOT 22 | 23 | 4.0.0 24 | zipkin-sparkstreaming-autoconfigure-consumer-storage 25 | Zipkin Spark Streaming Auto Configure: Storage Consumer 26 | 27 | 28 | ${project.basedir}/../.. 29 | 30 | 31 | 32 | 33 | io.zipkin.sparkstreaming 34 | zipkin-sparkstreaming-consumer-storage 35 | 36 | 37 | 38 | io.zipkin.java 39 | zipkin-autoconfigure-storage-elasticsearch-http 40 | ${zipkin.version} 41 | 42 | 43 | 44 | io.zipkin.java 45 | zipkin-autoconfigure-storage-mysql 46 | ${zipkin.version} 47 | 48 | 49 | 50 | io.zipkin.java 51 | zipkin-autoconfigure-storage-cassandra 52 | ${zipkin.version} 53 | 54 | 55 | 56 | io.zipkin.java 57 | zipkin-autoconfigure-storage-cassandra3 58 | ${zipkin.version} 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /autoconfigure/consumer-storage/src/main/java/zipkin/sparkstreaming/autoconfigure/consumer/storage/ZipkinStorageConsumerAutoConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.autoconfigure.consumer.storage; 15 | 16 | import java.io.IOException; 17 | import java.util.Iterator; 18 | import java.util.Properties; 19 | import org.springframework.beans.factory.BeanFactory; 20 | import org.springframework.beans.factory.annotation.Value; 21 | import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; 22 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 23 | import org.springframework.boot.autoconfigure.context.PropertyPlaceholderAutoConfiguration; 24 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 25 | import org.springframework.context.annotation.Bean; 26 | import org.springframework.context.annotation.Configuration; 27 | import org.springframework.context.annotation.Import; 28 | import org.springframework.core.env.ConfigurableEnvironment; 29 | import org.springframework.core.env.EnumerablePropertySource; 30 | import org.springframework.core.env.PropertiesPropertySource; 31 | import org.springframework.core.env.PropertySource; 32 | import zipkin.Component; 33 | import zipkin.autoconfigure.storage.cassandra.ZipkinCassandraStorageAutoConfiguration; 34 | import zipkin.autoconfigure.storage.cassandra3.ZipkinCassandra3StorageAutoConfiguration; 35 | import zipkin.autoconfigure.storage.elasticsearch.http.ZipkinElasticsearchHttpStorageAutoConfiguration; 36 | import zipkin.autoconfigure.storage.elasticsearch.http.ZipkinElasticsearchOkHttpAutoConfiguration; 37 | import zipkin.autoconfigure.storage.mysql.ZipkinMySQLStorageAutoConfiguration; 38 | import zipkin.internal.V2StorageComponent; 39 | import zipkin.sparkstreaming.consumer.storage.StorageConsumer; 40 | import zipkin.storage.StorageComponent; 41 | import zipkin.storage.cassandra.CassandraStorage; 42 | import zipkin.storage.elasticsearch.http.ElasticsearchHttpStorage; 43 | import zipkin.storage.mysql.MySQLStorage; 44 | 45 | @Configuration 46 | @ConditionalOnProperty("zipkin.storage.type") 47 | @Import({ 48 | ZipkinCassandraStorageAutoConfiguration.class, 49 | ZipkinCassandra3StorageAutoConfiguration.class, 50 | ZipkinElasticsearchOkHttpAutoConfiguration.class, 51 | ZipkinElasticsearchHttpStorageAutoConfiguration.class, 52 | ZipkinMySQLStorageAutoConfiguration.class 53 | }) 54 | public class ZipkinStorageConsumerAutoConfiguration { 55 | @ConditionalOnBean(StorageComponent.class) 56 | @Bean StorageConsumer storageConsumer( 57 | StorageComponent component, 58 | @Value("${zipkin.sparkstreaming.consumer.storage.fail-fast:true}") boolean failFast, 59 | BeanFactory bf 60 | ) throws IOException { 61 | if (failFast) checkStorageOk(component); 62 | Properties properties = extractZipkinProperties(bf.getBean(ConfigurableEnvironment.class)); 63 | if (component instanceof V2StorageComponent) { 64 | zipkin2.storage.StorageComponent v2Storage = ((V2StorageComponent) component).delegate(); 65 | if (v2Storage instanceof ElasticsearchHttpStorage) { 66 | return new ElasticsearchStorageConsumer(properties); 67 | } else if (v2Storage instanceof zipkin2.storage.cassandra.CassandraStorage) { 68 | return new Cassandra3StorageConsumer(properties); 69 | } else { 70 | throw new UnsupportedOperationException(v2Storage + " not yet supported"); 71 | } 72 | } else if (component instanceof CassandraStorage) { 73 | return new CassandraStorageConsumer(properties); 74 | } else if (component instanceof MySQLStorage) { 75 | return new MySQLStorageConsumer(properties); 76 | } else { 77 | throw new UnsupportedOperationException(component + " not yet supported"); 78 | } 79 | } 80 | 81 | // fail fast because it is easier to detect problems here than after the cluster starts! 82 | void checkStorageOk(StorageComponent component) throws IOException { 83 | Component.CheckResult result = component.check(); 84 | if (!result.ok) throw new IllegalStateException("Storage not ok", result.exception); 85 | component.close(); // we don't use this directly as job instantiates their own 86 | } 87 | 88 | static final class ElasticsearchStorageConsumer extends AutoConfigurationStorageConsumer { 89 | ElasticsearchStorageConsumer(Properties properties) { 90 | super(properties); 91 | } 92 | 93 | @Override void registerAutoConfiguration(AnnotationConfigApplicationContext context) { 94 | context.register(ZipkinElasticsearchOkHttpAutoConfiguration.class); 95 | context.register(ZipkinElasticsearchHttpStorageAutoConfiguration.class); 96 | } 97 | } 98 | 99 | static final class CassandraStorageConsumer extends AutoConfigurationStorageConsumer { 100 | CassandraStorageConsumer(Properties properties) { 101 | super(properties); 102 | } 103 | 104 | @Override void registerAutoConfiguration(AnnotationConfigApplicationContext context) { 105 | context.register(ZipkinCassandraStorageAutoConfiguration.class); 106 | } 107 | } 108 | 109 | static final class Cassandra3StorageConsumer extends AutoConfigurationStorageConsumer { 110 | Cassandra3StorageConsumer(Properties properties) { 111 | super(properties); 112 | } 113 | 114 | @Override void registerAutoConfiguration(AnnotationConfigApplicationContext context) { 115 | context.register(ZipkinCassandra3StorageAutoConfiguration.class); 116 | } 117 | } 118 | 119 | static final class MySQLStorageConsumer extends AutoConfigurationStorageConsumer { 120 | MySQLStorageConsumer(Properties properties) { 121 | super(properties); 122 | } 123 | 124 | @Override void registerAutoConfiguration(AnnotationConfigApplicationContext context) { 125 | context.register(ZipkinMySQLStorageAutoConfiguration.class); 126 | } 127 | } 128 | 129 | static Properties extractZipkinProperties(ConfigurableEnvironment env) { 130 | Properties properties = new Properties(); 131 | Iterator> it = env.getPropertySources().iterator(); 132 | while (it.hasNext()) { 133 | PropertySource next = it.next(); 134 | if (!(next instanceof EnumerablePropertySource)) continue; 135 | EnumerablePropertySource source = (EnumerablePropertySource) next; 136 | for (String name : source.getPropertyNames()) { 137 | if (name.startsWith("zipkin")) properties.put(name, source.getProperty(name)); 138 | } 139 | } 140 | return properties; 141 | } 142 | 143 | /** 144 | * This holds only serializable state, in this case properties used to re-construct the zipkin 145 | * storage component later. 146 | */ 147 | static abstract class AutoConfigurationStorageConsumer extends StorageConsumer { 148 | final Properties properties; 149 | 150 | AutoConfigurationStorageConsumer(Properties properties) { 151 | this.properties = properties; 152 | } 153 | 154 | @Override protected StorageComponent tryCompute() { 155 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 156 | PropertiesPropertySource source = new PropertiesPropertySource("consumer", properties); 157 | context.getEnvironment().getPropertySources().addLast(source); 158 | 159 | context.register(PropertyPlaceholderAutoConfiguration.class); 160 | registerAutoConfiguration(context); 161 | context.refresh(); 162 | 163 | return context.getBean(StorageComponent.class); 164 | } 165 | 166 | abstract void registerAutoConfiguration(AnnotationConfigApplicationContext context); 167 | } 168 | } 169 | 170 | -------------------------------------------------------------------------------- /autoconfigure/consumer-storage/src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- 1 | org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ 2 | zipkin.sparkstreaming.autoconfigure.consumer.storage.ZipkinStorageConsumerAutoConfiguration 3 | -------------------------------------------------------------------------------- /autoconfigure/consumer-storage/src/test/java/zipkin/sparkstreaming/autoconfigure/consumer/storage/ZipkinStorageConsumerAutoConfigurationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.autoconfigure.consumer.storage; 15 | 16 | import org.junit.After; 17 | import org.junit.Rule; 18 | import org.junit.Test; 19 | import org.junit.rules.ExpectedException; 20 | import org.springframework.beans.factory.BeanCreationException; 21 | import org.springframework.boot.autoconfigure.context.PropertyPlaceholderAutoConfiguration; 22 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 23 | import zipkin.sparkstreaming.consumer.storage.StorageConsumer; 24 | 25 | import static org.assertj.core.api.Assertions.assertThat; 26 | import static org.springframework.boot.test.util.EnvironmentTestUtils.addEnvironment; 27 | 28 | public class ZipkinStorageConsumerAutoConfigurationTest { 29 | 30 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 31 | 32 | @Rule 33 | public ExpectedException thrown = ExpectedException.none(); 34 | 35 | @After 36 | public void close() { 37 | if (context != null) context.close(); 38 | } 39 | 40 | @Test 41 | public void providesStorageComponent_whenStorageTypeCassandra() { 42 | addEnvironment(context, 43 | "zipkin.sparkstreaming.consumer.storage.fail-fast:false", 44 | "zipkin.storage.type:cassandra"); 45 | context.register(PropertyPlaceholderAutoConfiguration.class, 46 | ZipkinStorageConsumerAutoConfiguration.class); 47 | context.refresh(); 48 | 49 | assertThat(storage()).isInstanceOf( 50 | ZipkinStorageConsumerAutoConfiguration.CassandraStorageConsumer.class); 51 | } 52 | 53 | @Test 54 | public void providesStorageComponent_whenStorageTypeCassandra3() { 55 | addEnvironment(context, 56 | "zipkin.sparkstreaming.consumer.storage.fail-fast:false", 57 | "zipkin.storage.type:cassandra3"); 58 | context.register(PropertyPlaceholderAutoConfiguration.class, 59 | ZipkinStorageConsumerAutoConfiguration.class); 60 | context.refresh(); 61 | 62 | assertThat(storage()).isInstanceOf( 63 | ZipkinStorageConsumerAutoConfiguration.Cassandra3StorageConsumer.class); 64 | } 65 | 66 | @Test 67 | public void providesStorageComponent_whenStorageTypeElasticsearch() { 68 | addEnvironment(context, 69 | "zipkin.sparkstreaming.consumer.storage.fail-fast:false", 70 | "zipkin.storage.type:elasticsearch" 71 | ); 72 | context.register(PropertyPlaceholderAutoConfiguration.class, 73 | ZipkinStorageConsumerAutoConfiguration.class); 74 | context.refresh(); 75 | 76 | assertThat(storage()).isInstanceOf( 77 | ZipkinStorageConsumerAutoConfiguration.ElasticsearchStorageConsumer.class); 78 | } 79 | 80 | @Test 81 | public void providesStorageComponent_whenStorageTypeMysql() { 82 | addEnvironment(context, 83 | "zipkin.sparkstreaming.consumer.storage.fail-fast:false", 84 | "zipkin.storage.type:mysql" 85 | ); 86 | context.register(PropertyPlaceholderAutoConfiguration.class, 87 | ZipkinStorageConsumerAutoConfiguration.class); 88 | context.refresh(); 89 | 90 | assertThat(storage()).isInstanceOf( 91 | ZipkinStorageConsumerAutoConfiguration.MySQLStorageConsumer.class); 92 | } 93 | 94 | /** fail fast is default, which helps discover storage errors before the job runs */ 95 | @Test public void failFast() { 96 | addEnvironment(context, 97 | "zipkin.storage.type:elasticsearch", 98 | "zipkin.storage.elasticsearch.hosts:http://host1:9200" 99 | ); 100 | context.register(PropertyPlaceholderAutoConfiguration.class, 101 | ZipkinStorageConsumerAutoConfiguration.class); 102 | 103 | thrown.expect(BeanCreationException.class); 104 | context.refresh(); 105 | } 106 | 107 | StorageConsumer storage() { 108 | return context.getBean(StorageConsumer.class); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /autoconfigure/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 4.0.0 19 | 20 | 21 | io.zipkin.sparkstreaming 22 | zipkin-sparkstreaming-parent 23 | 0.3.10-SNAPSHOT 24 | 25 | 26 | zipkin-sparkstreaming-autoconfigure-parent 27 | Zipkin Spark Streaming Auto Configuration 28 | 29 | pom 30 | 31 | 32 | ${project.basedir}/.. 33 | 34 | 35 | 36 | stream-kafka 37 | adjuster-finagle 38 | consumer-storage 39 | 40 | 41 | 42 | 43 | 44 | 45 | org.springframework.boot 46 | spring-boot-dependencies 47 | ${spring-boot.version} 48 | pom 49 | import 50 | 51 | 52 | 53 | 54 | 55 | 56 | io.zipkin.sparkstreaming 57 | zipkin-sparkstreaming 58 | 59 | 60 | org.springframework.boot 61 | spring-boot-autoconfigure 62 | ${spring-boot.version} 63 | 64 | 65 | org.springframework.boot 66 | spring-boot-test 67 | ${spring-boot.version} 68 | test 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /autoconfigure/stream-kafka/README.md: -------------------------------------------------------------------------------- 1 | # autoconfigure-stream-kafka 2 | 3 | ## ZipkinKafkaStreamFactoryAutoConfiguration 4 | 5 | This is a Spring Boot [AutoConfiguration](http://docs.spring.io/spring-boot/docs/current/reference/html/using-boot-auto-configuration.html) 6 | module built into the [Spark Streaming Job](../../sparkstreaming-job) 7 | which reads encoded lists of spans from a Kafka topic. Internally, this 8 | module wraps the [KafkaStreamFactory](../../stream/kafka) to expose 9 | configuration options via properties. 10 | 11 | ## Usage 12 | 13 | In order to connect, you minimally need to set 14 | `zipkin.sparkstreaming.stream.kafka.bootstrap-servers` or 15 | `zipkin.sparkstreaming.stream.kafka.zookeeper.connect`. 16 | 17 | Ex. 18 | ```bash 19 | java -jar zipkin-sparkstreaming-job.jar \ 20 | --zipkin.sparkstreaming.stream.kafka.bootstrap-servers=127.0.0.1:9092 \ 21 | ... 22 | ``` 23 | 24 | ### Configuration 25 | 26 | Configuration properties can be set via commandline parameters, system 27 | properties or any other alternative [supported by Spring Boot](https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html). 28 | 29 | Besides setting kafka properties, you can also override the following. 30 | All of the below have a prefix of "zipkin.sparkstreaming.stream.kafka" 31 | 32 | Property | Default | Description 33 | --- | --- | --- 34 | topic | zipkin | Kafka topic encoded lists of spans are be consumed from. 35 | group-id | zipkin | Consumer group this process is consuming on behalf of. 36 | bootstrap-servers | none | Initial set of kafka servers to connect to; others may be discovered. Values are in comma-separated host:port syntax. Ex "host1:9092,host2:9092". 37 | zookeeper.connect | none | Looks up bootstrap-servers from Zookeeper. Values is a connect string (comma-separated host:port with optional suffix) Ex "host1:2181,host2:2181". 38 | zookeeper.session-timeout | 10000 | Session timeout for looking up bootstrap-servers. 39 | 40 | ## More Examples 41 | 42 | Ex. to lookup bootstrap servers using Zookeeper 43 | 44 | ```bash 45 | java -jar zipkin-sparkstreaming-job.jar \ 46 | --zipkin.sparkstreaming.stream.kafka.zookeeper.connect=127.0.0.1:2181 47 | ... 48 | ``` 49 | -------------------------------------------------------------------------------- /autoconfigure/stream-kafka/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | io.zipkin.sparkstreaming 20 | zipkin-sparkstreaming-autoconfigure-parent 21 | 0.3.10-SNAPSHOT 22 | 23 | 4.0.0 24 | zipkin-sparkstreaming-autoconfigure-stream-kafka 25 | Zipkin Spark Streaming Auto Configure: Kafka Stream 26 | 27 | 28 | ${project.basedir}/../.. 29 | 30 | 31 | 32 | 33 | io.zipkin.sparkstreaming 34 | zipkin-sparkstreaming-stream-kafka 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /autoconfigure/stream-kafka/src/main/java/zipkin/sparkstreaming/autoconfigure/stream/kafka/ZipkinKafkaStreamFactoryAutoConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.autoconfigure.stream.kafka; 15 | 16 | import org.springframework.boot.autoconfigure.condition.ConditionOutcome; 17 | import org.springframework.boot.autoconfigure.condition.SpringBootCondition; 18 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 19 | import org.springframework.context.annotation.Bean; 20 | import org.springframework.context.annotation.ConditionContext; 21 | import org.springframework.context.annotation.Conditional; 22 | import org.springframework.context.annotation.Configuration; 23 | import org.springframework.core.type.AnnotatedTypeMetadata; 24 | import zipkin.sparkstreaming.StreamFactory; 25 | 26 | @Configuration 27 | @EnableConfigurationProperties(ZipkinKafkaStreamFactoryProperties.class) 28 | @Conditional(ZipkinKafkaStreamFactoryAutoConfiguration.KafkaServersSetCondition.class) 29 | public class ZipkinKafkaStreamFactoryAutoConfiguration { 30 | 31 | @Bean StreamFactory kafkaStream(ZipkinKafkaStreamFactoryProperties properties) { 32 | return properties.toBuilder().build(); 33 | } 34 | 35 | static final class KafkaServersSetCondition extends SpringBootCondition { 36 | static final String BOOTSTRAP = "zipkin.sparkstreaming.stream.kafka.bootstrap-servers"; 37 | static final String CONNECT = "zipkin.sparkstreaming.stream.kafka.zookeeper.connect"; 38 | 39 | @Override 40 | public ConditionOutcome getMatchOutcome(ConditionContext context, AnnotatedTypeMetadata a) { 41 | String bootstrap = context.getEnvironment().getProperty(BOOTSTRAP); 42 | String connect = context.getEnvironment().getProperty(CONNECT); 43 | return (bootstrap == null || bootstrap.isEmpty()) && (connect == null || connect.isEmpty()) ? 44 | ConditionOutcome.noMatch("neither " + BOOTSTRAP + " nor " + CONNECT + " are set") : 45 | ConditionOutcome.match(); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /autoconfigure/stream-kafka/src/main/java/zipkin/sparkstreaming/autoconfigure/stream/kafka/ZipkinKafkaStreamFactoryProperties.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.autoconfigure.stream.kafka; 15 | 16 | import java.util.List; 17 | import org.springframework.boot.context.properties.ConfigurationProperties; 18 | import zipkin.sparkstreaming.stream.kafka.KafkaStreamFactory; 19 | import zipkin.sparkstreaming.stream.kafka.ZookeeperBootstrapServers; 20 | 21 | @ConfigurationProperties("zipkin.sparkstreaming.stream.kafka") 22 | public class ZipkinKafkaStreamFactoryProperties { 23 | private String topic; 24 | private String groupId; 25 | private List bootstrapServers; 26 | private Zookeeper zookeeper = new Zookeeper(); 27 | 28 | public String getTopic() { 29 | return topic; 30 | } 31 | 32 | public void setTopic(String topic) { 33 | this.topic = emptyToNull(topic); 34 | } 35 | 36 | public String getGroupId() { 37 | return groupId; 38 | } 39 | 40 | public void setGroupId(String groupId) { 41 | this.groupId = groupId; 42 | } 43 | 44 | public List getBootstrapServers() { 45 | return bootstrapServers; 46 | } 47 | 48 | public void setBootstrapServers(List bootstrapServers) { 49 | if (bootstrapServers != null && !bootstrapServers.isEmpty()) { 50 | this.bootstrapServers = bootstrapServers; 51 | } 52 | } 53 | 54 | public Zookeeper getZookeeper() { 55 | return zookeeper; 56 | } 57 | 58 | public void setZookeeper(Zookeeper zookeeper) { 59 | if (zookeeper != null) this.zookeeper = zookeeper; 60 | } 61 | 62 | public static class Zookeeper { 63 | private String connect; 64 | private Integer sessionTimeout; 65 | 66 | public String getConnect() { 67 | return connect; 68 | } 69 | 70 | public void setConnect(String connect) { 71 | this.connect = emptyToNull(connect); 72 | } 73 | 74 | public Integer getSessionTimeout() { 75 | return sessionTimeout; 76 | } 77 | 78 | public void setSessionTimeout(Integer sessionTimeout) { 79 | if (sessionTimeout > 0) this.sessionTimeout = sessionTimeout; 80 | } 81 | } 82 | 83 | KafkaStreamFactory.Builder toBuilder() { 84 | KafkaStreamFactory.Builder result = KafkaStreamFactory.newBuilder(); 85 | if (topic != null) result.topic(topic); 86 | if (groupId != null) result.groupId(groupId); 87 | if (bootstrapServers != null) result.bootstrapServers(bootstrapServers); 88 | 89 | if (zookeeper.getConnect() == null) return result; // Zookeeper bootstrap is optional 90 | 91 | ZookeeperBootstrapServers.Builder supplier = ZookeeperBootstrapServers.newBuilder(); 92 | supplier.connect(zookeeper.getConnect()); 93 | if (zookeeper.sessionTimeout != null) supplier.sessionTimeout(zookeeper.sessionTimeout); 94 | result.bootstrapServers(supplier.build()); 95 | return result; 96 | } 97 | 98 | private static String emptyToNull(String s) { 99 | return (s != null && !s.isEmpty()) ? s : null; 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /autoconfigure/stream-kafka/src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- 1 | org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ 2 | zipkin.sparkstreaming.autoconfigure.stream.kafka.ZipkinKafkaStreamFactoryAutoConfiguration -------------------------------------------------------------------------------- /autoconfigure/stream-kafka/src/test/java/zipkin/sparkstreaming/autoconfigure/stream/kafka/ZipkinKafkaStreamFactoryPropertiesTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.autoconfigure.stream.kafka; 15 | 16 | import java.util.Arrays; 17 | import java.util.function.Function; 18 | import org.junit.After; 19 | import org.junit.Test; 20 | import org.junit.runner.RunWith; 21 | import org.junit.runners.Parameterized; 22 | import org.springframework.boot.autoconfigure.context.PropertyPlaceholderAutoConfiguration; 23 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 24 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 25 | import org.springframework.context.annotation.Configuration; 26 | 27 | import static org.assertj.core.api.Assertions.assertThat; 28 | import static org.springframework.boot.test.util.EnvironmentTestUtils.addEnvironment; 29 | 30 | @RunWith(Parameterized.class) 31 | public class ZipkinKafkaStreamFactoryPropertiesTest { 32 | 33 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 34 | 35 | @After 36 | public void close() { 37 | if (context != null) context.close(); 38 | } 39 | 40 | @Parameterized.Parameter(0) public String property; 41 | @Parameterized.Parameter(1) public Object value; 42 | @Parameterized.Parameter(2) public Function extractor; 43 | 44 | @Parameterized.Parameters(name = "{0}") 45 | public static Iterable data() { 46 | return Arrays.asList(new Object[][] { 47 | parameters("topic", "zapkin", p -> p.getTopic()), 48 | parameters("group-id", "zapkin", p -> p.getGroupId()), 49 | parameters("bootstrap-servers", "127.0.0.1:9092", p -> p.getBootstrapServers().get(0)), 50 | parameters("zookeeper.connect", "127.0.0.1:3001", p -> p.getZookeeper().getConnect()), 51 | parameters("zookeeper.session-timeout", 9999, p -> p.getZookeeper().getSessionTimeout()), 52 | }); 53 | } 54 | 55 | /** to allow us to define with a lambda */ 56 | static Object[] parameters(String propertySuffix, T value, 57 | Function extractor) { 58 | return new Object[] {"zipkin.sparkstreaming.stream.kafka." + propertySuffix, value, extractor}; 59 | } 60 | 61 | @Test 62 | public void canOverrideValueOf() { 63 | addEnvironment(context, property + ":" + value); 64 | 65 | context.register( 66 | PropertyPlaceholderAutoConfiguration.class, 67 | EnableKafkaStreamFactoryProperties.class 68 | ); 69 | context.refresh(); 70 | 71 | assertThat(context.getBean(ZipkinKafkaStreamFactoryProperties.class)) 72 | .extracting(extractor) 73 | .containsExactly(value); 74 | } 75 | 76 | @Configuration 77 | @EnableConfigurationProperties(ZipkinKafkaStreamFactoryProperties.class) 78 | static class EnableKafkaStreamFactoryProperties { 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /autoconfigure/stream-kafka/src/test/java/zipkin/sparkstreaming/stream/kafka/ZipkinKafkaStreamFactoryAutoConfigurationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.stream.kafka; 15 | 16 | import org.junit.After; 17 | import org.junit.Rule; 18 | import org.junit.Test; 19 | import org.junit.rules.ExpectedException; 20 | import org.springframework.beans.factory.NoSuchBeanDefinitionException; 21 | import org.springframework.boot.autoconfigure.context.PropertyPlaceholderAutoConfiguration; 22 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 23 | import zipkin.sparkstreaming.autoconfigure.stream.kafka.ZipkinKafkaStreamFactoryAutoConfiguration; 24 | import zipkin.sparkstreaming.autoconfigure.stream.kafka.ZipkinKafkaStreamFactoryProperties; 25 | 26 | import static org.assertj.core.api.Assertions.assertThat; 27 | import static org.springframework.boot.test.util.EnvironmentTestUtils.addEnvironment; 28 | 29 | public class ZipkinKafkaStreamFactoryAutoConfigurationTest { 30 | static final String KAFKA_BOOTSTRAP = "127.0.0.1:9092"; 31 | static final String KAFKA_ZOOKEEPER = "127.0.0.1:2181"; 32 | 33 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 34 | 35 | @Rule 36 | public ExpectedException thrown = ExpectedException.none(); 37 | 38 | @After 39 | public void close() { 40 | if (context != null) context.close(); 41 | } 42 | 43 | @Test 44 | public void doesntProvideCollectorComponent_whenKafkaZookeeperUnset() { 45 | context.register( 46 | PropertyPlaceholderAutoConfiguration.class, 47 | ZipkinKafkaStreamFactoryAutoConfiguration.class 48 | ); 49 | context.refresh(); 50 | 51 | thrown.expect(NoSuchBeanDefinitionException.class); 52 | context.getBean(KafkaStreamFactory.class); 53 | } 54 | 55 | @Test 56 | public void providesCollectorComponent_whenBootstrapServersSet() { 57 | addEnvironment(context, 58 | "zipkin.sparkstreaming.stream.kafka.bootstrap-servers:" + KAFKA_BOOTSTRAP 59 | ); 60 | context.register( 61 | PropertyPlaceholderAutoConfiguration.class, 62 | ZipkinKafkaStreamFactoryAutoConfiguration.class 63 | ); 64 | context.refresh(); 65 | 66 | ZipkinKafkaStreamFactoryProperties props = 67 | context.getBean(ZipkinKafkaStreamFactoryProperties.class); 68 | assertThat(props.getBootstrapServers()) 69 | .containsExactly(KAFKA_BOOTSTRAP); 70 | } 71 | 72 | @Test 73 | public void providesCollectorComponent_whenKafkaZookeeperSet() { 74 | addEnvironment(context, 75 | "zipkin.sparkstreaming.stream.kafka.zookeeper.connect:" + KAFKA_ZOOKEEPER 76 | ); 77 | context.register( 78 | PropertyPlaceholderAutoConfiguration.class, 79 | ZipkinKafkaStreamFactoryAutoConfiguration.class 80 | ); 81 | context.refresh(); 82 | 83 | ZipkinKafkaStreamFactoryProperties props = 84 | context.getBean(ZipkinKafkaStreamFactoryProperties.class); 85 | assertThat(props.getZookeeper().getConnect()) 86 | .isEqualTo(KAFKA_ZOOKEEPER); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /build-support/go-offline.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2017 The OpenZipkin Authors 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 6 | # in compliance with the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software distributed under the License 11 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 12 | # or implied. See the License for the specific language governing permissions and limitations under 13 | # the License. 14 | # 15 | 16 | # Due to https://issues.apache.org/jira/browse/MDEP-323 and cross-module dependencies, 17 | # we can't easily run mvn dependency:go-offline. This is a workaround for that. 18 | # It removes all dependencies on io.zipkin.java and ${project.groupId} using XSLT, 19 | # then runs go-offline on the resulting POMs. 20 | 21 | set -xeuo pipefail 22 | 23 | rm -rf go-offline-builddir 24 | mkdir -p go-offline-builddir 25 | trap "rm -rf $(pwd)/go-offline-builddir" EXIT 26 | 27 | for f in $(find . -name 'pom.xml'); do 28 | echo $f 29 | mkdir -p $(dirname go-offline-builddir/$f) 30 | xsltproc ./build-support/pom-no-crossmodule-dependencies.xsl $f > go-offline-builddir/$f 31 | done 32 | 33 | cd go-offline-builddir 34 | ../mvnw dependency:go-offline 35 | -------------------------------------------------------------------------------- /build-support/pom-no-crossmodule-dependencies.xsl: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /build-support/publish-snapshot.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2017 The OpenZipkin Authors 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 6 | # in compliance with the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software distributed under the License 11 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 12 | # or implied. See the License for the specific language governing permissions and limitations under 13 | # the License. 14 | # 15 | 16 | set -euo pipefail 17 | set -x 18 | 19 | if ./mvnw help:evaluate -N -Dexpression=project.version | grep -v '\[' | grep -q SNAPSHOT; then 20 | ./mvnw --batch-mode -s ./.settings.xml -Prelease -nsu -DskipTests deploy 21 | else 22 | echo "Not building release versions, those are built by the tag builder using the publish-stable.sh script" 23 | fi 24 | -------------------------------------------------------------------------------- /build-support/publish-stable.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2017 The OpenZipkin Authors 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 6 | # in compliance with the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software distributed under the License 11 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 12 | # or implied. See the License for the specific language governing permissions and limitations under 13 | # the License. 14 | # 15 | 16 | set -euo pipefail 17 | set -x 18 | 19 | ./mvnw -DskipTests install -nsu 20 | ./mvnw --batch-mode -s ./.settings.xml -Prelease -nsu -DskipTests deploy 21 | ./mvnw --batch-mode -s ./.settings.xml -nsu -N io.zipkin.centralsync-maven-plugin:centralsync-maven-plugin:sync 22 | -------------------------------------------------------------------------------- /build-support/trigger-publish.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2017 The OpenZipkin Authors 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 6 | # in compliance with the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software distributed under the License 11 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 12 | # or implied. See the License for the specific language governing permissions and limitations under 13 | # the License. 14 | # 15 | 16 | set -euo pipefail 17 | set -x 18 | 19 | release_version() { 20 | echo "${CIRCLE_TAG}" | sed 's/^release-//' 21 | } 22 | 23 | safe_checkout_master() { 24 | # We need to be on a branch for release:perform to be able to create commits, and we want that branch to be master. 25 | # But we also want to make sure that we build and release exactly the tagged version, so we verify that the remote 26 | # master is where our tag is. 27 | git checkout -B master 28 | git fetch origin master:origin/master 29 | commit_local_master="$(git show --pretty='format:%H' master)" 30 | commit_remote_master="$(git show --pretty='format:%H' origin/master)" 31 | if [ "$commit_local_master" != "$commit_remote_master" ]; then 32 | echo "Master on remote 'origin' has commits since the version under release, aborting" 33 | exit 1 34 | fi 35 | } 36 | 37 | safe_checkout_master 38 | ./mvnw --batch-mode -s ./.settings.xml -Prelease -nsu -DreleaseVersion="$(release_version)" -Darguments="-DskipTests" release:prepare 39 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2017 The OpenZipkin Authors 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | # in compliance with the License. You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software distributed under the License 10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | # or implied. See the License for the specific language governing permissions and limitations under 12 | # the License. 13 | # 14 | 15 | machine: 16 | java: 17 | version: openjdk8 18 | 19 | dependencies: 20 | override: 21 | - sudo apt-get install xsltproc 22 | - ./build-support/go-offline.sh 23 | 24 | test: 25 | post: 26 | # parameters used during release 27 | # allocate commits to CI, not the owner of the deploy key 28 | - git config user.name "zipkinci" 29 | - git config user.email "zipkinci+zipkin-dev@googlegroups.com" 30 | # setup https authentication credentials, used by ./mvnw release:prepare 31 | - git config credential.helper "store --file=.git/credentials" 32 | - echo "https://$GH_TOKEN:@github.com" > .git/credentials 33 | # copy test reports to CircleCI test reports directory 34 | - mkdir -p $CIRCLE_TEST_REPORTS/junit/ 35 | - find . -type f -regex ".*/target/surefire-reports/.*xml" -exec cp {} $CIRCLE_TEST_REPORTS/junit/ \; 36 | 37 | # Send notifications to Gitter 38 | notify: 39 | webhooks: 40 | - url: https://webhooks.gitter.im/e/22adbb9973299092e6ae 41 | 42 | deployment: 43 | trigger: 44 | owner: openzipkin 45 | tag: /release-\d+\.\d+\.\d+/ 46 | commands: 47 | - ./build-support/trigger-publish.sh 48 | publish-stable: 49 | owner: openzipkin 50 | tag: /\d+\.\d+\.\d+/ 51 | commands: 52 | - ./build-support/publish-stable.sh 53 | publish-snapshot: 54 | owner: openzipkin 55 | branch: master 56 | commands: 57 | - ./build-support/publish-snapshot.sh 58 | -------------------------------------------------------------------------------- /consumer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 4.0.0 19 | 20 | 21 | io.zipkin.sparkstreaming 22 | zipkin-sparkstreaming-parent 23 | 0.3.10-SNAPSHOT 24 | 25 | 26 | zipkin-sparkstreaming-consumer-parent 27 | Zipkin Spark Streaming: Consumers 28 | pom 29 | 30 | 31 | ${project.basedir}/.. 32 | 33 | 34 | 35 | storage 36 | 37 | 38 | 39 | 40 | ${project.groupId} 41 | zipkin-sparkstreaming 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /consumer/storage/README.md: -------------------------------------------------------------------------------- 1 | # consumer-storage 2 | 3 | ## StorageConsumer 4 | 5 | This consumer writes spans to a Zipkin `StorageComponent`, such as 6 | Elasticsearch. 7 | 8 | ## Usage 9 | 10 | While the `StorageConsumer` can be used directly through the provided 11 | builder interface, most users will likely find more value in the Spring 12 | Boot autoconfiguraton module. Additional information for using the 13 | module can be found [here](../../autoconfigure/adjuster-finagle). 14 | -------------------------------------------------------------------------------- /consumer/storage/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | io.zipkin.sparkstreaming 20 | zipkin-sparkstreaming-consumer-parent 21 | 0.3.10-SNAPSHOT 22 | 23 | 4.0.0 24 | 25 | zipkin-sparkstreaming-consumer-storage 26 | Zipkin Spark Streaming Consumer: Zipkin Storage 27 | 28 | 29 | ${project.basedir}/../.. 30 | 31 | 32 | 33 | 34 | com.google.auto.value 35 | auto-value 36 | provided 37 | 38 | 39 | 40 | org.mockito 41 | mockito-core 42 | 2.8.9 43 | test 44 | 45 | 46 | com.portingle 47 | slf4jtesting 48 | 1.1.3 49 | test 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /consumer/storage/src/main/java/zipkin/sparkstreaming/consumer/storage/StorageConsumer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.consumer.storage; 15 | 16 | import java.io.IOException; 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | import zipkin.Component; 22 | import zipkin.Span; 23 | import zipkin.internal.CallbackCaptor; 24 | import zipkin.sparkstreaming.Consumer; 25 | import zipkin.storage.StorageComponent; 26 | 27 | /** A storage consumer which writes to storage on {@link #accept(Iterable)}. */ 28 | public abstract class StorageConsumer implements Consumer, Component { 29 | private static final Logger log = LoggerFactory.getLogger(StorageConsumer.class); 30 | 31 | Logger log() { // Override for testing. Instance variables won't work as Logger isn't serializable 32 | return log; 33 | } 34 | 35 | transient volatile StorageComponent instance; // not serializable 36 | 37 | /** Subclasses should initialize this from serializable state. */ 38 | protected abstract StorageComponent tryCompute(); 39 | 40 | @Override public final void accept(Iterable spansSharingId) { 41 | List list = asList(spansSharingId); 42 | if (list.isEmpty()) { 43 | log().debug("Input was empty"); 44 | return; 45 | } 46 | 47 | // Blocking as it is simpler to reason with thread this way while work is in progress 48 | CallbackCaptor blockingCallback = new CallbackCaptor<>(); 49 | try { 50 | get().asyncSpanConsumer().accept(list, blockingCallback); 51 | blockingCallback.get(); 52 | log().debug("Wrote {} spans", list.size()); 53 | } catch (RuntimeException e) { 54 | Throwable toLog = e.getClass().equals(RuntimeException.class) && e.getCause() != null 55 | ? e.getCause() // callback captor wraps checked exceptions 56 | : e; 57 | String message = "Dropped " + list.size() + " spans: " + toLog.getMessage(); 58 | 59 | // TODO: We are dropping vs diverting to a dead letter queue or otherwise. Do we want this? 60 | if (log().isWarnEnabled()) { 61 | log().warn(message, toLog); 62 | } else { 63 | log().warn(message); 64 | } 65 | } 66 | } 67 | 68 | final StorageComponent get() { 69 | StorageComponent result = instance; 70 | if (result == null) { 71 | synchronized (this) { 72 | result = instance; 73 | if (result == null) { 74 | instance = result = tryCompute(); 75 | } 76 | } 77 | } 78 | return result; 79 | } 80 | 81 | @Override public CheckResult check() { 82 | return get().check(); 83 | } 84 | 85 | @Override public final void close() throws IOException { 86 | synchronized (this) { 87 | if (instance != null) instance.close(); 88 | } 89 | } 90 | 91 | static List asList(Iterable iter) { 92 | List list = new ArrayList(); 93 | for (E item : iter) list.add(item); 94 | return list; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /consumer/storage/src/test/java/zipkin/sparkstreaming/consumer/storage/StorageConsumerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.consumer.storage; 15 | 16 | import java.io.IOException; 17 | import java.util.Collections; 18 | import java.util.concurrent.CountDownLatch; 19 | import java.util.concurrent.ExecutorService; 20 | import java.util.concurrent.Executors; 21 | import java.util.concurrent.TimeUnit; 22 | import java.util.concurrent.atomic.AtomicInteger; 23 | import java.util.function.Consumer; 24 | import org.junit.Rule; 25 | import org.junit.Test; 26 | import org.junit.rules.ExpectedException; 27 | import org.mockito.stubbing.Answer; 28 | import org.slf4j.Logger; 29 | import slf4jtest.LogLevel; 30 | import slf4jtest.Settings; 31 | import slf4jtest.TestLogger; 32 | import zipkin.TestObjects; 33 | import zipkin.storage.AsyncSpanConsumer; 34 | import zipkin.storage.Callback; 35 | import zipkin.storage.InMemoryStorage; 36 | import zipkin.storage.StorageComponent; 37 | 38 | import static org.assertj.core.api.Assertions.assertThat; 39 | import static org.assertj.core.api.Assertions.tuple; 40 | import static org.mockito.ArgumentMatchers.any; 41 | import static org.mockito.ArgumentMatchers.eq; 42 | import static org.mockito.Mockito.doAnswer; 43 | import static org.mockito.Mockito.mock; 44 | import static org.mockito.Mockito.when; 45 | 46 | public class StorageConsumerTest { 47 | @Rule public ExpectedException thrown = ExpectedException.none(); 48 | 49 | StorageComponent storage = mock(StorageComponent.class); 50 | TestLogger logger = new Settings().enableAll().buildLogging().getLogger(""); 51 | StorageConsumer storageConsumer = new StorageConsumer() { 52 | @Override Logger log() { 53 | return logger; 54 | } 55 | 56 | @Override protected StorageComponent tryCompute() { 57 | return storage; 58 | } 59 | }; 60 | 61 | @Test 62 | public void logsWhenEmpty() { 63 | storageConsumer.accept(Collections.emptyList()); 64 | assertThat(logger.lines()) 65 | .extracting("level", "text") 66 | .containsExactly(tuple(LogLevel.DebugLevel, "Input was empty")); 67 | } 68 | 69 | @Test 70 | public void acceptsTrace() { 71 | storage = new InMemoryStorage(); 72 | StorageConsumer storageConsumer = new StorageConsumer() { 73 | @Override Logger log() { 74 | return logger; 75 | } 76 | 77 | @Override protected StorageComponent tryCompute() { 78 | return storage; 79 | } 80 | }; 81 | 82 | storageConsumer.accept(TestObjects.TRACE); 83 | assertThat(logger.lines()) 84 | .extracting("level", "text") 85 | .containsExactly(tuple(LogLevel.DebugLevel, "Wrote 3 spans")); 86 | 87 | assertThat(storage.spanStore().getRawTrace( 88 | TestObjects.TRACE.get(0).traceIdHigh, 89 | TestObjects.TRACE.get(0).traceId 90 | )).isEqualTo(TestObjects.TRACE); 91 | } 92 | 93 | @Test 94 | public void logsOnAcceptError() { 95 | IllegalStateException acceptException = new IllegalStateException("failed"); 96 | 97 | AsyncSpanConsumer consumer = mock(AsyncSpanConsumer.class); 98 | when(storage.asyncSpanConsumer()).thenReturn(consumer); 99 | doAnswer(answer(c -> { 100 | throw acceptException; 101 | })).when(consumer).accept(eq(TestObjects.TRACE), any(Callback.class)); 102 | 103 | storageConsumer.accept(TestObjects.TRACE); 104 | 105 | assertThat(logger.lines()).hasSize(1); 106 | assertThat(logger.lines().iterator().next().toString()) 107 | .startsWith("LogMessage(,WARN,Dropped 3 spans: failed"); 108 | // TODO: test for acceptException 109 | } 110 | 111 | @Test 112 | public void logsOnCallbackError() { 113 | IllegalStateException callbackException = new IllegalStateException("failed"); 114 | 115 | AsyncSpanConsumer consumer = mock(AsyncSpanConsumer.class); 116 | when(storage.asyncSpanConsumer()).thenReturn(consumer); 117 | doAnswer(answer(c -> c.onError(callbackException))) 118 | .when(consumer).accept(eq(TestObjects.TRACE), any(Callback.class)); 119 | 120 | storageConsumer.accept(TestObjects.TRACE); 121 | 122 | assertThat(logger.lines()).hasSize(1); 123 | assertThat(logger.lines().iterator().next().toString()) 124 | .startsWith("LogMessage(,WARN,Dropped 3 spans: failed"); 125 | // TODO: test for callbackException 126 | } 127 | 128 | @Test 129 | public void doesntWrapCheckedExceptionOnCallbackError() { 130 | IOException callbackException = new IOException("failed"); 131 | 132 | AsyncSpanConsumer consumer = mock(AsyncSpanConsumer.class); 133 | when(storage.asyncSpanConsumer()).thenReturn(consumer); 134 | doAnswer(answer(c -> c.onError(callbackException))) 135 | .when(consumer).accept(eq(TestObjects.TRACE), any(Callback.class)); 136 | 137 | storageConsumer.accept(TestObjects.TRACE); 138 | 139 | assertThat(logger.lines()).hasSize(1); 140 | assertThat(logger.lines().iterator().next().toString()) 141 | .startsWith("LogMessage(,WARN,Dropped 3 spans: failed"); 142 | // TODO: test for callbackException 143 | } 144 | 145 | @Test(timeout = 1000L) 146 | public void get_memoizes() throws InterruptedException { 147 | AtomicInteger provisionCount = new AtomicInteger(); 148 | 149 | StorageConsumer storageConsumer = new StorageConsumer() { 150 | @Override protected StorageComponent tryCompute() { 151 | provisionCount.incrementAndGet(); 152 | return new InMemoryStorage(); 153 | } 154 | }; 155 | 156 | int getCount = 1000; 157 | CountDownLatch latch = new CountDownLatch(getCount); 158 | ExecutorService exec = Executors.newFixedThreadPool(10); 159 | for (int i = 0; i < getCount; i++) { 160 | exec.execute(() -> { 161 | storageConsumer.get(); 162 | latch.countDown(); 163 | }); 164 | } 165 | latch.await(); 166 | exec.shutdown(); 167 | exec.awaitTermination(1, TimeUnit.SECONDS); 168 | 169 | assertThat(provisionCount.get()).isEqualTo(1); 170 | } 171 | 172 | static Answer answer(Consumer> onCallback) { 173 | return invocation -> { 174 | onCallback.accept((Callback) invocation.getArguments()[invocation.getArguments().length - 1]); 175 | return null; 176 | }; 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Mingw, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 204 | if [ "$MVNW_VERBOSE" = true ]; then 205 | echo $MAVEN_PROJECTBASEDIR 206 | fi 207 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 208 | 209 | # For Cygwin, switch paths to Windows format before running java 210 | if $cygwin; then 211 | [ -n "$M2_HOME" ] && 212 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 213 | [ -n "$JAVA_HOME" ] && 214 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 215 | [ -n "$CLASSPATH" ] && 216 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 217 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 218 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 219 | fi 220 | 221 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 222 | 223 | exec "$JAVACMD" \ 224 | $MAVEN_OPTS \ 225 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 226 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 227 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 228 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 50 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | 121 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 122 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 123 | 124 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 125 | if ERRORLEVEL 1 goto error 126 | goto end 127 | 128 | :error 129 | set ERROR_CODE=1 130 | 131 | :end 132 | @endlocal & set ERROR_CODE=%ERROR_CODE% 133 | 134 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 135 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 136 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 137 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 138 | :skipRcPost 139 | 140 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 141 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 142 | 143 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 144 | 145 | exit /B %ERROR_CODE% 146 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 4.0.0 19 | 20 | io.zipkin.sparkstreaming 21 | zipkin-sparkstreaming-parent 22 | 0.3.10-SNAPSHOT 23 | pom 24 | 25 | 26 | sparkstreaming 27 | stream 28 | adjuster 29 | consumer 30 | autoconfigure 31 | sparkstreaming-job 32 | 33 | 34 | 35 | UTF-8 36 | UTF-8 37 | 38 | 39 | 1.7 40 | java17 41 | 42 | ${project.basedir} 43 | 44 | 45 | 2.10 46 | 1.6.3 47 | 48 | 49 | 2.5.0 50 | 3.4.0 51 | 1.5.10.RELEASE 52 | 53 | 54 | Zipkin Spark Streaming (Parent) 55 | Zipkin Spark Streaming (Parent) 56 | https://github.com/openzipkin/zipkin-sparkstreaming 57 | 2017 58 | 59 | 60 | OpenZipkin 61 | http://zipkin.io/ 62 | 63 | 64 | 65 | 66 | The Apache Software License, Version 2.0 67 | http://www.apache.org/licenses/LICENSE-2.0.txt 68 | repo 69 | 70 | 71 | 72 | 73 | https://github.com/openzipkin/zipkin-sparkstreaming 74 | scm:git:https://github.com/openzipkin/zipkin-sparkstreaming.git 75 | scm:git:https://github.com/openzipkin/zipkin-sparkstreaming.git 76 | 77 | HEAD 78 | 79 | 80 | 81 | 82 | 83 | openzipkin 84 | OpenZipkin Gitter 85 | https://gitter.im/openzipkin/zipkin 86 | 87 | 88 | 89 | 90 | 91 | bintray 92 | https://api.bintray.com/maven/openzipkin/maven/zipkin-sparkstreaming/;publish=1 93 | 94 | 95 | jfrog-snapshots 96 | http://oss.jfrog.org/artifactory/oss-snapshot-local 97 | 98 | 99 | 100 | 101 | Github 102 | https://github.com/openzipkin/zipkin-sparkstreaming/issues 103 | 104 | 105 | 106 | 107 | 108 | ${project.groupId} 109 | zipkin-sparkstreaming 110 | ${project.version} 111 | 112 | 113 | 114 | ${project.groupId} 115 | zipkin-autoconfigure-sparkstreaming 116 | ${project.version} 117 | 118 | 119 | 120 | ${project.groupId} 121 | zipkin-sparkstreaming-stream-kafka 122 | ${project.version} 123 | 124 | 125 | 126 | ${project.groupId} 127 | zipkin-sparkstreaming-autoconfigure-stream-kafka 128 | ${project.version} 129 | 130 | 131 | 132 | ${project.groupId} 133 | zipkin-sparkstreaming-consumer-storage 134 | ${project.version} 135 | 136 | 137 | 138 | ${project.groupId} 139 | zipkin-sparkstreaming-autoconfigure-consumer-storage 140 | ${project.version} 141 | 142 | 143 | 144 | ${project.groupId} 145 | zipkin-sparkstreaming-adjuster-finagle 146 | ${project.version} 147 | 148 | 149 | 150 | ${project.groupId} 151 | zipkin-sparkstreaming-autoconfigure-adjuster-finagle 152 | ${project.version} 153 | 154 | 155 | 156 | io.zipkin.java 157 | zipkin 158 | ${zipkin.version} 159 | 160 | 161 | 162 | org.apache.spark 163 | spark-core_${scala.binary.version} 164 | ${spark.version} 165 | 166 | 167 | org.slf4j 168 | slf4j-log4j12 169 | 170 | 171 | 172 | 173 | 174 | org.apache.spark 175 | spark-streaming_${scala.binary.version} 176 | ${spark.version} 177 | 178 | 179 | 180 | 181 | com.fasterxml.jackson.module 182 | jackson-module-scala_${scala.binary.version} 183 | 2.8.8 184 | 185 | 186 | 187 | 188 | com.datastax.cassandra 189 | cassandra-driver-core 190 | ${cassandra-driver-core.version} 191 | 192 | 193 | com.datastax.cassandra 194 | cassandra-driver-mapping 195 | ${cassandra-driver-core.version} 196 | 197 | 198 | 199 | com.google.auto.value 200 | auto-value 201 | 1.5.3 202 | 203 | 204 | 205 | 206 | 207 | 208 | io.zipkin.java 209 | zipkin 210 | 211 | 212 | 213 | org.slf4j 214 | slf4j-log4j12 215 | 1.7.25 216 | 217 | 218 | 219 | log4j 220 | log4j 221 | 1.2.17 222 | 223 | 224 | 225 | junit 226 | junit 227 | 4.12 228 | test 229 | 230 | 231 | 232 | org.assertj 233 | assertj-core 234 | 3.9.0 235 | test 236 | 237 | 238 | 239 | io.zipkin.java 240 | zipkin 241 | ${zipkin.version} 242 | test-jar 243 | test 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | io.takari 253 | maven 254 | 0.5.0 255 | 256 | 257 | 258 | 259 | 260 | 261 | true 262 | maven-compiler-plugin 263 | 3.7.0 264 | 265 | 266 | 1.8 267 | 1.8 268 | javac-with-errorprone 269 | true 270 | true 271 | 272 | 273 | 274 | org.codehaus.plexus 275 | plexus-compiler-javac-errorprone 276 | 2.8.2 277 | 278 | 279 | com.google.errorprone 280 | error_prone_core 281 | 2.2.0 282 | 283 | 284 | 285 | 286 | 287 | net.orfjackal.retrolambda 288 | retrolambda-maven-plugin 289 | 2.5.3 290 | 291 | 292 | 293 | process-main 294 | 295 | 296 | ${main.java.version} 297 | true 298 | true 299 | 300 | 301 | 302 | 303 | 304 | 305 | org.codehaus.mojo 306 | animal-sniffer-maven-plugin 307 | 1.16 308 | 309 | 310 | org.codehaus.mojo.signature 311 | ${main.signature.artifact} 312 | 1.0 313 | 314 | 315 | 316 | 317 | 318 | check 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | maven-install-plugin 327 | 2.5.2 328 | 329 | true 330 | 331 | 332 | 333 | 334 | com.mycila 335 | license-maven-plugin 336 | 3.0 337 | 338 |

${main.basedir}/src/etc/header.txt
339 | 340 | .travis.yml 341 | .gitignore 342 | .mvn/** 343 | mvnw* 344 | etc/header.txt 345 | **/.idea/** 346 | LICENSE 347 | **/*.md 348 | src/test/resources/** 349 | src/main/resources/** 350 | 351 | true 352 | 353 | 354 | 355 | com.mycila 356 | license-maven-plugin-git 357 | 3.0 358 | 359 | 360 | 361 | 362 | 363 | check 364 | 365 | compile 366 | 367 | 368 | 369 | 370 | 371 | maven-release-plugin 372 | 2.5.3 373 | 374 | false 375 | release 376 | true 377 | @{project.version} 378 | 379 | 380 | 381 | 382 | io.zipkin.centralsync-maven-plugin 383 | centralsync-maven-plugin 384 | 0.1.0 385 | 386 | zipkin-sparkstreaming 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | release 395 | 396 | 397 | 398 | 399 | maven-source-plugin 400 | 3.0.1 401 | 402 | 403 | attach-sources 404 | 405 | jar 406 | 407 | 408 | 409 | 410 | 411 | 412 | maven-javadoc-plugin 413 | 2.10.4 414 | 415 | false 416 | zipkin.sparkstreaming.internal,zipkin.sparkstreaming.internal.* 417 | 418 | 419 | -Xdoclint:none 420 | 421 | 422 | 423 | attach-javadocs 424 | 425 | jar 426 | 427 | package 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | 436 | -------------------------------------------------------------------------------- /sparkstreaming-job/README.md: -------------------------------------------------------------------------------- 1 | # sparkstreaming-job 2 | 3 | This is a Spring Boot Application that launches the spark streaming job. 4 | It primarily uses properties for configuration. 5 | 6 | ## Usage 7 | 8 | To start the job, you need to minimally set properties for a stream 9 | (like kafka) and a consumer (like storage). Built-in options are located 10 | in the [autoconfigure](../autoconfigure) module. 11 | 12 | Ex. To receive messages from Kafka and store them into Elasticsearch: 13 | ```bash 14 | java -jar zipkin-sparkstreaming-job.jar \ 15 | --zipkin.storage.type=elasticsearch \ 16 | --zipkin.storage.elasticsearch.hosts=http://127.0.0.1:9200 \ 17 | --zipkin.sparkstreaming.stream.kafka.bootstrap-servers=127.0.0.1:9092 18 | ``` 19 | 20 | ### Configuration 21 | 22 | Configuration properties can be set via commandline parameters, system 23 | properties or any other alternative [supported by Spring Boot](https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html). 24 | 25 | Here are the relevant setting and a short description. With the exception 26 | of "zipkin.log-level", properties all have a prefix of "zipkin.sparkstreaming" 27 | 28 | Property | Default | Description 29 | --- | --- | --- 30 | zipkin.log-level | info | Logging level for the category "zipkin". Set to debug for details. 31 | master | `local[*]` | The spark master used for this job. `local[*]` means run on-demand w/o connecting to a cluster. 32 | jars | the exec jar | Indicates which jars to distribute to the cluster. 33 | conf | "spark.ui.enabled=false" | Overrides the properties used to create a SparkConf 34 | batch-duration | 10000 | The time interval in millis at which streaming data will be divided into batches 35 | 36 | Ex. to manually control spark conf, add properties prefixed with `zipkin.sparkstreaming.conf`: 37 | ```bash 38 | java -jar zipkin-sparkstreaming-job.jar \ 39 | --zipkin.sparkstreaming.conf.spark.eventLog.enabled=false \ 40 | ... 41 | ``` 42 | -------------------------------------------------------------------------------- /sparkstreaming-job/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | io.zipkin.sparkstreaming 20 | 0.3.10-SNAPSHOT 21 | zipkin-sparkstreaming-parent 22 | 23 | 4.0.0 24 | 25 | zipkin-sparkstreaming-job 26 | Zipkin Spark Streaming Job 27 | 28 | 29 | ${project.basedir}/.. 30 | 31 | 32 | 33 | 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-dependencies 38 | ${spring-boot.version} 39 | pom 40 | import 41 | 42 | 43 | 44 | 45 | 46 | 47 | ${project.groupId} 48 | zipkin-sparkstreaming 49 | 50 | 51 | 52 | ${project.groupId} 53 | zipkin-sparkstreaming-autoconfigure-stream-kafka 54 | 55 | 56 | 57 | ${project.groupId} 58 | zipkin-sparkstreaming-autoconfigure-consumer-storage 59 | 60 | 61 | 62 | ${project.groupId} 63 | zipkin-sparkstreaming-autoconfigure-adjuster-finagle 64 | 65 | 66 | 67 | org.springframework.boot 68 | spring-boot-autoconfigure 69 | 70 | 71 | 72 | org.springframework.boot 73 | spring-boot-starter 74 | 75 | 76 | 77 | org.springframework.boot 78 | spring-boot-starter-logging 79 | 80 | 81 | 82 | 83 | 84 | org.springframework.boot 85 | spring-boot-starter-test 86 | test 87 | 88 | 89 | 90 | 91 | 92 | 93 | org.springframework.boot 94 | spring-boot-maven-plugin 95 | ${spring-boot.version} 96 | 97 | exec 98 | 99 | 100 | 101 | maven-shade-plugin 102 | 3.1.0 103 | 104 | 105 | package 106 | 107 | shade 108 | 109 | 110 | 111 | 112 | 113 | reference.conf 114 | 115 | 116 | zipkin.sparkstreaming.job.ZipkinSparkStreamingJob 117 | 118 | 119 | 120 | true 121 | 122 | 123 | 124 | io.zipkin.sparkstreaming:* 125 | 126 | ** 127 | 128 | 129 | 130 | org.mariadb.jdbc:mariadb-java-client 131 | 132 | ** 133 | 134 | 135 | 136 | org.springframework:spring-context 137 | 138 | ** 139 | 140 | 141 | 142 | org.springframework.boot:* 143 | 144 | ** 145 | 146 | 147 | 148 | com.fasterxml.jackson.core:jackson-databind 149 | 150 | ** 151 | 152 | 153 | 154 | org.apache.hadoop:hadoop-common 155 | 156 | ** 157 | 158 | 159 | 160 | log4j:log4j 161 | 162 | ** 163 | 164 | 165 | 166 | io.netty:* 167 | 168 | ** 169 | 170 | 171 | 172 | org.slf4j:* 173 | 174 | ** 175 | 176 | 177 | 178 | org.scala-lang:* 179 | 180 | ** 181 | 182 | 183 | 184 | org.apache.spark:* 185 | 186 | ** 187 | 188 | 189 | 190 | com.typesafe.akka:* 191 | 192 | ** 193 | 194 | 195 | 196 | 197 | *:* 198 | 199 | META-INF/*.SF 200 | META-INF/*.DSA 201 | META-INF/*.RSA 202 | 203 | 204 | 205 | false 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | -------------------------------------------------------------------------------- /sparkstreaming-job/src/main/java/zipkin/sparkstreaming/job/ZipkinSparkStreamingConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.job; 15 | 16 | import java.io.UnsupportedEncodingException; 17 | import java.net.URL; 18 | import java.net.URLDecoder; 19 | import java.util.ArrayList; 20 | import java.util.Collections; 21 | import java.util.LinkedHashSet; 22 | import java.util.List; 23 | import java.util.Set; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | import org.springframework.beans.factory.annotation.Autowired; 27 | import org.springframework.beans.factory.annotation.Value; 28 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 29 | import org.springframework.context.annotation.Bean; 30 | import org.springframework.context.annotation.Configuration; 31 | import zipkin.sparkstreaming.Adjuster; 32 | import zipkin.sparkstreaming.Consumer; 33 | import zipkin.sparkstreaming.SparkStreamingJob; 34 | import zipkin.sparkstreaming.StreamFactory; 35 | 36 | @Configuration 37 | @EnableConfigurationProperties(ZipkinSparkStreamingProperties.class) 38 | public class ZipkinSparkStreamingConfiguration { 39 | final Logger log = LoggerFactory.getLogger(ZipkinSparkStreamingConfiguration.class); 40 | 41 | @Autowired(required = false) 42 | List adjusters = Collections.emptyList(); 43 | 44 | @Bean SparkStreamingJob sparkStreaming( 45 | ZipkinSparkStreamingProperties sparkStreaming, 46 | @Value("${zipkin.log-level:info}") String zipkinLogLevel, 47 | StreamFactory streamFactory, 48 | Consumer consumer 49 | ) { 50 | SparkStreamingJob.Builder builder = sparkStreaming.toBuilder(); 51 | if (!"".equals(zipkinLogLevel)) builder.zipkinLogLevel(zipkinLogLevel); 52 | if (sparkStreaming.getMaster() != null && sparkStreaming.getJars() == null) { 53 | List pathToJars = pathToJars(ZipkinSparkStreamingJob.class, adjusters); 54 | if (pathToJars != null) { 55 | log.info("Will distribute the following jars to the cluster: " + pathToJars); 56 | builder.jars(pathToJars); 57 | } 58 | } 59 | return builder.streamFactory(streamFactory) 60 | .adjusters(adjusters) 61 | .consumer(consumer) 62 | .build() 63 | .start(); 64 | } 65 | 66 | /** 67 | * This assumes everything is in the uber-jar except perhaps the adjusters (which are themselves 68 | * self-contained jars). 69 | */ 70 | static List pathToJars(Class entryPoint, List adjusters) { 71 | Set jars = new LinkedHashSet<>(); 72 | jars.add(pathToJar(entryPoint)); 73 | for (Adjuster adjuster : adjusters) { 74 | jars.add(pathToJar(adjuster.getClass())); 75 | } 76 | jars.remove(null); 77 | return jars.isEmpty() ? null : new ArrayList<>(jars); 78 | } 79 | 80 | static String pathToJar(Class type) { 81 | URL jarFile = type.getProtectionDomain().getCodeSource().getLocation(); 82 | try { 83 | return URLDecoder.decode(jarFile.getPath(), "UTF-8"); 84 | } catch (UnsupportedEncodingException e) { 85 | return null; 86 | } 87 | } 88 | } 89 | 90 | -------------------------------------------------------------------------------- /sparkstreaming-job/src/main/java/zipkin/sparkstreaming/job/ZipkinSparkStreamingJob.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.job; 15 | 16 | import org.springframework.boot.autoconfigure.SpringBootApplication; 17 | import org.springframework.boot.builder.SpringApplicationBuilder; 18 | import org.springframework.context.annotation.Import; 19 | import zipkin.sparkstreaming.SparkStreamingJob; 20 | import zipkin.sparkstreaming.autoconfigure.adjuster.finagle.ZipkinFinagleAdjusterAutoConfiguration; 21 | import zipkin.sparkstreaming.autoconfigure.consumer.storage.ZipkinStorageConsumerAutoConfiguration; 22 | import zipkin.sparkstreaming.autoconfigure.stream.kafka.ZipkinKafkaStreamFactoryAutoConfiguration; 23 | 24 | @SpringBootApplication 25 | @Import({ 26 | ZipkinSparkStreamingConfiguration.class, 27 | // These need to be explicity included as the shade plugin squashes spring.properties 28 | ZipkinKafkaStreamFactoryAutoConfiguration.class, 29 | ZipkinFinagleAdjusterAutoConfiguration.class, 30 | ZipkinStorageConsumerAutoConfiguration.class 31 | }) 32 | public class ZipkinSparkStreamingJob { 33 | 34 | public static void main(String[] args) { 35 | new SpringApplicationBuilder(ZipkinSparkStreamingJob.class) 36 | .run(args) 37 | .getBean(SparkStreamingJob.class).awaitTermination(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /sparkstreaming-job/src/main/java/zipkin/sparkstreaming/job/ZipkinSparkStreamingProperties.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.job; 15 | 16 | import java.util.LinkedHashMap; 17 | import java.util.List; 18 | import java.util.Map; 19 | import org.springframework.boot.context.properties.ConfigurationProperties; 20 | import zipkin.sparkstreaming.SparkStreamingJob; 21 | 22 | @ConfigurationProperties("zipkin.sparkstreaming") 23 | public class ZipkinSparkStreamingProperties { 24 | String master; 25 | List jars; 26 | Map conf = new LinkedHashMap<>(); 27 | Long batchDuration; 28 | 29 | public String getMaster() { 30 | return master; 31 | } 32 | 33 | public void setMaster(String master) { 34 | this.master = "".equals(master) ? null : master; 35 | } 36 | 37 | public List getJars() { 38 | return jars; 39 | } 40 | 41 | public void setJars(List jars) { 42 | if (jars != null && !jars.isEmpty()) this.jars = jars; 43 | } 44 | 45 | public Map getConf() { 46 | return conf; 47 | } 48 | 49 | public void setConf(Map conf) { 50 | if (conf != null && !conf.isEmpty()) { 51 | this.conf = conf; 52 | } 53 | } 54 | 55 | public long getBatchDuration() { 56 | return batchDuration; 57 | } 58 | 59 | public void setBatchDuration(long batchDuration) { 60 | this.batchDuration = batchDuration; 61 | } 62 | 63 | SparkStreamingJob.Builder toBuilder() { 64 | SparkStreamingJob.Builder result = SparkStreamingJob.newBuilder(); 65 | if (master != null) result.master(master); 66 | if (jars != null) result.jars(jars); 67 | if (!conf.isEmpty()) result.conf(conf); 68 | if (batchDuration != null) result.batchDuration(batchDuration); 69 | return result; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /sparkstreaming-job/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Set everything to be logged to the console 19 | log4j.rootCategory=WARN, console 20 | log4j.appender.console=org.apache.log4j.ConsoleAppender 21 | log4j.appender.console.target=System.err 22 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 23 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 24 | 25 | # Settings to quiet third party logs that are too verbose 26 | log4j.logger.org.spark-project.jetty=WARN 27 | log4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR 28 | log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO 29 | log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO 30 | 31 | # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support 32 | log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL 33 | log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR 34 | -------------------------------------------------------------------------------- /sparkstreaming-job/src/test/java/zipkin/sparkstreaming/ZipkinSparkStreamingJobAutoConfigurationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming; 15 | 16 | import java.io.IOException; 17 | import java.util.ArrayDeque; 18 | import java.util.Collections; 19 | import java.util.Queue; 20 | import org.apache.spark.api.java.JavaRDD; 21 | import org.junit.After; 22 | import org.junit.Rule; 23 | import org.junit.Test; 24 | import org.junit.rules.ExpectedException; 25 | import org.springframework.boot.autoconfigure.context.PropertyPlaceholderAutoConfiguration; 26 | import org.springframework.context.annotation.AnnotationConfigApplicationContext; 27 | import org.springframework.context.annotation.Bean; 28 | import org.springframework.context.annotation.Configuration; 29 | import zipkin.Codec; 30 | import zipkin.TestObjects; 31 | import zipkin.sparkstreaming.job.ZipkinSparkStreamingConfiguration; 32 | 33 | import static org.assertj.core.api.Assertions.assertThat; 34 | import static org.assertj.core.data.MapEntry.entry; 35 | import static org.springframework.boot.test.util.EnvironmentTestUtils.addEnvironment; 36 | 37 | public class ZipkinSparkStreamingJobAutoConfigurationTest { 38 | @Rule 39 | public ExpectedException thrown = ExpectedException.none(); 40 | 41 | AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); 42 | SparkStreamingJob job; 43 | 44 | @After 45 | public void close() throws IOException { 46 | if (job != null) job.close(); 47 | context.close(); 48 | } 49 | 50 | @Test 51 | public void providesSparkStreaming() throws IOException { 52 | context.register(PropertyPlaceholderAutoConfiguration.class, 53 | DummyConfiguration.class, 54 | ZipkinSparkStreamingConfiguration.class); 55 | context.refresh(); 56 | 57 | job = context.getBean(SparkStreamingJob.class); 58 | assertThat(job).isNotNull(); 59 | } 60 | 61 | @Configuration 62 | static class DummyConfiguration { 63 | 64 | @Bean StreamFactory streamFactory() { 65 | return jsc -> { 66 | Queue> rddQueue = new ArrayDeque<>(); 67 | byte[] oneTrace = Codec.JSON.writeSpans(TestObjects.TRACE); 68 | rddQueue.add(jsc.sparkContext().parallelize(Collections.singletonList(oneTrace))); 69 | return jsc.queueStream(rddQueue); 70 | }; 71 | } 72 | 73 | @Bean Consumer consumer() { 74 | return trace -> { 75 | System.err.println(trace); 76 | }; 77 | } 78 | } 79 | 80 | @Configuration 81 | static class AdjusterConfiguration { 82 | 83 | static Adjuster one = new Adjuster() { 84 | }; 85 | static Adjuster two = new Adjuster() { 86 | }; 87 | 88 | @Bean Adjuster one() { 89 | return one; 90 | } 91 | 92 | @Bean Adjuster two() { 93 | return two; 94 | } 95 | } 96 | 97 | @Test 98 | public void providesAdjusters() throws IOException { 99 | context.register(PropertyPlaceholderAutoConfiguration.class, 100 | DummyConfiguration.class, 101 | AdjusterConfiguration.class, 102 | ZipkinSparkStreamingConfiguration.class); 103 | context.refresh(); 104 | 105 | job = context.getBean(SparkStreamingJob.class); 106 | assertThat(job).isNotNull(); 107 | assertThat(job.adjusters()) 108 | .contains(AdjusterConfiguration.one, AdjusterConfiguration.two); 109 | } 110 | 111 | @Test 112 | public void defaultConf() { 113 | context.register(PropertyPlaceholderAutoConfiguration.class, 114 | DummyConfiguration.class, 115 | AdjusterConfiguration.class, 116 | ZipkinSparkStreamingConfiguration.class); 117 | context.refresh(); 118 | 119 | job = context.getBean(SparkStreamingJob.class); 120 | assertThat(job.conf()).containsExactly( 121 | entry("spark.ui.enabled", "false") 122 | ); 123 | } 124 | 125 | @Test 126 | public void canOverrideConf() { 127 | addEnvironment(context, 128 | "zipkin.sparkstreaming.conf.spark.ui.enabled:" + true); 129 | context.register(PropertyPlaceholderAutoConfiguration.class, 130 | DummyConfiguration.class, 131 | AdjusterConfiguration.class, 132 | ZipkinSparkStreamingConfiguration.class); 133 | context.refresh(); 134 | 135 | job = context.getBean(SparkStreamingJob.class); 136 | assertThat(job.conf()).containsEntry( 137 | "spark.ui.enabled", "true" 138 | ); 139 | } 140 | 141 | @Test 142 | public void canOverrideLogLevel() { 143 | addEnvironment(context, "zipkin.log-level:debug"); 144 | context.register(PropertyPlaceholderAutoConfiguration.class, 145 | DummyConfiguration.class, 146 | AdjusterConfiguration.class, 147 | ZipkinSparkStreamingConfiguration.class); 148 | context.refresh(); 149 | 150 | job = context.getBean(SparkStreamingJob.class); 151 | assertThat(job.zipkinLogLevel()).isEqualTo("debug"); 152 | } 153 | 154 | /** Default is empty, which implies we lookup the current classpath. */ 155 | @Test 156 | public void defaultJars() { 157 | context.register(PropertyPlaceholderAutoConfiguration.class, 158 | DummyConfiguration.class, 159 | AdjusterConfiguration.class, 160 | ZipkinSparkStreamingConfiguration.class); 161 | context.refresh(); 162 | 163 | job = context.getBean(SparkStreamingJob.class); 164 | assertThat(job.jars()).isEmpty(); 165 | } 166 | 167 | @Test 168 | public void canOverrideJars() { 169 | addEnvironment(context, 170 | "zipkin.sparkstreaming.jars:foo.jar,bar.jar"); 171 | context.register(PropertyPlaceholderAutoConfiguration.class, 172 | DummyConfiguration.class, 173 | AdjusterConfiguration.class, 174 | ZipkinSparkStreamingConfiguration.class); 175 | context.refresh(); 176 | 177 | job = context.getBean(SparkStreamingJob.class); 178 | assertThat(job.jars()).containsExactly( 179 | "foo.jar", "bar.jar" 180 | ); 181 | } 182 | 183 | /** Default is empty, which implies we lookup the current classpath. */ 184 | @Test 185 | public void defaultBatchDuration() { 186 | context.register(PropertyPlaceholderAutoConfiguration.class, 187 | DummyConfiguration.class, 188 | AdjusterConfiguration.class, 189 | ZipkinSparkStreamingConfiguration.class); 190 | context.refresh(); 191 | 192 | job = context.getBean(SparkStreamingJob.class); 193 | assertThat(job.batchDuration()).isEqualTo(10_000); 194 | } 195 | 196 | @Test 197 | public void canOverrideBatchDuration() { 198 | addEnvironment(context, 199 | "zipkin.sparkstreaming.batch-duration:1000"); 200 | context.register(PropertyPlaceholderAutoConfiguration.class, 201 | DummyConfiguration.class, 202 | AdjusterConfiguration.class, 203 | ZipkinSparkStreamingConfiguration.class); 204 | context.refresh(); 205 | 206 | job = context.getBean(SparkStreamingJob.class); 207 | assertThat(job.batchDuration()).isEqualTo(1_000); 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /sparkstreaming/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | io.zipkin.sparkstreaming 20 | 0.3.10-SNAPSHOT 21 | zipkin-sparkstreaming-parent 22 | 23 | 4.0.0 24 | 25 | zipkin-sparkstreaming 26 | Zipkin Spark Streaming 27 | 28 | 29 | ${project.basedir}/.. 30 | 31 | 32 | 33 | 34 | com.google.auto.value 35 | auto-value 36 | provided 37 | 38 | 39 | 40 | org.apache.spark 41 | spark-core_${scala.binary.version} 42 | 43 | 44 | 45 | org.apache.spark 46 | spark-streaming_${scala.binary.version} 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /sparkstreaming/src/main/java/zipkin/sparkstreaming/AdjustAndConsumeSpansSharingTraceId.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming; 15 | 16 | import com.google.auto.value.AutoValue; 17 | import java.io.Serializable; 18 | import java.util.Iterator; 19 | import java.util.List; 20 | import org.apache.spark.api.java.function.VoidFunction; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | import zipkin.Span; 24 | 25 | @AutoValue 26 | abstract class AdjustAndConsumeSpansSharingTraceId implements Serializable, 27 | VoidFunction>> { 28 | private static final long serialVersionUID = 0L; 29 | private static final Logger log = LoggerFactory.getLogger(ReadSpans.class); 30 | 31 | Logger log() { // Override for testing. Instance variables won't work as Logger isn't serializable 32 | return log; 33 | } 34 | 35 | abstract Runnable logInitializer(); 36 | 37 | abstract List adjusters(); 38 | 39 | abstract Consumer consumer(); 40 | 41 | @Override public void call(Iterator> spansSharingTraceIds) { 42 | logInitializer().run(); 43 | while (spansSharingTraceIds.hasNext()) { 44 | Iterable spansSharingTraceId = spansSharingTraceIds.next(); 45 | for (Adjuster adjuster : adjusters()) { 46 | try { 47 | spansSharingTraceId = adjuster.adjust(spansSharingTraceId); 48 | } catch (RuntimeException e) { 49 | log().warn("unable to adjust spans: " + spansSharingTraceId + " with " + adjuster, e); 50 | } 51 | } 52 | consumer().accept(spansSharingTraceId); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /sparkstreaming/src/main/java/zipkin/sparkstreaming/Adjuster.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming; 15 | 16 | import java.io.Serializable; 17 | import java.util.Iterator; 18 | import java.util.NoSuchElementException; 19 | import org.apache.spark.api.java.JavaRDDLike; 20 | import zipkin.BinaryAnnotation; 21 | import zipkin.Endpoint; 22 | import zipkin.Span; 23 | 24 | /** 25 | * A {@linkplain Adjuster} is applied to {@link JavaRDDLike#map} of spans that share the same 26 | * trace ID. Note the input is not guaranteed to be a complete trace. 27 | * 28 | *

This can be used for tasks including pruning data, changing span ids, backfilling service 29 | * names. When dropping spans regardless of their trace, it is better to use a filter. 30 | * 31 | *

Implementations must be serializable, which implies you need to feed them static configuration 32 | */ 33 | // abstract class so we can add methods later w/o breaking compat 34 | public abstract class Adjuster implements Serializable { 35 | 36 | /** 37 | * The default implementation allows you to conditionally adjust spans, for example those with a 38 | * {@link BinaryAnnotation#key tag} or a common {@link Endpoint#serviceName service}. 39 | * 40 | *

This works like the following: 41 | * 42 | * 

43 |    *   
    
44 |    *     
  • If {@link #shouldAdjust(Span)}, {@link #adjust(Span)} is applied
    • 
45 |    *     
  • Otherwise, the span is returned as-is
    • 
46 |    *   

47 |    *
48 | */ 49 | public Iterable adjust(final Iterable trace) { 50 | if (trace == null) throw new NullPointerException("trace was null"); 51 | return () -> new AdjustingIterator(this, trace.iterator()); 52 | } 53 | 54 | /** By default, this doesn't adjust any spans. */ 55 | protected boolean shouldAdjust(Span span) { 56 | return false; 57 | } 58 | 59 | /** By default, this returns the same span. */ 60 | protected Span adjust(Span span) { 61 | return span; 62 | } 63 | 64 | static final class AdjustingIterator implements Iterator { 65 | final Adjuster adjuster; 66 | final Iterator delegate; 67 | 68 | AdjustingIterator(Adjuster adjuster, Iterator delegate) { 69 | this.adjuster = adjuster; 70 | this.delegate = delegate; 71 | } 72 | 73 | @Override 74 | public final boolean hasNext() { 75 | return delegate.hasNext(); 76 | } 77 | 78 | @Override 79 | public final Span next() { 80 | if (!hasNext()) throw new NoSuchElementException(); 81 | Span next = delegate.next(); 82 | if (!adjuster.shouldAdjust(next)) return next; 83 | return adjuster.adjust(next); 84 | } 85 | 86 | @Override 87 | public final void remove() { 88 | throw new UnsupportedOperationException(); 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /sparkstreaming/src/main/java/zipkin/sparkstreaming/Consumer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming; 15 | 16 | import java.io.Serializable; 17 | import zipkin.Span; 18 | 19 | /** Implementations must be serializable, which implies you need to feed them static configuration */ 20 | public interface Consumer extends Serializable { 21 | void accept(Iterable trace); 22 | } 23 | -------------------------------------------------------------------------------- /sparkstreaming/src/main/java/zipkin/sparkstreaming/LogInitializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming; 15 | 16 | import com.google.auto.value.AutoValue; 17 | import java.io.Serializable; 18 | import java.util.Enumeration; 19 | import org.apache.log4j.Appender; 20 | import org.apache.log4j.Level; 21 | import org.apache.log4j.LogManager; 22 | import org.apache.log4j.Logger; 23 | import zipkin.internal.DependencyLinker; 24 | 25 | /** 26 | * The explicitly initializes log configuration for zipkin categories to a predefined level. This is 27 | * intended to be called directly before a transformation occurs, such as {@link DependencyLinker}. 28 | * 29 | *

Motivation 30 | * 31 | *

If you don't use this, you might find that JUL statements from zipkin don't end up in console 32 | * of executors (usually stderr) even if it is in console output when in local mode. 33 | * 34 | *

Spark workers do not have the same classpath as local mode. For example, they have a different 35 | * log4j configuration (log4j-defaults.properties), and a classpath that doesn't load a JUL to log4J 36 | * bridge. This has two impacts: first, zipkin categories aren't defined, so would default to root. 37 | * Root logs at info level while zipkin libraries almost never log at info. Even if they did, unless 38 | * a bridge is installed, the log level defined in log4j wouldn't propagate to JUL. This means only 39 | * zipkin libraries who use log4j will end up in console output. 40 | * 41 | *

Manual fix 42 | * 43 | *

Knowing spark uses log4j 1.2, you could remedy this by adding jul-to-slf4j and slf4j-log4j12 44 | * to the boot classpath of the executors (ex spark.executor.extraLibraryPath) and a custom log4j 45 | * configuration file (ex via spark.executor.extraJavaOptions). Both of these have deployment impact 46 | * including potentially conflicting with existing classpaths. 47 | * 48 | *

How this works 49 | * 50 | *

Instead of creating and distributing static log configuration, this passes a function to setup 51 | * logging. Since log setup can be lost as a side-effect of deserialization, this takes care to 52 | * idempotently apply both setup of log4j and also synchronizing JUL to the same level. The result 53 | * is reliable setup with no custom bootstrap needed. The tradeoff is the explicitness of the task. 54 | */ 55 | // This is implemented as a runnable to avoid creating and publishing a new dependency shared across 56 | // all storage implementations. It would be nice to use some sort of lifecycle hook that would run 57 | // after an task deserializes on an executor. Until such is available, this is the least touch way. 58 | @AutoValue 59 | abstract class LogInitializer implements Serializable, Runnable { 60 | private static final long serialVersionUID = 0L; 61 | 62 | /** 63 | * Call this prior to any phase to ensure zipkin logging is setup 64 | */ 65 | static Runnable create(String zipkinLogLevel) { 66 | Level log4Jlevel = Level.toLevel(zipkinLogLevel); 67 | java.util.logging.Level julLevel = toJul(log4Jlevel); 68 | return new AutoValue_LogInitializer(log4Jlevel, julLevel); 69 | } 70 | 71 | abstract Level log4Jlevel(); 72 | 73 | abstract java.util.logging.Level julLevel(); 74 | 75 | @Override public void run() { 76 | Logger zipkinLogger = LogManager.getLogger("zipkin"); 77 | if (!log4Jlevel().equals(zipkinLogger.getLevel())) { 78 | zipkinLogger.setLevel(log4Jlevel()); 79 | if (zipkinLogger.getAdditivity()) { 80 | addLogAppendersFromRoot(zipkinLogger); 81 | } 82 | } 83 | java.util.logging.Logger.getLogger("zipkin").setLevel(julLevel()); 84 | } 85 | 86 | private static java.util.logging.Level toJul(Level log4Jlevel) { 87 | if (log4Jlevel.equals(Level.ALL)) return java.util.logging.Level.ALL; 88 | if (log4Jlevel.equals(Level.DEBUG)) return java.util.logging.Level.FINE; 89 | if (log4Jlevel.equals(Level.ERROR)) return java.util.logging.Level.SEVERE; 90 | if (log4Jlevel.equals(Level.FATAL)) return java.util.logging.Level.SEVERE; 91 | if (log4Jlevel.equals(Level.INFO)) return java.util.logging.Level.INFO; 92 | if (log4Jlevel.equals(Level.OFF)) return java.util.logging.Level.OFF; 93 | if (log4Jlevel.equals(Level.TRACE)) return java.util.logging.Level.FINEST; 94 | if (log4Jlevel.equals(Level.WARN)) return java.util.logging.Level.WARNING; 95 | throw new IllegalStateException("Unknown log level: " + log4Jlevel); 96 | } 97 | 98 | static void addLogAppendersFromRoot(Logger zipkinLogger) { 99 | zipkinLogger.setAdditivity(false); 100 | for (Enumeration e = Logger.getRootLogger().getAllAppenders(); e.hasMoreElements(); ) { 101 | zipkinLogger.addAppender((Appender) e.nextElement()); 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /sparkstreaming/src/main/java/zipkin/sparkstreaming/ReadSpans.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming; 15 | 16 | import com.google.auto.value.AutoValue; 17 | import java.io.Serializable; 18 | import java.util.Collections; 19 | import org.apache.spark.api.java.function.FlatMapFunction; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | import zipkin.Codec; 23 | import zipkin.Span; 24 | 25 | @AutoValue 26 | abstract class ReadSpans implements Serializable, FlatMapFunction { 27 | private static final long serialVersionUID = 0L; 28 | private static final Logger log = LoggerFactory.getLogger(ReadSpans.class); 29 | 30 | abstract Runnable logInitializer(); 31 | 32 | // In TBinaryProtocol encoding, the first byte is the TType, in a range 0-16 33 | // .. If the first byte isn't in that range, it isn't a thrift. 34 | // 35 | // When byte(0) == '[' (91), assume it is a list of json-encoded spans 36 | // 37 | // When byte(0) <= 16, assume it is a TBinaryProtocol-encoded thrift 38 | // .. When serializing a Span (Struct), the first byte will be the type of a field 39 | // .. When serializing a List[ThriftSpan], the first byte is the member type, TType.STRUCT(12) 40 | // .. As ThriftSpan has no STRUCT fields: so, if the first byte is TType.STRUCT(12), it is a list. 41 | @Override public Iterable call(byte[] bytes) throws Exception { 42 | logInitializer().run(); 43 | if (bytes.length == 0) return Collections.emptyList(); 44 | try { 45 | if (bytes[0] == '[') { 46 | return Codec.JSON.readSpans(bytes); 47 | } else { 48 | if (bytes[0] == 12 /* TType.STRUCT */) { 49 | return Codec.THRIFT.readSpans(bytes); 50 | } else { // historical kafka encoding of single thrift span per message 51 | return Collections.singletonList(Codec.THRIFT.readSpan(bytes)); 52 | } 53 | } 54 | } catch (RuntimeException e) { 55 | log.warn("unable to decode spans", e); 56 | return Collections.emptyList(); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /sparkstreaming/src/main/java/zipkin/sparkstreaming/SparkStreamingJob.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming; 15 | 16 | import com.google.auto.value.AutoValue; 17 | import com.google.auto.value.extension.memoized.Memoized; 18 | import java.io.Closeable; 19 | import java.io.IOException; 20 | import java.util.Collections; 21 | import java.util.LinkedHashMap; 22 | import java.util.List; 23 | import java.util.Map; 24 | import java.util.concurrent.atomic.AtomicBoolean; 25 | import org.apache.spark.SparkConf; 26 | import org.apache.spark.streaming.Duration; 27 | import org.apache.spark.streaming.api.java.JavaDStream; 28 | import org.apache.spark.streaming.api.java.JavaPairDStream; 29 | import org.apache.spark.streaming.api.java.JavaStreamingContext; 30 | import scala.Tuple2; 31 | import zipkin.Span; 32 | import zipkin.internal.Util; 33 | import zipkin.storage.StorageComponent; 34 | 35 | @AutoValue 36 | public abstract class SparkStreamingJob implements Closeable { 37 | 38 | public static Builder newBuilder() { 39 | Map conf = new LinkedHashMap<>(); 40 | conf.put("spark.ui.enabled", "false"); 41 | return new AutoValue_SparkStreamingJob.Builder() 42 | .master("local[*]") 43 | .jars(Collections.emptyList()) 44 | .conf(conf) 45 | .adjusters(Collections.emptyList()) 46 | .batchDuration(10_000) 47 | .zipkinLogLevel("INFO"); 48 | } 49 | 50 | @AutoValue.Builder 51 | public interface Builder { 52 | /** 53 | * The spark master used for this job. Defaults to "local[*]" 54 | * 55 | *

local[*] master lets us run & test the job locally without setting a Spark cluster 56 | */ 57 | Builder master(String master); 58 | 59 | /** When set, this indicates which jars to distribute to the cluster. */ 60 | Builder jars(List jars); 61 | 62 | /** Overrides the properties used to create a {@link SparkConf}. */ 63 | Builder conf(Map conf); 64 | 65 | /** The time interval at which streaming data will be divided into batches. Defaults to 10s. */ 66 | Builder batchDuration(long batchDurationMillis); 67 | 68 | /** Produces a stream of serialized span messages (thrift or json lists) */ 69 | Builder streamFactory(StreamFactory streamFactory); 70 | 71 | /** Conditionally adjusts spans grouped by trace ID. For example, pruning data */ 72 | Builder adjusters(List adjusters); 73 | 74 | /** Accepts spans grouped by trace ID. For example, writing to a {@link StorageComponent} */ 75 | Builder consumer(Consumer consumer); 76 | 77 | /** Log4J level used for the "zipkin" category. Important when running in a cluster. */ 78 | Builder zipkinLogLevel(String zipkinLogLevel); 79 | 80 | SparkStreamingJob build(); 81 | } 82 | 83 | abstract String master(); 84 | 85 | abstract List jars(); 86 | 87 | abstract Map conf(); 88 | 89 | abstract long batchDuration(); 90 | 91 | abstract StreamFactory streamFactory(); 92 | 93 | abstract List adjusters(); 94 | 95 | abstract Consumer consumer(); 96 | 97 | abstract String zipkinLogLevel(); 98 | 99 | final AtomicBoolean started = new AtomicBoolean(false); 100 | 101 | @Memoized 102 | JavaStreamingContext jsc() { 103 | SparkConf conf = new SparkConf(true) 104 | .setMaster(master()) 105 | .setAppName(getClass().getName()); 106 | if (!jars().isEmpty()) conf.setJars(jars().toArray(new String[0])); 107 | for (Map.Entry entry : conf().entrySet()) { 108 | conf.set(entry.getKey(), entry.getValue()); 109 | } 110 | return new JavaStreamingContext(conf, new Duration(batchDuration())); 111 | } 112 | 113 | /** Starts the streaming job. Use {@link #close()} to stop it */ 114 | public SparkStreamingJob start() { 115 | if (!started.compareAndSet(false, true)) return this; 116 | 117 | Runnable logInitializer = LogInitializer.create(zipkinLogLevel()); 118 | logInitializer.run(); // Ensures local log commands emit 119 | streamSpansToStorage( 120 | streamFactory().create(jsc()), 121 | new AutoValue_ReadSpans(logInitializer), 122 | new AutoValue_AdjustAndConsumeSpansSharingTraceId(logInitializer, adjusters(), consumer()) 123 | ); 124 | 125 | jsc().start(); 126 | return this; 127 | } 128 | 129 | /** Use this to block on {@link #close()} */ 130 | public void awaitTermination() { 131 | if (started.get()) jsc().awaitTermination(); 132 | } 133 | 134 | // NOTE: this is intentionally static to remind us that all state passed in must be serializable 135 | // Otherwise, tasks cannot be distributed across the cluster. 136 | static void streamSpansToStorage( 137 | JavaDStream stream, 138 | ReadSpans readSpans, 139 | AdjustAndConsumeSpansSharingTraceId adjustAndConsumeSpansSharingTraceId 140 | ) { 141 | JavaDStream spans = stream.flatMap(readSpans); 142 | 143 | // TODO: plug in some filter to drop spans regardless of trace ID 144 | // spans = spans.filter(spanFilter); 145 | 146 | JavaPairDStream> tracesById = spans 147 | .mapToPair(s -> new Tuple2<>(Util.toLowerHex(s.traceIdHigh, s.traceId), s)) 148 | .groupByKey(); 149 | 150 | tracesById.foreachRDD(rdd -> { 151 | rdd.values().foreachPartition(adjustAndConsumeSpansSharingTraceId); 152 | }); 153 | } 154 | 155 | @Override public void close() throws IOException { 156 | jsc().close(); 157 | // not sure how to get spark to close things 158 | if (consumer() instanceof Closeable) { 159 | ((Closeable) consumer()).close(); 160 | } 161 | } 162 | 163 | SparkStreamingJob() { 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /sparkstreaming/src/main/java/zipkin/sparkstreaming/StreamFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming; 15 | 16 | import java.io.Serializable; 17 | import org.apache.spark.streaming.api.java.JavaDStream; 18 | import org.apache.spark.streaming.api.java.JavaStreamingContext; 19 | 20 | /** Returns a stream that includes encoded json or thrift lists */ 21 | public interface StreamFactory extends Serializable { 22 | JavaDStream create(JavaStreamingContext jsc); 23 | } 24 | -------------------------------------------------------------------------------- /sparkstreaming/src/test/java/zipkin/sparkstreaming/AdjusterTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming; 15 | 16 | import com.google.common.collect.Lists; 17 | import org.junit.Rule; 18 | import org.junit.Test; 19 | import org.junit.rules.ExpectedException; 20 | import zipkin.Annotation; 21 | import zipkin.Constants; 22 | import zipkin.Span; 23 | import zipkin.TestObjects; 24 | import zipkin.internal.ApplyTimestampAndDuration; 25 | import zipkin.internal.MergeById; 26 | 27 | import static java.util.Arrays.asList; 28 | import static org.assertj.core.api.Assertions.assertThat; 29 | import static zipkin.Constants.CLIENT_RECV; 30 | import static zipkin.Constants.CLIENT_SEND; 31 | import static zipkin.Constants.SERVER_RECV; 32 | import static zipkin.Constants.SERVER_SEND; 33 | import static zipkin.TestObjects.APP_ENDPOINT; 34 | import static zipkin.TestObjects.TODAY; 35 | import static zipkin.TestObjects.WEB_ENDPOINT; 36 | 37 | public class AdjusterTest { 38 | 39 | @Rule 40 | public final ExpectedException thrown = ExpectedException.none(); 41 | 42 | @Test 43 | public void unmodifiable() { 44 | thrown.expect(UnsupportedOperationException.class); 45 | 46 | Adjuster adjuster = new Adjuster() { 47 | }; 48 | Iterable adjusted = adjuster.adjust(TestObjects.TRACE); 49 | adjusted.iterator().remove(); 50 | } 51 | 52 | @Test 53 | public void defaultsToPassAllWithoutAdjusting() { 54 | Adjuster adjuster = new Adjuster() { 55 | }; 56 | 57 | assertThat(adjuster.adjust(TestObjects.TRACE)) 58 | .containsExactlyElementsOf(TestObjects.TRACE); 59 | } 60 | 61 | @Test 62 | public void canOverrideEverything() { 63 | Span clientSide = Span.builder().traceId(10L).id(10L).name("") 64 | .addAnnotation(Annotation.create((TODAY + 50) * 1000, CLIENT_SEND, WEB_ENDPOINT)) 65 | .addAnnotation(Annotation.create((TODAY + 300) * 1000, CLIENT_RECV, WEB_ENDPOINT)) 66 | .build(); 67 | Span serverSide = Span.builder().traceId(10L).id(10L).name("get") 68 | .addAnnotation(Annotation.create((TODAY + 100) * 1000, SERVER_RECV, APP_ENDPOINT)) 69 | .addAnnotation(Annotation.create((TODAY + 250) * 1000, SERVER_SEND, APP_ENDPOINT)) 70 | .build(); 71 | 72 | Adjuster adjuster = new Adjuster() { 73 | @Override public Iterable adjust(Iterable trace) { 74 | // TODO remove guava conversion once https://github.com/openzipkin/zipkin/pull/1519 75 | return MergeById.apply(Lists.newArrayList(trace)); 76 | } 77 | }; 78 | 79 | assertThat(adjuster.adjust(asList(clientSide, serverSide))) 80 | .containsExactlyElementsOf(MergeById.apply(asList(clientSide, serverSide))); 81 | } 82 | 83 | @Test 84 | public void adjust() { 85 | Adjuster adjuster = new Adjuster() { 86 | @Override protected Span adjust(Span span) { 87 | return ApplyTimestampAndDuration.apply(span); 88 | } 89 | }; 90 | 91 | assertThat(adjuster.adjust(TestObjects.TRACE)) 92 | .extracting(s -> s.timestamp, s -> s.duration) 93 | .doesNotContainNull(); 94 | } 95 | 96 | @Test 97 | public void shouldAdjust() { 98 | Adjuster adjuster = new Adjuster() { 99 | @Override protected boolean shouldAdjust(Span span) { 100 | for (Annotation a : span.annotations) { 101 | if (a.value.equals(Constants.SERVER_RECV) && span.parentId == null) return true; 102 | } 103 | return false; 104 | } 105 | 106 | @Override protected Span adjust(Span span) { 107 | return ApplyTimestampAndDuration.apply(span); 108 | } 109 | }; 110 | 111 | assertThat(adjuster.adjust(TestObjects.TRACE)).containsExactly( 112 | ApplyTimestampAndDuration.apply(TestObjects.TRACE.get(0)), 113 | TestObjects.TRACE.get(1), 114 | TestObjects.TRACE.get(2) 115 | ); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/etc/header.txt: -------------------------------------------------------------------------------- 1 | Copyright ${license.git.copyrightYears} The OpenZipkin Authors 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | in compliance with the License. You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software distributed under the License 9 | is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | or implied. See the License for the specific language governing permissions and limitations under 11 | the License. 12 | -------------------------------------------------------------------------------- /stream/kafka/README.md: -------------------------------------------------------------------------------- 1 | # stream-kafka 2 | 3 | ## KafkaStreamFactory 4 | 5 | This stream ingests Spans form a Kafka topic advertised by Zookeeper, 6 | using Spark Kafka libraries. 7 | 8 | Kafka messages should contain a list of spans in json or TBinaryProtocol 9 | big-endian encoding. Details on message encode is available [here](https://github.com/openzipkin/zipkin/blob/master/zipkin-collector/kafka/README.md#encoding-spans-into-kafka-messages) 10 | 11 | ## Usage 12 | 13 | While the `KafkaStreamFactory` can be used directly through the provided 14 | builder interface, most users will likely find more value in the Spring 15 | Boot autoconfiguraton module. Additional information for using the 16 | module can be found [here](../../autoconfigure/stream-kafka). 17 | -------------------------------------------------------------------------------- /stream/kafka/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | io.zipkin.sparkstreaming 20 | zipkin-sparkstreaming-stream-parent 21 | 0.3.10-SNAPSHOT 22 | 23 | 4.0.0 24 | 25 | zipkin-sparkstreaming-stream-kafka 26 | Zipkin Spark Streaming Stream: Kafka 27 | 28 | 29 | ${project.basedir}/../.. 30 | 31 | 32 | 33 | 34 | com.google.auto.value 35 | auto-value 36 | provided 37 | 38 | 39 | org.apache.spark 40 | spark-streaming-kafka_2.10 41 | ${spark.version} 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /stream/kafka/src/main/java/zipkin/sparkstreaming/stream/kafka/BootstrapServers.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.stream.kafka; 15 | 16 | import java.util.List; 17 | import org.apache.kafka.clients.producer.ProducerConfig; 18 | 19 | /** 20 | * Initial set of kafka servers to connect to, rest of cluster will be discovered (comma 21 | * separated). Values are in host:port syntax. No default 22 | * 23 | * @see ProducerConfig#BOOTSTRAP_SERVERS_CONFIG 24 | */ 25 | public interface BootstrapServers { 26 | List get(); 27 | } 28 | -------------------------------------------------------------------------------- /stream/kafka/src/main/java/zipkin/sparkstreaming/stream/kafka/KafkaStreamFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.stream.kafka; 15 | 16 | import com.google.auto.value.AutoValue; 17 | import com.google.auto.value.extension.memoized.Memoized; 18 | import java.util.Collections; 19 | import java.util.LinkedHashMap; 20 | import java.util.List; 21 | import java.util.Map; 22 | import kafka.serializer.DefaultDecoder; 23 | import org.apache.commons.lang.StringUtils; 24 | import org.apache.kafka.clients.producer.ProducerConfig; 25 | import org.apache.spark.streaming.api.java.JavaDStream; 26 | import org.apache.spark.streaming.api.java.JavaStreamingContext; 27 | import org.apache.spark.streaming.kafka.KafkaUtils; 28 | import zipkin.sparkstreaming.StreamFactory; 29 | 30 | import static zipkin.internal.Util.checkNotNull; 31 | 32 | /** 33 | * Ingests Spans form a Kafka topic advertised by Zookeeper 34 | */ 35 | @AutoValue 36 | public abstract class KafkaStreamFactory implements StreamFactory { 37 | 38 | public static Builder newBuilder() { 39 | return new AutoValue_KafkaStreamFactory.Builder() 40 | .topic("zipkin") 41 | .groupId("zipkin"); 42 | } 43 | 44 | @AutoValue.Builder 45 | public static abstract class Builder { 46 | 47 | /** Kafka topic encoded lists of spans are be consumed from. Defaults to "zipkin" */ 48 | public abstract Builder topic(String topic); 49 | 50 | /** Consumer group this process is consuming on behalf of. Defaults to "zipkin" */ 51 | public abstract Builder groupId(String groupId); 52 | 53 | /** 54 | * Initial set of kafka servers to connect to; others may be discovered. Values are in host:port 55 | * syntax. No default. 56 | * 57 | * @see ProducerConfig#BOOTSTRAP_SERVERS_CONFIG 58 | */ 59 | public final Builder bootstrapServers(final List bootstrapServers) { 60 | checkNotNull(bootstrapServers, "bootstrapServers"); 61 | return bootstrapServers(new BootstrapServers() { 62 | @Override public List get() { 63 | return bootstrapServers; 64 | } 65 | 66 | @Override public String toString() { 67 | return bootstrapServers.toString(); 68 | } 69 | }); 70 | } 71 | 72 | /** 73 | * Like {@link #bootstrapServers(List)}, except the value is deferred. 74 | * 75 | *

This was added to support dynamic endpoint resolution for Amazon Elasticsearch. This value 76 | * is only read once. 77 | */ 78 | public abstract Builder bootstrapServers(BootstrapServers bootstrapServers); 79 | 80 | public abstract KafkaStreamFactory build(); 81 | 82 | Builder() { 83 | } 84 | } 85 | 86 | abstract String topic(); 87 | 88 | abstract String groupId(); 89 | 90 | abstract BootstrapServers bootstrapServers(); 91 | 92 | @Override public JavaDStream create(JavaStreamingContext jsc) { 93 | return KafkaUtils.createDirectStream( 94 | jsc, 95 | byte[].class, 96 | byte[].class, 97 | DefaultDecoder.class, 98 | DefaultDecoder.class, 99 | kafkaParams(), 100 | Collections.singleton(topic())) 101 | .map(m -> m._2); // get value 102 | } 103 | 104 | @Memoized 105 | Map kafkaParams() { 106 | Map kafkaParams = new LinkedHashMap<>(); 107 | kafkaParams.put("metadata.broker.list", StringUtils.join(bootstrapServers().get(), ",")); 108 | kafkaParams.put("group.id", groupId()); 109 | return Collections.unmodifiableMap(kafkaParams); 110 | } 111 | 112 | KafkaStreamFactory() { 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /stream/kafka/src/main/java/zipkin/sparkstreaming/stream/kafka/ZookeeperBootstrapServers.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.stream.kafka; 15 | 16 | import com.fasterxml.jackson.databind.ObjectMapper; 17 | import com.google.auto.value.AutoValue; 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | import java.util.Map; 21 | import org.apache.zookeeper.WatchedEvent; 22 | import org.apache.zookeeper.Watcher; 23 | import org.apache.zookeeper.ZooKeeper; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | /** Retrieves the Kafka Bootstrap Servers from ZooKeeper. */ 28 | @AutoValue 29 | public abstract class ZookeeperBootstrapServers implements BootstrapServers { 30 | private static final Logger log = LoggerFactory.getLogger(ZookeeperBootstrapServers.class); 31 | 32 | public static Builder newBuilder() { 33 | return new AutoValue_ZookeeperBootstrapServers.Builder() 34 | .sessionTimeout(10000); 35 | } 36 | 37 | @AutoValue.Builder 38 | public interface Builder { 39 | 40 | /** 41 | * Zookeeper host string. host:port pairs corresponding to a Zookeeper server with an optional 42 | * chroot suffix. No default 43 | * 44 | * @see ZooKeeper#ZooKeeper(String, int, Watcher) 45 | */ 46 | Builder connect(String connect); 47 | 48 | /** 49 | * Zookeeper session timeout in milliseconds. Defaults to 10000 50 | */ 51 | Builder sessionTimeout(int sessionTimeout); 52 | 53 | ZookeeperBootstrapServers build(); 54 | } 55 | 56 | abstract String connect(); 57 | 58 | abstract int sessionTimeout(); 59 | 60 | @Override public List get() { 61 | ZooKeeper zkClient = null; 62 | try { 63 | zkClient = new ZooKeeper(connect(), sessionTimeout(), new NoOpWatcher()); 64 | List ids = zkClient.getChildren("/brokers/ids", false); 65 | ObjectMapper objectMapper = new ObjectMapper(); 66 | 67 | List brokerConnections = new ArrayList<>(); 68 | for (String id : ids) { 69 | brokerConnections.add(getBrokerInfo(zkClient, objectMapper, id)); 70 | } 71 | return brokerConnections; 72 | } catch (Exception e) { 73 | throw new IllegalStateException("Error loading brokers from zookeeper", e); 74 | } finally { 75 | if (zkClient != null) { 76 | try { 77 | zkClient.close(); 78 | } catch (InterruptedException e) { 79 | Thread.currentThread().interrupt(); 80 | } 81 | } 82 | } 83 | } 84 | 85 | /** 86 | * Builds string to create KafkaParams for Spark job 87 | * 88 | * @param zkClient ZooKeeper client with predefined configurations 89 | * @param om ObjectMapper used to read zkClient's children (brokers) 90 | * @param id broker id 91 | * @return "host:port" string 92 | */ 93 | private String getBrokerInfo(ZooKeeper zkClient, ObjectMapper om, String id) { 94 | try { 95 | Map map = om.readValue(zkClient.getData("/brokers/ids/" + id, false, null), Map.class); 96 | return map.get("host") + ":" + map.get("port"); 97 | } catch (Exception e) { 98 | throw new IllegalStateException("Error reading zkClient's broker id's", e); 99 | } 100 | } 101 | 102 | static final class NoOpWatcher implements Watcher { 103 | @Override 104 | public void process(WatchedEvent event) { 105 | log.debug("{}", event); 106 | } 107 | } 108 | 109 | ZookeeperBootstrapServers() { 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /stream/kafka/src/test/java/zipkin/sparkstreaming/stream/kafka/KafkaStreamFactoryTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.stream.kafka; 15 | 16 | import org.junit.Rule; 17 | import org.junit.Test; 18 | import org.junit.rules.ExpectedException; 19 | 20 | import static java.util.Arrays.asList; 21 | import static org.assertj.core.api.Assertions.assertThat; 22 | 23 | // TODO: actually test the code 24 | public class KafkaStreamFactoryTest { 25 | 26 | @Rule 27 | public ExpectedException thrown = ExpectedException.none(); 28 | 29 | @Test 30 | public void buildKafkaStreamFactory() throws Exception { 31 | KafkaStreamFactory streamFactory = KafkaStreamFactory.newBuilder() 32 | .bootstrapServers(asList("127.0.0.1:9092")) 33 | .build(); 34 | assertThat(streamFactory).isNotNull(); 35 | } 36 | 37 | @Test 38 | public void buildFailOnMissingProperties() throws Exception { 39 | thrown.expect(IllegalStateException.class); 40 | thrown.expectMessage("Missing required properties: bootstrapServers"); 41 | KafkaStreamFactory streamFactory = KafkaStreamFactory.newBuilder().build(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /stream/kafka/src/test/java/zipkin/sparkstreaming/stream/kafka/ZookeeperBootstrapServersTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2017 The OpenZipkin Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the License 10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 | * or implied. See the License for the specific language governing permissions and limitations under 12 | * the License. 13 | */ 14 | package zipkin.sparkstreaming.stream.kafka; 15 | 16 | import org.junit.Rule; 17 | import org.junit.Test; 18 | import org.junit.rules.ExpectedException; 19 | 20 | import static org.assertj.core.api.Assertions.assertThat; 21 | 22 | // TODO: actually test the code 23 | public class ZookeeperBootstrapServersTest { 24 | 25 | @Rule 26 | public ExpectedException thrown = ExpectedException.none(); 27 | 28 | @Test 29 | public void buildZookeeperBootstrapServers() throws Exception { 30 | ZookeeperBootstrapServers servers = ZookeeperBootstrapServers.newBuilder() 31 | .connect("127.0.0.1:2181") 32 | .build(); 33 | assertThat(servers).isNotNull(); 34 | } 35 | 36 | @Test 37 | public void buildFailOnMissingProperties() throws Exception { 38 | thrown.expect(IllegalStateException.class); 39 | thrown.expectMessage("Missing required properties: connect"); 40 | ZookeeperBootstrapServers servers = ZookeeperBootstrapServers.newBuilder().build(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /stream/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 4.0.0 19 | 20 | 21 | io.zipkin.sparkstreaming 22 | zipkin-sparkstreaming-parent 23 | 0.3.10-SNAPSHOT 24 | 25 | 26 | zipkin-sparkstreaming-stream-parent 27 | Zipkin Spark Streaming: Stream Factories 28 | pom 29 | 30 | 31 | ${project.basedir}/.. 32 | 33 | 34 | 35 | kafka 36 | 37 | 38 | 39 | 40 | ${project.groupId} 41 | zipkin-sparkstreaming 42 | 43 | 44 | 45 | --------------------------------------------------------------------------------