├── .gitignore ├── LICENSE ├── README.md ├── Vagrantfile ├── bootstrap.sh ├── build.gradle ├── codahale └── src │ ├── main │ └── java │ │ └── ly │ │ └── stealth │ │ └── kafka │ │ └── metrics │ │ ├── KafkaGraphiteConsumer.java │ │ ├── KafkaMetricsReport.java │ │ └── KafkaReporter.java │ └── test │ ├── java │ └── ly │ │ └── stealth │ │ └── kafka │ │ └── metrics │ │ ├── MetricsReportingSteps.java │ │ └── Stories.java │ └── resources │ ├── log4j.properties │ └── stories │ └── metrics_reporting.story ├── config ├── config.rb ├── dash-config.json ├── riemann.config └── server.properties ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── graphite └── src │ └── main │ └── java │ └── ly │ └── stealth │ └── graphite │ ├── GraphiteClient.java │ └── GraphiteException.java ├── metrics-test └── src │ └── main │ └── scala │ └── ly │ └── stealth │ └── psutil │ └── PsutilKafka.scala ├── psutil-entities └── src │ └── main │ └── scala │ └── ly │ └── stealth │ └── psutil │ └── entities │ ├── Cpu_times.scala │ ├── Cpu_times_percent.scala │ ├── Disk_io_counters.scala │ ├── Disk_usage.scala │ ├── Net_connection.scala │ ├── PsutilsReport.scala │ ├── Swap_memory.scala │ ├── User.scala │ └── Virtual_memory.scala ├── psutil-statsd └── src │ └── main │ └── scala │ └── ly │ └── stealth │ └── kafka │ └── metrics │ └── statsd │ └── KafkaPsutilStatsDConsumer.scala ├── psutil └── src │ └── main │ └── python │ ├── config.json │ ├── psutil_producer.py │ └── psutil_producer_test.py ├── riemann └── src │ ├── main │ ├── resources │ │ └── log4j.properties │ └── scala │ │ └── ly │ │ └── stealth │ │ └── kafka │ │ └── riemann │ │ ├── CodaHaleMetricsConsumer.scala │ │ ├── PsutilMetricsConsumer.scala │ │ └── RiemannMetricsConsumer.scala │ └── test │ └── scala │ └── ly │ └── stealth │ └── kafka │ └── metrics │ ├── CodaHaleMetricsAndRiemannSpec.scala │ └── PsutilMetricsAndRiemannSpec.scala ├── settings.gradle ├── shutdown.sh ├── statsd └── src │ └── main │ └── scala │ └── ly │ └── stealth │ └── metrics │ └── kafka │ └── statsd │ └── StatsD.scala ├── vagrant ├── README.md ├── broker.sh ├── kafka.sh ├── provision.sh ├── psutil.sh ├── riemann.sh ├── verify.sh ├── verify_hash.sh └── zk.sh └── yammer └── src └── main └── java └── ly └── stealth └── kafka └── metrics ├── KafkaBrokerReporter.java └── TopicReporter.java /.gitignore: -------------------------------------------------------------------------------- 1 | */build/* 2 | codahale/* 3 | .gradle 4 | metrics/build 5 | metrics/logs 6 | riemann/build 7 | riemann/logs 8 | *.class 9 | *.log 10 | *.iml 11 | 12 | # sbt specific 13 | .cache/ 14 | .history/ 15 | .lib/ 16 | .vagrant/* 17 | .idea/* 18 | logs/* 19 | dist/* 20 | target/ 21 | lib_managed/ 22 | src_managed/ 23 | project/boot/ 24 | project/plugins/project/ 25 | 26 | # Scala-IDE specific 27 | .scala_dependencies 28 | .worksheet 29 | psutil/* 30 | metrics-client/* -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Kafka CodaHale Metrics Reporter + Riemann metrics consumer 2 | ============= 3 | The goals of this repo is to have an end to end working environment to provide the ability for systems (applications 4 | and infrastructure) to send their metrics/sensor data to Kafka and then to report on that data for both alerts and charts. 5 | 6 | More about this project in this blog post [http://allthingshadoop.com/2014/04/18/metrics-kafka/](http://allthingshadoop.com/2014/04/18/metrics-kafka/). 7 | 8 | Quick up and running 9 | ==================== 10 | 11 | Use Vagrant to get up and running. 12 | 13 | 1) Install Docker [http://docs.docker.com/installation/](http://docs.docker.com/installation/) 14 | 2) Install Virtual Box [https://www.virtualbox.org/](https://www.virtualbox.org/) 15 | 16 | In the main metrics-kafka folder 17 | 18 | 1) sudo ./bootstrap.sh 19 | 2) ./gradlew test 20 | 3) sudo ./shutdown.sh 21 | 22 | once this is done 23 | * Zookeeper will be running localhost:2181 24 | * Broker 1 on localhost:9092 25 | * Riemann on localhost:5555 26 | * Riemann dash on localhost:4567 27 | * All the tests in metrics/src/test/java/* and riemann/src/test/scala/* should pass 28 | 29 | You can access the brokers, zookeeper, riemann and riemann-dash by their IP from your local without having to go into vm. 30 | 31 | e.g. 32 | 33 | bin/kafka-console-producer.sh --broker-list localhost:9092 --topic 34 | 35 | bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic --from-beginning 36 | 37 | http://192.168.86.5:4567/#Riemann 38 | ./gradlew :psutil:installDependencies 39 | ./gradlew :metrics-test:run 40 | 41 | Kafka Yammer Metrics reporting 42 | ============================== 43 | 44 | In order to assemble jar for metrics yammer do the following steps: 45 | 1) ./gradlew :yammer:jar 46 | 2) put the jar from metrics-yammer/build/libs to libs dir in root kafka folder 47 | 3) add "kafka.metrics.reporters=ly.stealth.kafka.metrics.KafkaBrokerReporter" to config/server.properties in kafka root folder -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # -*- mode: ruby -*- 16 | # vi: set ft=ruby : 17 | 18 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 19 | VAGRANTFILE_API_VERSION = "2" 20 | 21 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 22 | config.vm.box = "precise64" 23 | 24 | # The url from where the 'config.vm.box' box will be fetched if it 25 | # doesn't already exist on the user's system. 26 | config.vm.box_url = "http://files.vagrantup.com/precise64.box" 27 | 28 | config.vm.define "trm" do |trm| 29 | trm.vm.provision "docker", images: ["ubuntu"] 30 | trm.vm.network :private_network, ip: "192.168.86.5" 31 | trm.vm.provider :virtualbox do |vb| 32 | vb.customize ["modifyvm", :id, "--memory", "2048"] 33 | end 34 | trm.vm.provision "shell", path: "vagrant/provision.sh" 35 | end 36 | 37 | end 38 | -------------------------------------------------------------------------------- /bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export START_KAFKA_SCRIPT=https://raw.githubusercontent.com/stealthly/docker-kafka/master/start-broker.sh 4 | curl -Ls $START_KAFKA_SCRIPT | bash /dev/stdin 1 9092 localhost 5 | 6 | export START_RIEMANN_SCRIPT=https://raw.githubusercontent.com/stealthly/docker-riemann/master/start-riemann.sh 7 | curl -Ls $START_RIEMANN_SCRIPT | bash /dev/stdin localhost 8 | 9 | sudo docker run --name psutil --link broker1:localhost -d -v $(pwd)/psutil/src/main/python:/psutil stealthly/docker-python python /psutil/psutil_producer.py --topic psutil-kafka-topic --url localhost:9092 10 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | subprojects { 2 | apply plugin:'java' 3 | apply plugin: "maven" 4 | apply plugin: "signing" 5 | 6 | version = "0.1" 7 | 8 | repositories { 9 | mavenCentral() 10 | maven { 11 | url = 'http://clojars.org/repo' 12 | } 13 | } 14 | 15 | configurations { 16 | deployerJars 17 | } 18 | 19 | uploadArchives { 20 | signing { 21 | sign configurations.archives 22 | repositories.mavenDeployer { 23 | beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } 24 | pom.groupId = 'ly.stealth' 25 | pom.artifactId = project.name 26 | pom.version = version 27 | pom.project { 28 | name project.name 29 | packaging 'jar' 30 | url 'http://github.com/stealthly/metrics-kafka' 31 | description 'Metrics produced to Kafka and consumers for monitoring them' 32 | scm { 33 | url 'http://github.com/stealthly/metrics-kafka' 34 | connection 'scm:https://github.com/stealthly/metrics-kafka' 35 | developerConnection 'scm:git:git://github.com/stealthly/metrics-kafka' 36 | } 37 | licenses { 38 | license { 39 | name 'The Apache Software License, Version 2.0' 40 | url 'http://www.apache.org/licenses/LICENSE-2.0.txt' 41 | distribution 'repo' 42 | } 43 | } 44 | developers { 45 | developer { 46 | id 'stealthly' 47 | name 'Joe Stein' 48 | email 'joe.stein@stealth.ly' 49 | } 50 | } 51 | } 52 | configuration = configurations.deployerJars 53 | repository(url: "${mavenURL}") { 54 | authentication(userName: "${mavenUsername}", password: "${mavenPassword}") 55 | } 56 | } 57 | } 58 | } 59 | 60 | dependencies { 61 | deployerJars "org.apache.maven.wagon:wagon-http:2.2" 62 | } 63 | } 64 | 65 | tasks.withType(ScalaDoc) { 66 | task srcJar(type:Jar) { 67 | classifier = 'sources' 68 | from '../LICENSE' 69 | from '../NOTICE' 70 | from sourceSets.main.scala 71 | } 72 | 73 | task javadocJar(type: Jar, dependsOn: javadoc) { 74 | classifier 'javadoc' 75 | from '../LICENSE' 76 | from javadoc.destinationDir 77 | } 78 | 79 | task docsJar(type: Jar, dependsOn: javadocJar) { } 80 | 81 | artifacts { 82 | archives srcJar 83 | archives javadocJar 84 | } 85 | } 86 | 87 | project (':codahale') { 88 | apply plugin:'scala' 89 | 90 | test { 91 | doFirst { 92 | file('target').mkdirs(); 93 | } 94 | 95 | testLogging { 96 | events "passed", "skipped", "failed" 97 | exceptionFormat = 'full' 98 | } 99 | 100 | doLast { 101 | file('target').delete(); 102 | } 103 | } 104 | 105 | dependencies { 106 | compile 'org.apache.kafka:kafka_2.10:0.8.1' 107 | compile 'org.slf4j:slf4j-log4j12:1.7.7' 108 | compile 'com.codahale.metrics:metrics-core:3.0.1' 109 | compile 'com.codahale.metrics:metrics-json:3.0.1' 110 | compile project(':graphite') 111 | 112 | testCompile 'junit:junit:4.11' 113 | testCompile 'org.jbehave.site:jbehave-site-resources:3.1.1' 114 | testCompile 'de.codecentric:jbehave-junit-runner:1.0.1' 115 | testCompile 'org.jbehave:jbehave-core:3.8' 116 | } 117 | } 118 | 119 | project (':yammer') { 120 | 121 | dependencies { 122 | compile 'org.apache.kafka:kafka_2.10:0.8.1' 123 | compile 'org.slf4j:slf4j-log4j12:1.7.7' 124 | compile 'com.codahale.metrics:metrics-core:3.0.1' 125 | compile 'com.codahale.metrics:metrics-json:3.0.1' 126 | 127 | testCompile 'junit:junit:4.11' 128 | testCompile 'org.jbehave.site:jbehave-site-resources:3.1.1' 129 | testCompile 'de.codecentric:jbehave-junit-runner:1.0.1' 130 | testCompile 'org.jbehave:jbehave-core:3.8' 131 | } 132 | } 133 | 134 | project (':riemann') { 135 | apply plugin:'scala' 136 | 137 | task specs(type: JavaExec, dependsOn: testClasses) { 138 | main = 'org.specs2.files' 139 | args = ['console'] 140 | classpath sourceSets.main.runtimeClasspath 141 | classpath sourceSets.test.runtimeClasspath 142 | classpath configurations.testRuntime 143 | classpath configurations.runtime 144 | } 145 | 146 | test.dependsOn specs 147 | test.dependsOn ":psutil:test" 148 | 149 | dependencies { 150 | compile project(':codahale') 151 | compile project(':psutil-entities') 152 | compile 'org.scala-lang:scala-library:2.10.3' 153 | compile 'net.benmur:riemann-scala-client_2.10:0.3.1' 154 | 155 | testCompile 'org.specs2:specs2_2.10:2.2.2' 156 | } 157 | } 158 | 159 | project (':metrics-test') { 160 | apply plugin: 'scala' 161 | apply plugin: 'application' 162 | 163 | mainClassName = "ly.stealth.psutil.PsutilKafka" 164 | 165 | run { 166 | args Eval.me("['psutil-kafka-topic','localhost','5555','localhost:2181']") 167 | } 168 | 169 | dependencies { 170 | compile project(":riemann") 171 | compile project(":psutil") 172 | compile project(":psutil-statsd") 173 | } 174 | } 175 | 176 | project (':psutil') { 177 | 178 | jar { 179 | from 'src/main/python/psutil_producer.py' 180 | } 181 | 182 | task installDependencies(type:Exec) { 183 | commandLine 'sudo', "$rootDir/vagrant/psutil.sh" 184 | } 185 | 186 | task runPython(type:Exec) { 187 | commandLine 'sudo', "docker", "run", "--link", "broker1:localhost", "--rm", "-v", "$rootDir/psutil/src/main/python:/psutil", "stealthly/docker-python", "python", "/psutil/psutil_producer_test.py" 188 | } 189 | 190 | test.dependsOn runPython 191 | } 192 | 193 | project (':psutil-entities') { 194 | apply plugin:'scala' 195 | 196 | dependencies { 197 | compile 'org.apache.kafka:kafka_2.10:0.8.1' 198 | compile 'com.fasterxml.jackson.core:jackson-core:2.4.1' 199 | compile 'com.fasterxml.jackson.core:jackson-annotations:2.4.1' 200 | compile 'com.fasterxml.jackson.core:jackson-databind:2.4.1' 201 | compile 'org.slf4j:slf4j-log4j12:1.7.7' 202 | } 203 | } 204 | 205 | project (':psutil-statsd') { 206 | apply plugin:'scala' 207 | 208 | dependencies { 209 | compile 'org.apache.kafka:kafka_2.10:0.8.1' 210 | compile 'com.fasterxml.jackson.core:jackson-annotations:2.4.1' 211 | compile 'org.slf4j:slf4j-log4j12:1.7.7' 212 | 213 | compile project(':statsd') 214 | compile project(':psutil-entities') 215 | } 216 | } 217 | 218 | project (':statsd') { 219 | apply plugin:'scala' 220 | 221 | dependencies { 222 | compile 'org.apache.kafka:kafka_2.10:0.8.1' 223 | compile 'org.slf4j:slf4j-log4j12:1.7.7' 224 | compile group: 'com.typesafe.akka', name: 'akka-actor_2.10', version: '2.2.3' 225 | 226 | testCompile 'junit:junit:4.11' 227 | } 228 | } 229 | 230 | project (':graphite') { 231 | } -------------------------------------------------------------------------------- /codahale/src/main/java/ly/stealth/kafka/metrics/KafkaGraphiteConsumer.java: -------------------------------------------------------------------------------- 1 | package ly.stealth.kafka.metrics; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import kafka.consumer.*; 5 | import kafka.message.MessageAndMetadata; 6 | import kafka.serializer.StringDecoder; 7 | import ly.stealth.graphite.GraphiteClient; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.Map; 12 | import java.util.Properties; 13 | import java.util.concurrent.atomic.AtomicBoolean; 14 | 15 | public class KafkaGraphiteConsumer { 16 | private static final Logger log = LoggerFactory.getLogger(KafkaGraphiteConsumer.class); 17 | private ConsumerConnector consumerConnector; 18 | private KafkaStream messageStream; 19 | private GraphiteClient graphiteClient; 20 | private ObjectMapper mapper = new ObjectMapper(); 21 | private AtomicBoolean stopped = new AtomicBoolean(false); 22 | 23 | public KafkaGraphiteConsumer(String graphiteHost, 24 | int graphitePort, 25 | String zkConnect, 26 | String topic, 27 | String groupId, 28 | int zkSessionTimeoutMs, 29 | boolean readFromStartOfStream) { 30 | Properties props = new Properties(); 31 | props.put("group.id", groupId); 32 | props.put("zookeeper.connect", zkConnect); 33 | props.put("auto.offset.reset", readFromStartOfStream ? "smallest" : "largest"); 34 | props.put("zookeeper.session.timeout.ms", String.valueOf(zkSessionTimeoutMs)); 35 | 36 | ConsumerConfig config = new ConsumerConfig(props); 37 | consumerConnector = Consumer.create(config); 38 | TopicFilter filterSpec = new Whitelist(topic); 39 | 40 | log.info("Trying to start consumer: topic=%s for zk=%s and groupId=%s".format(topic, zkConnect, groupId)); 41 | messageStream = consumerConnector.createMessageStreamsByFilter(filterSpec, 42 | 1, 43 | new StringDecoder(null), 44 | new StringDecoder(null)).head(); 45 | log.info("Started consumer: topic=%s for zk=%s and groupId=%s".format(topic, zkConnect, groupId)); 46 | graphiteClient = new GraphiteClient(graphiteHost, graphitePort); 47 | } 48 | 49 | public void start() { 50 | new Thread(new Runnable() { 51 | public void run() { 52 | log.info("reading on stream now"); 53 | ConsumerIterator it = messageStream.iterator(); 54 | while (it.hasNext()) { 55 | MessageAndMetadata messageAndTopic = it.next(); 56 | try { 57 | KafkaMetricsReport report = mapper.readValue(messageAndTopic.message(), KafkaMetricsReport.class); 58 | if (report.getCounters() != null) { 59 | for (Map.Entry entry : report.getCounters().entrySet()) { 60 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "total"), entry.getValue().getCount()); 61 | } 62 | } 63 | 64 | if (report.getHistograms() != null) { 65 | for (Map.Entry entry : report.getHistograms().entrySet()) { 66 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "total"), entry.getValue().getCount()); 67 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "max"), entry.getValue().getMax()); 68 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "min"), entry.getValue().getMin()); 69 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "mean"), entry.getValue().getMean()); 70 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "P50"), entry.getValue().getP50()); 71 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "P75"), entry.getValue().getP75()); 72 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "P95"), entry.getValue().getP95()); 73 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "P98"), entry.getValue().getP98()); 74 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "P99"), entry.getValue().getP99()); 75 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "P999"), entry.getValue().getP999()); 76 | graphiteClient.sendMetric("%s [%s]".format(entry.getKey(), "standard deviation"), entry.getValue().getStddev()); 77 | } 78 | } 79 | 80 | if (report.getTimers() != null) { 81 | for (Map.Entry entry : report.getTimers().entrySet()) { 82 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "max duration", entry.getValue().getDuration_units()), entry.getValue().getMax()); 83 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "min duration", entry.getValue().getDuration_units()), entry.getValue().getMin()); 84 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "mean duration", entry.getValue().getDuration_units()), entry.getValue().getMean()); 85 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "P50 duration", entry.getValue().getDuration_units()), entry.getValue().getP50()); 86 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "P75 duration", entry.getValue().getDuration_units()), entry.getValue().getP75()); 87 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "P95 duration", entry.getValue().getDuration_units()), entry.getValue().getP95()); 88 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "P98 duration", entry.getValue().getDuration_units()), entry.getValue().getP98()); 89 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "P99 duration", entry.getValue().getDuration_units()), entry.getValue().getP99()); 90 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "P999 duration", entry.getValue().getDuration_units()), entry.getValue().getP999()); 91 | 92 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "M1 rate", entry.getValue().getRate_units()), entry.getValue().getM1_rate()); 93 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "M5 rate", entry.getValue().getRate_units()), entry.getValue().getM5_rate()); 94 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "M15 rate", entry.getValue().getRate_units()), entry.getValue().getM15_rate()); 95 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "mean rate", entry.getValue().getRate_units()), entry.getValue().getMean_rate()); 96 | 97 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "standard deviation", entry.getValue().getRate_units()), entry.getValue().getStddev()); 98 | } 99 | } 100 | 101 | if (report.getMeters() != null) { 102 | for (Map.Entry entry : report.getMeters().entrySet()) { 103 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "total", entry.getValue().getUnits()), entry.getValue().getCount()); 104 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "M1 rate", entry.getValue().getUnits()), entry.getValue().getM1_rate()); 105 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "M5 rate", entry.getValue().getUnits()), entry.getValue().getM5_rate()); 106 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "M15 rate", entry.getValue().getUnits()), entry.getValue().getM15_rate()); 107 | graphiteClient.sendMetric("%s [%s(%s)]".format(entry.getKey(), "mean rate", entry.getValue().getUnits()), entry.getValue().getMean_rate()); 108 | } 109 | } 110 | } catch (Throwable e) { 111 | if (stopped.get()) { 112 | log.info("Consumer worker has been stopped"); 113 | return; 114 | } else { 115 | log.warn("Error processing message, skipping this message: ", e); 116 | } 117 | } 118 | } 119 | } 120 | }).start(); 121 | } 122 | 123 | public void stop() { 124 | log.info("Trying to stop consumer"); 125 | consumerConnector.shutdown(); 126 | stopped.set(true); 127 | log.info("Consumer has been stopped"); 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /codahale/src/main/java/ly/stealth/kafka/metrics/KafkaMetricsReport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package ly.stealth.kafka.metrics; 18 | 19 | import java.io.Serializable; 20 | import java.util.SortedMap; 21 | 22 | /** 23 | * Created by ilyutov on 4/10/14. 24 | */ 25 | public class KafkaMetricsReport implements Serializable { 26 | private static final long serialVersionUID = 1L; 27 | 28 | private String version; 29 | private SortedMap gauges; 30 | private SortedMap counters; 31 | private SortedMap histograms; 32 | private SortedMap meters; 33 | private SortedMap timers; 34 | 35 | public String getVersion() { 36 | return version; 37 | } 38 | 39 | public SortedMap getGauges() { 40 | return gauges; 41 | } 42 | 43 | public SortedMap getCounters() { 44 | return counters; 45 | } 46 | 47 | public SortedMap getHistograms() { 48 | return histograms; 49 | } 50 | 51 | public SortedMap getMeters() { 52 | return meters; 53 | } 54 | 55 | public SortedMap getTimers() { 56 | return timers; 57 | } 58 | 59 | public static class Gauge { 60 | private Object value; 61 | 62 | public Object getValue() { 63 | return value; 64 | } 65 | } 66 | 67 | public static class Counter { 68 | private int count; 69 | 70 | public int getCount() { 71 | return count; 72 | } 73 | } 74 | 75 | public static class Histogram { 76 | private long count; 77 | private long max; 78 | private long mean; 79 | private long min; 80 | private long p50; 81 | private long p75; 82 | private long p95; 83 | private long p98; 84 | private long p99; 85 | private long p999; 86 | private long stddev; 87 | private Object values; 88 | 89 | public long getCount() { 90 | return count; 91 | } 92 | 93 | public long getMax() { 94 | return max; 95 | } 96 | 97 | public long getMean() { 98 | return mean; 99 | } 100 | 101 | public long getMin() { 102 | return min; 103 | } 104 | 105 | public long getP50() { 106 | return p50; 107 | } 108 | 109 | public long getP75() { 110 | return p75; 111 | } 112 | 113 | public long getP95() { 114 | return p95; 115 | } 116 | 117 | public long getP98() { 118 | return p98; 119 | } 120 | 121 | public long getP99() { 122 | return p99; 123 | } 124 | 125 | public long getP999() { 126 | return p999; 127 | } 128 | 129 | public long getStddev() { 130 | return stddev; 131 | } 132 | 133 | public Object getValues() { 134 | return values; 135 | } 136 | } 137 | 138 | public static class Meter { 139 | private long count; 140 | private long m15_rate; 141 | private long m1_rate; 142 | private long m5_rate; 143 | private long mean_rate; 144 | private String units; 145 | 146 | public long getCount() { 147 | return count; 148 | } 149 | 150 | public long getM15_rate() { 151 | return m15_rate; 152 | } 153 | 154 | public long getM1_rate() { 155 | return m1_rate; 156 | } 157 | 158 | public long getM5_rate() { 159 | return m5_rate; 160 | } 161 | 162 | public long getMean_rate() { 163 | return mean_rate; 164 | } 165 | 166 | public String getUnits() { 167 | return units; 168 | } 169 | } 170 | 171 | public static class Timer { 172 | private long count; 173 | private long max; 174 | private long mean; 175 | private long min; 176 | 177 | private long p50; 178 | private long p75; 179 | private long p95; 180 | private long p98; 181 | private long p99; 182 | private long p999; 183 | 184 | private long stddev; 185 | private long m15_rate; 186 | private long m1_rate; 187 | private long m5_rate; 188 | private long mean_rate; 189 | 190 | private String duration_units; 191 | private String rate_units; 192 | 193 | private Object values; 194 | 195 | public long getCount() { 196 | return count; 197 | } 198 | 199 | public long getMax() { 200 | return max; 201 | } 202 | 203 | public long getMean() { 204 | return mean; 205 | } 206 | 207 | public long getMin() { 208 | return min; 209 | } 210 | 211 | public long getP50() { 212 | return p50; 213 | } 214 | 215 | public long getP75() { 216 | return p75; 217 | } 218 | 219 | public long getP95() { 220 | return p95; 221 | } 222 | 223 | public long getP98() { 224 | return p98; 225 | } 226 | 227 | public long getP99() { 228 | return p99; 229 | } 230 | 231 | public long getP999() { 232 | return p999; 233 | } 234 | 235 | public long getStddev() { 236 | return stddev; 237 | } 238 | 239 | public long getM15_rate() { 240 | return m15_rate; 241 | } 242 | 243 | public long getM1_rate() { 244 | return m1_rate; 245 | } 246 | 247 | public long getM5_rate() { 248 | return m5_rate; 249 | } 250 | 251 | public long getMean_rate() { 252 | return mean_rate; 253 | } 254 | 255 | public String getDuration_units() { 256 | return duration_units; 257 | } 258 | 259 | public String getRate_units() { 260 | return rate_units; 261 | } 262 | 263 | public Object getValues() { 264 | return values; 265 | } 266 | } 267 | } 268 | -------------------------------------------------------------------------------- /codahale/src/main/java/ly/stealth/kafka/metrics/KafkaReporter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package ly.stealth.kafka.metrics; 18 | 19 | import com.codahale.metrics.*; 20 | import com.codahale.metrics.json.MetricsModule; 21 | import com.fasterxml.jackson.databind.ObjectMapper; 22 | import kafka.javaapi.producer.Producer; 23 | import kafka.producer.KeyedMessage; 24 | import kafka.producer.ProducerConfig; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | import java.io.IOException; 29 | import java.io.StringWriter; 30 | import java.util.Properties; 31 | import java.util.SortedMap; 32 | import java.util.concurrent.TimeUnit; 33 | 34 | public class KafkaReporter extends ScheduledReporter { 35 | private static final Logger log = LoggerFactory.getLogger(KafkaReporter.class); 36 | 37 | private final Producer kafkaProducer; 38 | private final String kafkaTopic; 39 | private final ObjectMapper mapper; 40 | private final MetricRegistry registry; 41 | 42 | private KafkaReporter(MetricRegistry registry, 43 | String name, 44 | MetricFilter filter, 45 | TimeUnit rateUnit, 46 | TimeUnit durationUnit, 47 | String kafkaTopic, 48 | Properties kafkaProperties) { 49 | super(registry, name, filter, rateUnit, durationUnit); 50 | this.registry = registry; 51 | mapper = new ObjectMapper().registerModule(new MetricsModule(rateUnit, 52 | durationUnit, 53 | false)); 54 | this.kafkaTopic = kafkaTopic; 55 | kafkaProducer = new Producer(new ProducerConfig(kafkaProperties)); 56 | } 57 | 58 | @Override 59 | public synchronized void report(SortedMap gauges, 60 | SortedMap counters, 61 | SortedMap histograms, 62 | SortedMap meters, 63 | SortedMap timers) { 64 | try { 65 | log.info("Trying to report metrics to Kafka kafkaTopic {}", kafkaTopic); 66 | StringWriter report = new StringWriter(); 67 | mapper.writeValue(report, registry); 68 | log.debug("Created metrics report: {}", report); 69 | kafkaProducer.send(new KeyedMessage(kafkaTopic, report.toString())); 70 | log.info("Metrics were successfully reported to Kafka kafkaTopic {}", kafkaTopic); 71 | } catch (IOException e) { 72 | throw new RuntimeException(e.getMessage(), e); 73 | } 74 | } 75 | 76 | public static Builder builder(MetricRegistry registry, String brokerList, String kafkaTopic) { 77 | return new Builder(registry, kafkaTopic, brokerList); 78 | } 79 | 80 | public static class Builder { 81 | private MetricRegistry registry; 82 | private String kafkaTopic; 83 | private String brokerList; 84 | 85 | private boolean synchronously = false; 86 | private int compressionCodec = 0; 87 | private int batchSize = 200; 88 | private int messageSendMaxRetries = 3; 89 | 90 | private String name = "KafkaReporter"; 91 | private MetricFilter filter = MetricFilter.ALL; 92 | private TimeUnit rateUnit = TimeUnit.SECONDS; 93 | private TimeUnit durationUnit = TimeUnit.SECONDS; 94 | 95 | public Builder(MetricRegistry registry, String topic, String brokerList) { 96 | this.registry = registry; 97 | this.kafkaTopic = topic; 98 | this.brokerList = brokerList; 99 | } 100 | 101 | public String getKafkaTopic() { 102 | return kafkaTopic; 103 | } 104 | 105 | public Builder setKafkaTopic(String kafkaTopic) { 106 | this.kafkaTopic = kafkaTopic; 107 | return this; 108 | } 109 | 110 | public String getBrokerList() { 111 | return brokerList; 112 | } 113 | 114 | public Builder setBrokerList(String brokerList) { 115 | this.brokerList = brokerList; 116 | return this; 117 | } 118 | 119 | public boolean isSynchronously() { 120 | return synchronously; 121 | } 122 | 123 | public Builder setSynchronously(boolean synchronously) { 124 | this.synchronously = synchronously; 125 | return this; 126 | } 127 | 128 | public int getCompressionCodec() { 129 | return compressionCodec; 130 | } 131 | 132 | public Builder setCompressionCodec(int compressionCodec) { 133 | this.compressionCodec = compressionCodec; 134 | return this; 135 | } 136 | 137 | public int getBatchSize() { 138 | return batchSize; 139 | } 140 | 141 | public Builder setBatchSize(int batchSize) { 142 | this.batchSize = batchSize; 143 | return this; 144 | } 145 | 146 | public int getMessageSendMaxRetries() { 147 | return messageSendMaxRetries; 148 | } 149 | 150 | public Builder setMessageSendMaxRetries(int messageSendMaxRetries) { 151 | this.messageSendMaxRetries = messageSendMaxRetries; 152 | return this; 153 | } 154 | 155 | public MetricRegistry getRegistry() { 156 | return registry; 157 | } 158 | 159 | public Builder setRegistry(MetricRegistry registry) { 160 | this.registry = registry; 161 | return this; 162 | } 163 | 164 | public String getName() { 165 | return name; 166 | } 167 | 168 | public Builder setName(String name) { 169 | this.name = name; 170 | return this; 171 | } 172 | 173 | public MetricFilter getFilter() { 174 | return filter; 175 | } 176 | 177 | public Builder setFilter(MetricFilter filter) { 178 | this.filter = filter; 179 | return this; 180 | } 181 | 182 | public TimeUnit getRateUnit() { 183 | return rateUnit; 184 | } 185 | 186 | public Builder setRateUnit(TimeUnit rateUnit) { 187 | this.rateUnit = rateUnit; 188 | return this; 189 | } 190 | 191 | public TimeUnit getDurationUnit() { 192 | return durationUnit; 193 | } 194 | 195 | public Builder setDurationUnit(TimeUnit durationUnit) { 196 | this.durationUnit = durationUnit; 197 | return this; 198 | } 199 | 200 | public KafkaReporter build() { 201 | Properties props = new Properties(); 202 | props.put("metadata.broker.list", brokerList); 203 | props.put("serializer.class", "kafka.serializer.StringEncoder"); 204 | props.put("producer.type", synchronously ? "sync" : "async"); 205 | props.put("compression.codec", String.valueOf(compressionCodec)); 206 | props.put("batch.num.messages", String.valueOf(batchSize)); 207 | props.put("message.send.max.retries", String.valueOf(messageSendMaxRetries)); 208 | props.put("compression.codec", String.valueOf(compressionCodec)); 209 | 210 | return new KafkaReporter(registry, name, filter, rateUnit, durationUnit, kafkaTopic, props); 211 | } 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /codahale/src/test/java/ly/stealth/kafka/metrics/MetricsReportingSteps.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package ly.stealth.kafka.metrics; 18 | 19 | import com.codahale.metrics.MetricRegistry; 20 | import com.fasterxml.jackson.databind.ObjectMapper; 21 | import kafka.consumer.*; 22 | import kafka.javaapi.consumer.ConsumerConnector; 23 | import kafka.serializer.StringDecoder; 24 | import org.jbehave.core.annotations.Given; 25 | import org.jbehave.core.annotations.Then; 26 | import org.jbehave.core.annotations.When; 27 | 28 | import java.io.IOException; 29 | import java.util.*; 30 | 31 | import static org.junit.Assert.assertNotNull; 32 | 33 | public class MetricsReportingSteps { 34 | private final String zkConnect = "localhost:2181"; 35 | private final String kafkaConnect = "localhost:9092"; 36 | private final String topic = UUID.randomUUID().toString(); 37 | private KafkaReporter kafkaReporter; 38 | private MetricRegistry registry; 39 | 40 | @Given("Kafka broker is up and 'metrics' topic is created.") 41 | public void startingKafkaReporterAndCon() { 42 | registry = new MetricRegistry(); 43 | registry.counter("test_counter").inc(); 44 | 45 | kafkaReporter = KafkaReporter.builder(registry, 46 | kafkaConnect, 47 | topic).build(); 48 | } 49 | 50 | @When("KafkaReporter sends data to Kafka topic.") 51 | public void reporterWritesMetrics() { 52 | kafkaReporter.report(); 53 | } 54 | 55 | @Then("Kafka consumer should be able to read this data.") 56 | public void consumerReadsMetrics() throws IOException { 57 | ConsumerConnector consumer = Consumer.createJavaConsumerConnector(createConsumerConfig()); 58 | String message = readMessage(consumer); 59 | assertNotNull(message); 60 | ObjectMapper objectMapper = new ObjectMapper(); 61 | KafkaMetricsReport report = objectMapper.readValue(message, KafkaMetricsReport.class); 62 | assertNotNull(report); 63 | } 64 | 65 | private ConsumerConfig createConsumerConfig() { 66 | Properties props = new Properties(); 67 | props.put("zookeeper.connect", zkConnect); 68 | props.put("group.id", UUID.randomUUID().toString()); 69 | props.put("auto.offset.reset", "smallest"); 70 | props.put("zookeeper.session.timeout.ms", "30000"); 71 | props.put("consumer.timeout.ms", "30000"); 72 | return new ConsumerConfig(props); 73 | } 74 | 75 | public String readMessage(ConsumerConnector consumer) { 76 | Map topicCountMap = new HashMap(); 77 | topicCountMap.put(topic, new Integer(1)); 78 | KafkaStream messageStream = consumer.createMessageStreamsByFilter(new Whitelist(topic), 79 | 1, 80 | new StringDecoder(null), 81 | new StringDecoder(null)).get(0); 82 | 83 | return messageStream.iterator().next().message(); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /codahale/src/test/java/ly/stealth/kafka/metrics/Stories.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package ly.stealth.kafka.metrics; 18 | 19 | import java.util.List; 20 | 21 | import de.codecentric.jbehave.junit.monitoring.JUnitReportingRunner; 22 | import org.jbehave.core.configuration.Configuration; 23 | import org.jbehave.core.configuration.MostUsefulConfiguration; 24 | import org.jbehave.core.io.CodeLocations; 25 | import org.jbehave.core.io.LoadFromClasspath; 26 | import org.jbehave.core.io.StoryFinder; 27 | import org.jbehave.core.junit.JUnitStories; 28 | import org.jbehave.core.reporters.Format; 29 | import org.jbehave.core.reporters.StoryReporterBuilder; 30 | import org.jbehave.core.steps.InjectableStepsFactory; 31 | import org.jbehave.core.steps.InstanceStepsFactory; 32 | import org.junit.runner.RunWith; 33 | 34 | /** 35 | * Created by ilyutov on 4/11/14. 36 | */ 37 | @RunWith(JUnitReportingRunner.class) 38 | public class Stories extends JUnitStories { 39 | @Override 40 | protected List storyPaths() { 41 | return new StoryFinder().findPaths(CodeLocations.codeLocationFromPath("src/test/resources"), 42 | "**/*.story", 43 | "**/exclude_*.story"); 44 | } 45 | 46 | @Override 47 | public Configuration configuration() { 48 | return new MostUsefulConfiguration() 49 | .useStoryLoader(new LoadFromClasspath(this.getClass())) 50 | .useStoryReporterBuilder(new StoryReporterBuilder() 51 | .withFormats(Format.TXT) 52 | .withRelativeDirectory("../build/jbehave/")); 53 | } 54 | 55 | @Override 56 | public InjectableStepsFactory stepsFactory() { 57 | return new InstanceStepsFactory(configuration(), new MetricsReportingSteps()); 58 | } 59 | } -------------------------------------------------------------------------------- /codahale/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | kafka.logs.dir=logs 17 | 18 | log4j.rootLogger=INFO, stdout 19 | 20 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 21 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 22 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n 23 | 24 | log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender 25 | log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH 26 | log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log 27 | log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout 28 | log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 29 | 30 | log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender 31 | log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH 32 | log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log 33 | log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout 34 | log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 35 | 36 | log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender 37 | log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH 38 | log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log 39 | log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout 40 | log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 41 | 42 | log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender 43 | log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH 44 | log4j.appender.cleanerAppender.File=log-cleaner.log 45 | log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout 46 | log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 47 | 48 | log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender 49 | log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH 50 | log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log 51 | log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout 52 | log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 53 | 54 | # Turn on all our debugging info 55 | #log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender 56 | #log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender 57 | #log4j.logger.kafka.perf=DEBUG, kafkaAppender 58 | #log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender 59 | #log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG 60 | log4j.logger.kafka=INFO, kafkaAppender 61 | 62 | log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender 63 | log4j.additivity.kafka.network.RequestChannel$=false 64 | 65 | #log4j.logger.kafka.network.Processor=TRACE, requestAppender 66 | #log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender 67 | #log4j.additivity.kafka.server.KafkaApis=false 68 | log4j.logger.kafka.request.logger=WARN, requestAppender 69 | log4j.additivity.kafka.request.logger=false 70 | 71 | log4j.logger.kafka.controller=TRACE, controllerAppender 72 | log4j.additivity.kafka.controller=false 73 | 74 | log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender 75 | log4j.additivity.kafka.log.LogCleaner=false 76 | log4j.logger.kafka.log.Cleaner=INFO, cleanerAppender 77 | log4j.additivity.kafka.log.Cleaner=false 78 | 79 | log4j.logger.state.change.logger=TRACE, stateChangeAppender 80 | log4j.additivity.state.change.logger=false 81 | -------------------------------------------------------------------------------- /codahale/src/test/resources/stories/metrics_reporting.story: -------------------------------------------------------------------------------- 1 | Meta: 2 | 3 | Narrative: 4 | As a user 5 | I want to send metrics via Reporter to Kafka topic 6 | So that I can read this data via kafka consumer 7 | 8 | Scenario: Metrics reporter sends data to kafka topic and kafka consumer should be able to read it. 9 | Given Kafka broker is up and 'metrics' topic is created. 10 | When KafkaReporter sends data to Kafka topic. 11 | Then Kafka consumer should be able to read this data. -------------------------------------------------------------------------------- /config/config.rb: -------------------------------------------------------------------------------- 1 | # Serve HTTP traffic on this port 2 | set :bind, "192.168.86.55" 3 | set :port, 4567 4 | 5 | riemann_base = '/var/lib/gems/1.9.1/gems/riemann-dash-0.2.8' 6 | riemann_src = "#{riemann_base}/lib/riemann/dash" 7 | 8 | # Add custom controllers in controller/ 9 | config.store[:controllers] = ["#{riemann_src}/controller"] 10 | 11 | # Use the local view directory instead of the default 12 | config.store[:views] = "#{riemann_src}/views" 13 | 14 | # Specify a custom path to your workspace config.json 15 | config.store[:ws_config] = "/vagrant/config/dash-config.json" 16 | 17 | # Serve static files from this directory 18 | config.store[:public] = "#{riemann_src}/public" 19 | -------------------------------------------------------------------------------- /config/dash-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "server": "192.168.86.55:5556", 3 | "server_type": "ws", 4 | "workspaces": [ 5 | { 6 | "view": { 7 | "type": "Balloon", 8 | "weight": 1, 9 | "child": { 10 | "children": [ 11 | { 12 | "children": [ 13 | { 14 | "children": [ 15 | { 16 | "type": "Grid", 17 | "cols": "", 18 | "title": "Health", 19 | "weight": 1, 20 | "query": "service = \"cpu percent\" or service = \"memory used %\" or service = \"disk used %\" ", 21 | "max": "", 22 | "rows": "" 23 | }, 24 | { 25 | "type": "TimeSeries", 26 | "speed": "", 27 | "opacity": "", 28 | "title": "Health timeseries", 29 | "weight": 1, 30 | "lineWidth": "", 31 | "query": "service = \"cpu percent\" or service = \"memory used %\" or service = \"disk used %\" ", 32 | "delay": "" 33 | }, 34 | { 35 | "type": "Gauge", 36 | "title": "CPU load", 37 | "weight": 1, 38 | "query": "service = \"cpu percent\"" 39 | } 40 | ], 41 | "type": "VStack", 42 | "weight": 1 43 | }, 44 | { 45 | "children": [ 46 | { 47 | "type": "Flot", 48 | "graphType": "line", 49 | "title": "CPU", 50 | "weight": 1, 51 | "stackMode": "false", 52 | "timeRange": 300, 53 | "query": "tagged \"cpu\"" 54 | }, 55 | { 56 | "type": "Flot", 57 | "graphType": "line", 58 | "title": "Memory", 59 | "weight": 1, 60 | "stackMode": "false", 61 | "timeRange": 300, 62 | "query": "tagged \"memory\"" 63 | }, 64 | { 65 | "type": "Flot", 66 | "graphType": "line", 67 | "title": "Disk", 68 | "weight": 1, 69 | "stackMode": "false", 70 | "timeRange": 300, 71 | "query": "tagged \"disk\"" 72 | } 73 | ], 74 | "type": "VStack", 75 | "weight": 1 76 | } 77 | ], 78 | "type": "HStack", 79 | "weight": 1 80 | } 81 | ], 82 | "type": "VStack", 83 | "weight": 1 84 | } 85 | }, 86 | "name": "Riemann", 87 | "id": "4d0dd4a79a195d4ac83a78310d9f2fa07c3035e0" 88 | } 89 | ] 90 | } -------------------------------------------------------------------------------- /config/riemann.config: -------------------------------------------------------------------------------- 1 | ; -*- mode: clojure; -*- 2 | ; vim: filetype=clojure 3 | 4 | (logging/init :file "riemann.log") 5 | 6 | ; Listen on the local interface over TCP (5555), UDP (5555), and websockets 7 | ; (5556) 8 | (let [host "192.168.86.55"] 9 | (tcp-server :host host) 10 | (udp-server :host host) 11 | (ws-server :host host)) 12 | 13 | ; Expire old events from the index every 5 seconds. 14 | (periodically-expire 5) 15 | 16 | ; Keep events in the index for 5 minutes by default. 17 | (let [index (default :ttl 300 (update-index (index)))] 18 | 19 | ; Inbound events will be passed to these streams: 20 | (streams 21 | 22 | ; Index all events immediately. 23 | index 24 | 25 | ; Calculate an overall rate of events. 26 | (with {:metric 1 :host nil :state "ok" :service "events/sec"} 27 | (rate 5 index)) 28 | 29 | ; Log expired events. 30 | (expired 31 | (fn [event] (info "expired" event))) 32 | )) 33 | 34 | (streams 35 | prn) 36 | -------------------------------------------------------------------------------- /config/server.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # see kafka.server.KafkaConfig for additional details and defaults 16 | 17 | ############################# Server Basics ############################# 18 | 19 | # The id of the broker. This must be set to a unique integer for each broker. 20 | broker.id=0 21 | 22 | ############################# Socket Server Settings ############################# 23 | 24 | # The port the socket server listens on 25 | port=9092 26 | 27 | kafka.metrics.reporters=ly.stealth.kafka.core.metrics.KafkaBrokerReporter 28 | 29 | # Hostname the broker will bind to and advertise to producers and consumers. 30 | # If not set, the server will bind to all interfaces and advertise the value returned from 31 | # from java.net.InetAddress.getCanonicalHostName(). 32 | #host.name=localhost 33 | 34 | # The number of threads handling network requests 35 | num.network.threads=2 36 | 37 | # The number of threads doing disk I/O 38 | num.io.threads=2 39 | 40 | # The send buffer (SO_SNDBUF) used by the socket server 41 | socket.send.buffer.bytes=1048576 42 | 43 | # The receive buffer (SO_RCVBUF) used by the socket server 44 | socket.receive.buffer.bytes=1048576 45 | 46 | # The maximum size of a request that the socket server will accept (protection against OOM) 47 | socket.request.max.bytes=104857600 48 | 49 | 50 | ############################# Log Basics ############################# 51 | 52 | # A comma seperated list of directories under which to store log files 53 | log.dirs=/tmp/kafka-logs 54 | 55 | # The number of logical partitions per topic per server. More partitions allow greater parallelism 56 | # for consumption, but also mean more files. 57 | num.partitions=1 58 | 59 | ############################# Log Flush Policy ############################# 60 | 61 | # The following configurations control the flush of data to disk. This is among the most 62 | # important performance knob in kafka. 63 | # There are a few important trade-offs here: 64 | # 1. Durability: Unflushed data may be lost if you are not using replication. 65 | # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. 66 | # 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. 67 | # The settings below allow one to configure the flush policy to flush data after a period of time or 68 | # every N messages (or both). This can be done globally and overridden on a per-topic basis. 69 | 70 | # The number of messages to accept before forcing a flush of data to disk 71 | log.flush.interval.messages=10000 72 | 73 | # The maximum amount of time a message can sit in a log before we force a flush 74 | log.flush.interval.ms=1000 75 | 76 | # Per-topic overrides for log.flush.interval.ms 77 | #log.flush.intervals.ms.per.topic=topic1:1000, topic2:3000 78 | 79 | ############################# Log Retention Policy ############################# 80 | 81 | # The following configurations control the disposal of log segments. The policy can 82 | # be set to delete segments after a period of time, or after a given size has accumulated. 83 | # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens 84 | # from the end of the log. 85 | 86 | # The minimum age of a log file to be eligible for deletion 87 | log.retention.hours=168 88 | 89 | # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining 90 | # segments don't drop below log.retention.bytes. 91 | #log.retention.bytes=1073741824 92 | 93 | # The maximum size of a log segment file. When this size is reached a new log segment will be created. 94 | log.segment.bytes=536870912 95 | 96 | # The interval at which log segments are checked to see if they can be deleted according 97 | # to the retention policies 98 | log.cleanup.interval.mins=1 99 | 100 | ############################# Zookeeper ############################# 101 | 102 | # Zookeeper connection string (see zookeeper docs for details). 103 | # This is a comma separated host:port pairs, each corresponding to a zk 104 | # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". 105 | # You can also append an optional chroot string to the urls to specify the 106 | # root directory for all kafka znodes. 107 | zookeeper.connect=localhost:2181 108 | 109 | # Timeout in ms for connecting to zookeeper 110 | zookeeper.connection.timeout.ms=1000000 111 | 112 | 113 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | gradleVersion="2.0" 2 | 3 | mavenURL="" 4 | mavenUsername="" 5 | mavenPassword="" 6 | 7 | org.gradle.jvmargs=-XX:MaxPermSize=1g -Xmx1g 8 | signing.keyId= 9 | signing.password= 10 | signing.secretKeyRingFile= 11 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stealthly/metrics-kafka/a4b638ef7e0190ea067dd6e9459deaa312ce0351/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Tue Apr 15 19:51:47 EEST 2014 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=http\://services.gradle.org/distributions/gradle-1.11-bin.zip 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # For Cygwin, ensure paths are in UNIX format before anything is touched. 46 | if $cygwin ; then 47 | [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 48 | fi 49 | 50 | # Attempt to set APP_HOME 51 | # Resolve links: $0 may be a link 52 | PRG="$0" 53 | # Need this for relative symlinks. 54 | while [ -h "$PRG" ] ; do 55 | ls=`ls -ld "$PRG"` 56 | link=`expr "$ls" : '.*-> \(.*\)$'` 57 | if expr "$link" : '/.*' > /dev/null; then 58 | PRG="$link" 59 | else 60 | PRG=`dirname "$PRG"`"/$link" 61 | fi 62 | done 63 | SAVED="`pwd`" 64 | cd "`dirname \"$PRG\"`/" >&- 65 | APP_HOME="`pwd -P`" 66 | cd "$SAVED" >&- 67 | 68 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 69 | 70 | # Determine the Java command to use to start the JVM. 71 | if [ -n "$JAVA_HOME" ] ; then 72 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 73 | # IBM's JDK on AIX uses strange locations for the executables 74 | JAVACMD="$JAVA_HOME/jre/sh/java" 75 | else 76 | JAVACMD="$JAVA_HOME/bin/java" 77 | fi 78 | if [ ! -x "$JAVACMD" ] ; then 79 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 80 | 81 | Please set the JAVA_HOME variable in your environment to match the 82 | location of your Java installation." 83 | fi 84 | else 85 | JAVACMD="java" 86 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 87 | 88 | Please set the JAVA_HOME variable in your environment to match the 89 | location of your Java installation." 90 | fi 91 | 92 | # Increase the maximum file descriptors if we can. 93 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 94 | MAX_FD_LIMIT=`ulimit -H -n` 95 | if [ $? -eq 0 ] ; then 96 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 97 | MAX_FD="$MAX_FD_LIMIT" 98 | fi 99 | ulimit -n $MAX_FD 100 | if [ $? -ne 0 ] ; then 101 | warn "Could not set maximum file descriptor limit: $MAX_FD" 102 | fi 103 | else 104 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 105 | fi 106 | fi 107 | 108 | # For Darwin, add options to specify how the application appears in the dock 109 | if $darwin; then 110 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 111 | fi 112 | 113 | # For Cygwin, switch paths to Windows format before running java 114 | if $cygwin ; then 115 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 116 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 158 | function splitJvmOpts() { 159 | JVM_OPTS=("$@") 160 | } 161 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 162 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 163 | 164 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 165 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /graphite/src/main/java/ly/stealth/graphite/GraphiteClient.java: -------------------------------------------------------------------------------- 1 | package ly.stealth.graphite; 2 | 3 | import java.io.IOException; 4 | import java.io.OutputStream; 5 | import java.io.PrintWriter; 6 | import java.net.Socket; 7 | import java.net.UnknownHostException; 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | public class GraphiteClient { 12 | 13 | private final String graphiteHost; 14 | private final int graphitePort; 15 | 16 | /** 17 | * Create a new Graphite client. 18 | * 19 | * @param graphiteHost Host name to write to. 20 | * @param graphitePort Graphite socket. Default is 2003 21 | */ 22 | public GraphiteClient(String graphiteHost, int graphitePort) { 23 | this.graphiteHost = graphiteHost; 24 | this.graphitePort = graphitePort; 25 | } 26 | 27 | /** 28 | * Send a set of metrics with the current time as timestamp to graphite. 29 | * 30 | * @param metrics the metrics as key-value-pairs 31 | */ 32 | public void sendMetrics(Map metrics) { 33 | sendMetrics(metrics, getCurrentTimestamp()); 34 | } 35 | 36 | /** 37 | * Send a set of metrics with a given timestamp to graphite. 38 | * 39 | * @param metrics the metrics as key-value-pairs 40 | * @param timeStamp the timestamp 41 | */ 42 | public void sendMetrics(Map metrics, long timeStamp) { 43 | try { 44 | Socket socket = createSocket(); 45 | OutputStream s = socket.getOutputStream(); 46 | PrintWriter out = new PrintWriter(s, true); 47 | for (Map.Entry metric: metrics.entrySet()) { 48 | out.printf("%s %d %d%n", metric.getKey(), metric.getValue(), timeStamp); 49 | } 50 | out.close(); 51 | socket.close(); 52 | } catch (UnknownHostException e) { 53 | throw new GraphiteException("Unknown host: " + graphiteHost); 54 | } catch (IOException e) { 55 | throw new GraphiteException("Error while writing data to graphite: " + e.getMessage(), e); 56 | } 57 | } 58 | 59 | /** 60 | * Send a single metric with the current time as timestamp to graphite. 61 | * 62 | * @param key The metric key 63 | * @param value the metric value 64 | * 65 | * @throws GraphiteException if writing to graphite fails 66 | */ 67 | public void sendMetric(String key, long value) { 68 | sendMetric(key, value, getCurrentTimestamp()); 69 | } 70 | 71 | /** 72 | * Send a single metric with a given timestamp to graphite. 73 | * 74 | * @param key The metric key 75 | * @param value The metric value 76 | * @param timeStamp the timestamp to use 77 | * 78 | * @throws GraphiteException if writing to graphite fails 79 | */ 80 | @SuppressWarnings("serial") 81 | public void sendMetric(final String key, final Long value, long timeStamp) { 82 | sendMetrics(new HashMap() {{ 83 | put(key, value); 84 | }}, timeStamp); 85 | } 86 | 87 | protected Socket createSocket() throws UnknownHostException, IOException { 88 | return new Socket(graphiteHost, graphitePort); 89 | } 90 | 91 | /*** 92 | * Compute the current graphite timestamp. 93 | * 94 | * @return Seconds passed since 1.1.1970 95 | */ 96 | protected long getCurrentTimestamp() { 97 | return System.currentTimeMillis() / 1000; 98 | } 99 | } 100 | 101 | -------------------------------------------------------------------------------- /graphite/src/main/java/ly/stealth/graphite/GraphiteException.java: -------------------------------------------------------------------------------- 1 | package ly.stealth.graphite; 2 | 3 | public class GraphiteException extends RuntimeException { 4 | 5 | private static final long serialVersionUID = 1L; 6 | 7 | public GraphiteException(String message) { 8 | super(message); 9 | } 10 | 11 | public GraphiteException(String message, Throwable cause) { 12 | super(message, cause); 13 | } 14 | 15 | } -------------------------------------------------------------------------------- /metrics-test/src/main/scala/ly/stealth/psutil/PsutilKafka.scala: -------------------------------------------------------------------------------- 1 | package ly.stealth.psutil 2 | 3 | import java.util.UUID 4 | import java.util.concurrent.TimeUnit 5 | import ly.stealth.kafka.metrics.statsd.KafkaPsutilStatsDConsumer 6 | 7 | object PsutilKafka extends App { 8 | val psutilStatsdConsumer = new KafkaPsutilStatsDConsumer(null, args(1), 8125, false, 1024, args(0), 9 | UUID.randomUUID().toString, args(3), 30000, true) 10 | 11 | psutilStatsdConsumer.start() 12 | TimeUnit.SECONDS.sleep(30) 13 | psutilStatsdConsumer.stop() 14 | } 15 | -------------------------------------------------------------------------------- /psutil-entities/src/main/scala/ly/stealth/psutil/entities/Cpu_times.scala: -------------------------------------------------------------------------------- 1 | 2 | package ly.stealth.psutil.entities 3 | 4 | import com.fasterxml.jackson.annotation.{JsonAutoDetect, JsonInclude} 5 | import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility 6 | 7 | @JsonInclude(JsonInclude.Include.NON_NULL) 8 | @JsonAutoDetect(fieldVisibility = Visibility.ANY, getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) 9 | case class Cpu_times() { 10 | var user: Double = 0 11 | var nice: Double = 0 12 | var system: Double = 0 13 | var idle: Double = 0 14 | var iowait: Double = 0 15 | var irq: Double = 0 16 | var softirq: Double = 0 17 | var steal: Long = 0 18 | var guest: Long = 0 19 | var guest_nice: Long = 0 20 | } 21 | -------------------------------------------------------------------------------- /psutil-entities/src/main/scala/ly/stealth/psutil/entities/Cpu_times_percent.scala: -------------------------------------------------------------------------------- 1 | 2 | package ly.stealth.psutil.entities 3 | 4 | import com.fasterxml.jackson.annotation.{JsonAutoDetect, JsonInclude} 5 | import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility 6 | 7 | @JsonInclude(JsonInclude.Include.NON_NULL) 8 | @JsonAutoDetect(fieldVisibility = Visibility.ANY, getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) 9 | case class Cpu_times_percent() { 10 | var user: Double = 0 11 | var nice: Long = 0 12 | var system: Double = 0 13 | var idle: Double = 0 14 | var iowait: Double = 0 15 | var irq: Long = 0 16 | var softirq: Long = 0 17 | var steal: Long = 0 18 | var guest: Long = 0 19 | var guest_nice: Long = 0 20 | } 21 | -------------------------------------------------------------------------------- /psutil-entities/src/main/scala/ly/stealth/psutil/entities/Disk_io_counters.scala: -------------------------------------------------------------------------------- 1 | 2 | package ly.stealth.psutil.entities 3 | 4 | import com.fasterxml.jackson.annotation.{JsonAutoDetect, JsonInclude} 5 | import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility 6 | 7 | @JsonInclude(JsonInclude.Include.NON_NULL) 8 | @JsonAutoDetect(fieldVisibility = Visibility.ANY, getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) 9 | case class Disk_io_counters() { 10 | var read_count: Long = 0 11 | var write_count: Long = 0 12 | var read_bytes: Long = 0 13 | var write_bytes: Long = 0 14 | var read_time: Long = 0 15 | var write_time: Long = 0 16 | } 17 | -------------------------------------------------------------------------------- /psutil-entities/src/main/scala/ly/stealth/psutil/entities/Disk_usage.scala: -------------------------------------------------------------------------------- 1 | 2 | package ly.stealth.psutil.entities 3 | 4 | import com.fasterxml.jackson.annotation.{JsonAutoDetect, JsonInclude} 5 | import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility 6 | 7 | @JsonInclude(JsonInclude.Include.NON_NULL) 8 | @JsonAutoDetect(fieldVisibility = Visibility.ANY, getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) 9 | case class Disk_usage() { 10 | var total: Long = 0 11 | var used: Long = 0 12 | var free: Long = 0 13 | var percent: Double = 0 14 | } 15 | -------------------------------------------------------------------------------- /psutil-entities/src/main/scala/ly/stealth/psutil/entities/Net_connection.scala: -------------------------------------------------------------------------------- 1 | 2 | package ly.stealth.psutil.entities 3 | 4 | import java.util.ArrayList 5 | import java.util.List 6 | import com.fasterxml.jackson.annotation.{JsonAutoDetect, JsonProperty, JsonInclude} 7 | import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility 8 | 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | @JsonAutoDetect(fieldVisibility = Visibility.ANY, getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) 11 | case class Net_connection() { 12 | var fd: Long = 0 13 | var family: Long = 0 14 | @JsonProperty("type") 15 | var connectionType: Long = 0 16 | var laddr: List[String] = new ArrayList[String]() 17 | var raddr: List[Object] = new ArrayList[Object]() 18 | var status: String = null 19 | var pid: Object = null 20 | } 21 | -------------------------------------------------------------------------------- /psutil-entities/src/main/scala/ly/stealth/psutil/entities/PsutilsReport.scala: -------------------------------------------------------------------------------- 1 | 2 | package ly.stealth.psutil.entities 3 | 4 | import java.util.ArrayList 5 | import java.util.List 6 | import com.fasterxml.jackson.annotation.{JsonIgnoreProperties, JsonAutoDetect, JsonInclude} 7 | import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility 8 | 9 | @JsonInclude(JsonInclude.Include.NON_NULL) 10 | @JsonIgnoreProperties(ignoreUnknown = true) 11 | @JsonAutoDetect(fieldVisibility = Visibility.ANY, getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) 12 | case class PsutilsReport() { 13 | var net_connections: List[Net_connection] = new ArrayList[Net_connection]() 14 | var disk_usage: Disk_usage = null 15 | var cpu_times: Cpu_times = null 16 | var users: List[User] = new ArrayList[User]() 17 | var cpu_percent: Double = 0 18 | var cpu_count: Long = 0 19 | var cpu_times_percent: Cpu_times_percent = null 20 | var boot_time: Long = 0 21 | var swap_memory: Swap_memory = null 22 | var virtual_memory: Virtual_memory = null 23 | var disk_io_counters: Disk_io_counters = null 24 | } 25 | -------------------------------------------------------------------------------- /psutil-entities/src/main/scala/ly/stealth/psutil/entities/Swap_memory.scala: -------------------------------------------------------------------------------- 1 | 2 | package ly.stealth.psutil.entities 3 | 4 | import com.fasterxml.jackson.annotation.{JsonAutoDetect, JsonInclude} 5 | import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility 6 | 7 | @JsonInclude(JsonInclude.Include.NON_NULL) 8 | @JsonAutoDetect(fieldVisibility = Visibility.ANY, getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) 9 | case class Swap_memory() { 10 | var total: Long = 0 11 | var used: Long = 0 12 | var free: Long = 0 13 | var percent: Long = 0 14 | var sin: Long = 0 15 | var sout: Long = 0 16 | } 17 | -------------------------------------------------------------------------------- /psutil-entities/src/main/scala/ly/stealth/psutil/entities/User.scala: -------------------------------------------------------------------------------- 1 | 2 | package ly.stealth.psutil.entities 3 | 4 | import com.fasterxml.jackson.annotation.{JsonAutoDetect, JsonInclude} 5 | import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility 6 | 7 | @JsonInclude(JsonInclude.Include.NON_NULL) 8 | @JsonAutoDetect(fieldVisibility = Visibility.ANY, getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) 9 | case class User() { 10 | var name: String = null 11 | var terminal: String = null 12 | var host: String = null 13 | var started: Long = 0 14 | } 15 | -------------------------------------------------------------------------------- /psutil-entities/src/main/scala/ly/stealth/psutil/entities/Virtual_memory.scala: -------------------------------------------------------------------------------- 1 | 2 | package ly.stealth.psutil.entities 3 | 4 | import com.fasterxml.jackson.annotation.{JsonAutoDetect, JsonInclude} 5 | import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility 6 | 7 | @JsonInclude(JsonInclude.Include.NON_NULL) 8 | @JsonAutoDetect(fieldVisibility = Visibility.ANY, getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) 9 | case class Virtual_memory() { 10 | var total: Long = 0 11 | var available: Long = 0 12 | var percent: Double = 0 13 | var used: Long = 0 14 | var free: Long = 0 15 | var active: Long = 0 16 | var inactive: Long = 0 17 | var buffers: Long = 0 18 | var cached: Long = 0 19 | } 20 | -------------------------------------------------------------------------------- /psutil-statsd/src/main/scala/ly/stealth/kafka/metrics/statsd/KafkaPsutilStatsDConsumer.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package ly.stealth.kafka.metrics.statsd 19 | 20 | import java.util.Properties 21 | import java.util.concurrent.atomic.AtomicBoolean 22 | 23 | import akka.actor.ActorContext 24 | import com.fasterxml.jackson.databind.ObjectMapper 25 | import kafka.consumer.{Consumer, ConsumerConfig, Whitelist} 26 | import kafka.serializer.StringDecoder 27 | import kafka.utils.Logging 28 | import ly.stealth.metrics.kafka.statsd.StatsD 29 | import ly.stealth.psutil.entities.PsutilsReport 30 | 31 | class KafkaPsutilStatsDConsumer(actorContext: ActorContext = null, 32 | statsdHost: String, 33 | statsdPort: Integer, 34 | multiMetrics: Boolean = true, 35 | packetBufferSize: Int = 1024, 36 | topic: String, 37 | groupId: String, 38 | zookeeperConnect: String, 39 | zkSessionTimeoutMs: Int = 30000, 40 | readFromStartOfStream: Boolean = true) extends Logging { 41 | val props = new Properties() 42 | props.put("group.id", groupId) 43 | props.put("zookeeper.connect", zookeeperConnect) 44 | props.put("auto.offset.reset", if (readFromStartOfStream) "smallest" else "largest") 45 | props.put("zookeeper.session.timeout.ms", zkSessionTimeoutMs.toString) 46 | 47 | val config = new ConsumerConfig(props) 48 | val connector = Consumer.create(config) 49 | 50 | val filterSpec = new Whitelist(topic) 51 | val mapper = new ObjectMapper 52 | 53 | info("Trying to start consumer: topic=%s for zk=%s and groupId=%s".format(topic, zookeeperConnect, groupId)) 54 | val stream = connector.createMessageStreamsByFilter(filterSpec, 1, new StringDecoder(), new StringDecoder()).head 55 | info("Started consumer: topic=%s for zk=%s and groupId=%s".format(topic, zookeeperConnect, groupId)) 56 | 57 | val client = new StatsD(actorContext, statsdHost, statsdPort, multiMetrics, packetBufferSize) 58 | val stopped = new AtomicBoolean(false) 59 | 60 | def start() = { 61 | new Thread(new Runnable { 62 | def run(): Unit = { 63 | info("reading on stream now") 64 | val it = stream.iterator() 65 | while (it.hasNext) { 66 | val messageAndTopic = it.next 67 | try { 68 | val report = mapper.readValue(messageAndTopic.message(), classOf[PsutilsReport]) 69 | if (report.boot_time > 0) { 70 | client.timing("boot time", report.boot_time) 71 | } 72 | if (report.cpu_count > 0) { 73 | client.gauge("cpu count", report.cpu_count) 74 | } 75 | if (report.cpu_percent > 0) { 76 | client.gauge("cpu percent", report.cpu_percent) 77 | } 78 | if (report.virtual_memory != null) { 79 | client.gauge("memory used %", report.virtual_memory.percent) 80 | client.gauge("memory used", report.virtual_memory.total) 81 | client.gauge("memory free", report.virtual_memory.free) 82 | client.gauge("memory used", report.virtual_memory.used) 83 | client.gauge("memory available", report.virtual_memory.available) 84 | client.gauge("memory buffers", report.virtual_memory.buffers) 85 | client.gauge("memory total", report.virtual_memory.total) 86 | client.gauge("memory inactive", report.virtual_memory.inactive) 87 | client.gauge("memory active", report.virtual_memory.active) 88 | client.gauge("memory cached", report.virtual_memory.cached) 89 | } 90 | if (report.disk_usage != null) { 91 | client.gauge("disk used %", report.disk_usage.percent) 92 | client.gauge("disk used", report.disk_usage.used) 93 | client.gauge("disk free", report.disk_usage.free) 94 | client.gauge("disk total", report.disk_usage.total) 95 | } 96 | } catch { 97 | case e: Throwable => 98 | if (stopped.get()) { 99 | info("consumer worker has been stoppped") 100 | return 101 | } else { 102 | warn("Error processing message, skipping this message: ", e) 103 | } 104 | } 105 | } 106 | } 107 | }).start() 108 | } 109 | 110 | def stop() = { 111 | info("stopping consumer") 112 | connector.shutdown() 113 | stopped.set(true) 114 | info("stopped consumer") 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /psutil/src/main/python/config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "cpu_times", 4 | "enabled": true, 5 | "perCpu": false 6 | }, 7 | { 8 | "name": "cpu_percent", 9 | "enabled": true, 10 | "interval": 1.0 11 | }, 12 | { 13 | "name": "cpu_times_percent", 14 | "enabled": true, 15 | "interval": 1.0, 16 | "perCpu": false 17 | }, 18 | { 19 | "name": "cpu_count", 20 | "enabled": true, 21 | "logical": false 22 | }, 23 | 24 | 25 | { 26 | "name": "virtual_memory", 27 | "enabled": true 28 | }, 29 | { 30 | "name": "swap_memory", 31 | "enabled": true 32 | }, 33 | 34 | 35 | { 36 | "name": "disk_partitions", 37 | "enabled": false 38 | }, 39 | { 40 | "name": "disk_usage", 41 | "enabled": true, 42 | "path": "/" 43 | }, 44 | { 45 | "name": "disk_io_counters", 46 | "enabled": true, 47 | "perDisk": false 48 | }, 49 | 50 | 51 | { 52 | "name": "net_io_counters", 53 | "enabled": true, 54 | "pernic": true 55 | }, 56 | { 57 | "name": "net_connections", 58 | "enabled": true 59 | }, 60 | 61 | 62 | { 63 | "name": "users", 64 | "enabled": true 65 | }, 66 | { 67 | "name": "boot_time", 68 | "enabled": true 69 | } 70 | ] -------------------------------------------------------------------------------- /psutil/src/main/python/psutil_producer.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import os 16 | import sys 17 | import getopt 18 | import simplejson 19 | import time 20 | 21 | from kafka.client import KafkaClient 22 | from kafka.producer import SimpleProducer, Producer 23 | import kafka.producer 24 | import kafka.client 25 | import psutil 26 | import logging 27 | 28 | 29 | class PsutilsKafkaProducer: 30 | def __init__(self, url, topic, 31 | configLocation=os.path.dirname(os.path.realpath(__file__)) + os.sep + 'config.json', 32 | reportInterval=15, 33 | async=False, 34 | sendInBatch=False, 35 | batchSize=kafka.producer.BATCH_SEND_MSG_COUNT, 36 | batchTimeout=kafka.producer.BATCH_SEND_DEFAULT_INTERVAL): 37 | logging.getLogger("kafka").addHandler(NullHandler()) 38 | 39 | self.topic = topic 40 | self.configLocation = configLocation 41 | self.reportInterval = reportInterval 42 | 43 | kafka = KafkaClient(url) 44 | self.producer = SimpleProducer(kafka, async, Producer.ACK_AFTER_LOCAL_WRITE, 45 | Producer.DEFAULT_ACK_TIMEOUT, sendInBatch, batchSize, 46 | batchTimeout) 47 | self.failures = 0 48 | self.max_failures = 3 49 | 50 | def start(self): 51 | while True and self.failures <= self.max_failures: 52 | metricsConfiguration = simplejson.load(open(self.configLocation)) 53 | report = {} 54 | for metric in metricsConfiguration: 55 | metricName = metric['name'] 56 | if metric['enabled']: 57 | del metric['name'] 58 | del metric['enabled'] 59 | arguments = metric.values() 60 | metricCallable = getattr(psutil, metricName) 61 | 62 | if type(metricCallable) is None: 63 | pass 64 | 65 | report[metricName] = apply(metricCallable, arguments) 66 | 67 | try: 68 | self.producer.send_messages(self.topic, simplejson.dumps(report)) 69 | except Exception: 70 | self.max_failures += 1 71 | 72 | time.sleep(self.reportInterval) 73 | 74 | if self.failures > self.max_failures: 75 | raise Exception("Failed to send message") 76 | 77 | def stop(self): 78 | self.producer.stop() 79 | 80 | 81 | class OptionsConfiguration: 82 | url = '' 83 | topic = '' 84 | async = False 85 | sendInBatch = False 86 | batchSize = kafka.producer.BATCH_SEND_MSG_COUNT 87 | batchTimeout = kafka.producer.BATCH_SEND_DEFAULT_INTERVAL 88 | configLocation = os.path.dirname(os.path.realpath(__file__)) + os.sep + 'config.json' 89 | reportInterval = 15 90 | 91 | def __init__(self, argv): 92 | opts, args = getopt.getopt(argv, "", 93 | ["url=", "topic=", "configLocation=", "reportInterval=", "async", "sendInBatch", 94 | "batchSize=", "batchTimeout="]) 95 | for option, arg in opts: 96 | if option == '--url': 97 | self.url = arg 98 | elif option == '--topic': 99 | self.topic = arg 100 | elif option == '--async': 101 | self.async = True 102 | elif option == '--sendInBatch': 103 | self.sendInBatch = True 104 | elif option == '--batchSize': 105 | self.batchSize = arg 106 | elif option == '--batchTimeout': 107 | self.batchTimeout = arg 108 | elif option == '--configLocation': 109 | self.configLocation = arg 110 | elif option == 'reportInterval': 111 | self.reportInterval = arg 112 | 113 | 114 | def main(argv): 115 | try: 116 | options = OptionsConfiguration(argv) 117 | producer = PsutilsKafkaProducer(options.url, options.topic, 118 | options.configLocation, 119 | options.reportInterval, 120 | options.async, 121 | options.sendInBatch, 122 | options.batchSize, 123 | options.batchTimeout) 124 | producer.start() 125 | except getopt.GetoptError, e: 126 | raise Exception(e) 127 | 128 | 129 | class NullHandler(logging.Handler): 130 | def emit(self, record): 131 | pass 132 | 133 | if __name__ == "__main__": 134 | main(sys.argv[1:]) -------------------------------------------------------------------------------- /psutil/src/main/python/psutil_producer_test.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from psutil_producer import PsutilsKafkaProducer 16 | import kafka 17 | import unittest 18 | import threading 19 | import time 20 | 21 | class PsutilsKafkaProducerTest(unittest.TestCase): 22 | 23 | def setUp(self): 24 | self.url = "localhost:9092" 25 | self.topic = "psutil-kafka-topic" 26 | 27 | self.producer = PsutilsKafkaProducer(self.url, self.topic, reportInterval=5) 28 | 29 | self.thread = threading.Thread(target=self.producer.start) 30 | self.thread.daemon = True 31 | 32 | def testStart(self): 33 | self.thread.start() 34 | time.sleep(15) 35 | self.producer.stop() 36 | 37 | message = kafka.SimpleConsumer(kafka.KafkaClient(self.url), "group1", self.topic).get_message() 38 | assert message is not None 39 | 40 | 41 | if __name__ == '__main__': 42 | unittest.main() -------------------------------------------------------------------------------- /riemann/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | kafka.logs.dir=logs 17 | 18 | log4j.rootLogger=INFO, stdout 19 | 20 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 21 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 22 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n 23 | 24 | log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender 25 | log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH 26 | log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log 27 | log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout 28 | log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 29 | 30 | log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender 31 | log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH 32 | log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log 33 | log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout 34 | log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 35 | 36 | log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender 37 | log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH 38 | log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log 39 | log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout 40 | log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 41 | 42 | log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender 43 | log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH 44 | log4j.appender.cleanerAppender.File=log-cleaner.log 45 | log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout 46 | log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 47 | 48 | log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender 49 | log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH 50 | log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log 51 | log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout 52 | log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 53 | 54 | # Turn on all our debugging info 55 | #log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender 56 | #log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender 57 | #log4j.logger.kafka.perf=DEBUG, kafkaAppender 58 | #log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender 59 | #log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG 60 | log4j.logger.kafka=INFO, kafkaAppender 61 | 62 | log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender 63 | log4j.additivity.kafka.network.RequestChannel$=false 64 | 65 | #log4j.logger.kafka.network.Processor=TRACE, requestAppender 66 | #log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender 67 | #log4j.additivity.kafka.server.KafkaApis=false 68 | log4j.logger.kafka.request.logger=WARN, requestAppender 69 | log4j.additivity.kafka.request.logger=false 70 | 71 | log4j.logger.kafka.controller=TRACE, controllerAppender 72 | log4j.additivity.kafka.controller=false 73 | 74 | log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender 75 | log4j.additivity.kafka.log.LogCleaner=false 76 | log4j.logger.kafka.log.Cleaner=INFO, cleanerAppender 77 | log4j.additivity.kafka.log.Cleaner=false 78 | 79 | log4j.logger.state.change.logger=TRACE, stateChangeAppender 80 | log4j.additivity.state.change.logger=false 81 | -------------------------------------------------------------------------------- /riemann/src/main/scala/ly/stealth/kafka/riemann/CodaHaleMetricsConsumer.scala: -------------------------------------------------------------------------------- 1 | package ly.stealth.kafka.riemann 2 | 3 | import scala.collection.JavaConversions._ 4 | import ly.stealth.kafka.metrics.KafkaMetricsReport 5 | 6 | /** 7 | * Created by ilyutov on 4/23/14. 8 | */ 9 | class CodaHaleMetricsConsumer(riemannHost: String, 10 | riemannPort: Integer, 11 | descriptionText: String, 12 | tagList: String, 13 | topic: String, 14 | groupId: String, 15 | zookeeperConnect: String, 16 | zkSessionTimeoutMs: Int = 30000, 17 | readFromStartOfStream: Boolean = true, 18 | stateMatcher: (String, Double) => String = null, 19 | defaultState: String = "info") extends RiemannMetricsConsumer(riemannHost, riemannPort, 20 | descriptionText, tagList, 21 | topic, groupId, 22 | zookeeperConnect, zkSessionTimeoutMs, 23 | readFromStartOfStream, 24 | stateMatcher,defaultState) { 25 | 26 | def transfer[T](onlyOnce: Boolean = false) = { 27 | info("reading on stream now") 28 | val it = stream.iterator() 29 | while (it.hasNext) { 30 | val messageAndTopic = it.next 31 | try { 32 | val report = mapper.readValue(messageAndTopic.message(), classOf[KafkaMetricsReport]) 33 | if (report.getCounters() != null) { 34 | for ((name, counter) <- report.getCounters()) { 35 | sendMetric("%s [%s]".format(name, "total"), counter.getCount) 36 | } 37 | } 38 | 39 | if (report.getHistograms() != null) { 40 | for ((name, histogram) <- report.getHistograms()) { 41 | sendMetric("%s [%s]".format(name, "total"), histogram.getCount) 42 | sendMetric("%s [%s]".format(name, "max"), histogram.getMax) 43 | sendMetric("%s [%s]".format(name, "min"), histogram.getMin) 44 | sendMetric("%s [%s]".format(name, "mean"), histogram.getMean) 45 | sendMetric("%s [%s]".format(name, "P50"), histogram.getP50) 46 | sendMetric("%s [%s]".format(name, "P75"), histogram.getP75) 47 | sendMetric("%s [%s]".format(name, "P95"), histogram.getP95) 48 | sendMetric("%s [%s]".format(name, "P98"), histogram.getP98) 49 | sendMetric("%s [%s]".format(name, "P99"), histogram.getP99) 50 | sendMetric("%s [%s]".format(name, "P999"), histogram.getP999) 51 | sendMetric("%s [%s]".format(name, "standard deviation"), histogram.getStddev) 52 | } 53 | } 54 | 55 | if (report.getTimers() != null) { 56 | for ((name, timer) <- report.getTimers()) { 57 | sendMetric("%s [%s(%s)]".format(name, "total duration", timer.getDuration_units), timer.getCount) 58 | sendMetric("%s [%s(%s)]".format(name, "max duration", timer.getDuration_units), timer.getMax) 59 | sendMetric("%s [%s(%s)]".format(name, "min duration", timer.getDuration_units), timer.getMin) 60 | sendMetric("%s [%s(%s)]".format(name, "mean duration", timer.getDuration_units), timer.getMean) 61 | sendMetric("%s [%s(%s)]".format(name, "P50 duration", timer.getDuration_units), timer.getP50) 62 | sendMetric("%s [%s(%s)]".format(name, "P75 duration", timer.getDuration_units), timer.getP75) 63 | sendMetric("%s [%s(%s)]".format(name, "P95 duration", timer.getDuration_units), timer.getP95) 64 | sendMetric("%s [%s(%s)]".format(name, "P98 duration", timer.getDuration_units), timer.getP98) 65 | sendMetric("%s [%s(%s)]".format(name, "P99 duration", timer.getDuration_units), timer.getP99) 66 | sendMetric("%s [%s(%s)]".format(name, "P999 duration", timer.getDuration_units), timer.getP999) 67 | 68 | sendMetric("%s [%s(%s)]".format(name, "M1 rate", timer.getRate_units), timer.getM1_rate) 69 | sendMetric("%s [%s(%s)]".format(name, "M5 rate", timer.getRate_units), timer.getM5_rate) 70 | sendMetric("%s [%s(%s)]".format(name, "M15 rate", timer.getRate_units), timer.getM15_rate) 71 | sendMetric("%s [%s(%s)]".format(name, "mean rate", timer.getRate_units), timer.getMean_rate) 72 | 73 | sendMetric("%s [%s(%s)]".format(name, "standard deviation", timer.getRate_units), timer.getStddev) 74 | } 75 | } 76 | 77 | if (report.getMeters() != null) { 78 | for ((name, meter) <- report.getMeters()) { 79 | sendMetric("%s [%s(%s)]".format(name, "total", meter.getUnits), meter.getCount) 80 | sendMetric("%s [%s(%s)]".format(name, "M1 rate", meter.getUnits), meter.getM1_rate) 81 | sendMetric("%s [%s(%s)]".format(name, "M5 rate", meter.getUnits), meter.getM5_rate) 82 | sendMetric("%s [%s(%s)]".format(name, "M15 rate", meter.getUnits), meter.getM15_rate) 83 | sendMetric("%s [%s(%s)]".format(name, "mean rate", meter.getUnits), meter.getMean_rate) 84 | } 85 | } 86 | } catch { 87 | case e: Throwable => 88 | error("Error processing message, skipping this message: ", e) 89 | throw e 90 | } finally { 91 | if (onlyOnce) close() 92 | } 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /riemann/src/main/scala/ly/stealth/kafka/riemann/PsutilMetricsConsumer.scala: -------------------------------------------------------------------------------- 1 | package ly.stealth.kafka.riemann 2 | 3 | import ly.stealth.psutil.entities.PsutilsReport 4 | 5 | class PsutilMetricsConsumer(riemannHost: String, 6 | riemannPort: Integer, 7 | descriptionText: String, 8 | tagList: String, 9 | topic: String, 10 | groupId: String, 11 | zookeeperConnect: String, 12 | zkSessionTimeoutMs: Int = 30000, 13 | readFromStartOfStream: Boolean = true, 14 | stateMatcher: (String, Double) => String = null, 15 | defaultState: String = "info") extends RiemannMetricsConsumer(riemannHost, riemannPort, 16 | descriptionText, tagList, 17 | topic, groupId, 18 | zookeeperConnect, zkSessionTimeoutMs, 19 | readFromStartOfStream, 20 | stateMatcher,defaultState) { 21 | def transfer[T](onlyOnce: Boolean) = { 22 | info("reading on stream now") 23 | val it = stream.iterator() 24 | while (it.hasNext) { 25 | val messageAndTopic = it.next 26 | try { 27 | val report = mapper.readValue(messageAndTopic.message(), classOf[PsutilsReport]) 28 | if (report.boot_time > 0) { 29 | sendMetric("boot time", report.boot_time) 30 | } 31 | if (report.cpu_count > 0) { 32 | sendMetric("cpu count", report.cpu_count, "cpu") 33 | } 34 | if (report.cpu_percent > 0) { 35 | sendMetric("cpu percent", report.cpu_percent, "cpu") 36 | } 37 | if (report.virtual_memory != null) { 38 | sendMetric("memory used %", report.virtual_memory.percent, "memory") 39 | sendMetric("memory used", report.virtual_memory.total, "memory") 40 | sendMetric("memory free", report.virtual_memory.free, "memory") 41 | sendMetric("memory used", report.virtual_memory.used, "memory") 42 | sendMetric("memory available", report.virtual_memory.available, "memory") 43 | sendMetric("memory buffers", report.virtual_memory.buffers, "memory") 44 | sendMetric("memory total", report.virtual_memory.total, "memory") 45 | sendMetric("memory inactive", report.virtual_memory.inactive, "memory") 46 | sendMetric("memory active", report.virtual_memory.active, "memory") 47 | sendMetric("memory cached", report.virtual_memory.cached, "memory") 48 | } 49 | if (report.disk_usage != null) { 50 | sendMetric("disk used %", report.disk_usage.percent, "disk") 51 | sendMetric("disk used", report.disk_usage.used, "disk") 52 | sendMetric("disk free", report.disk_usage.free, "disk") 53 | sendMetric("disk total", report.disk_usage.total, "disk") 54 | } 55 | } catch { 56 | case e: Throwable => 57 | error("Error processing message, skipping this message: ", e) 58 | throw e 59 | } finally { 60 | if (onlyOnce) close() 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /riemann/src/main/scala/ly/stealth/kafka/riemann/RiemannMetricsConsumer.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package ly.stealth.kafka.riemann 18 | 19 | import kafka.consumer.{ConsumerConfig, Consumer} 20 | import kafka.serializer._ 21 | import java.util.Properties 22 | import kafka.utils.Logging 23 | import scala.collection.JavaConversions._ 24 | import net.benmur.riemann.client._ 25 | import RiemannClient._ 26 | import akka.util.Timeout 27 | import akka.actor.ActorSystem 28 | import java.net.InetSocketAddress 29 | import com.fasterxml.jackson.databind.ObjectMapper 30 | import kafka.consumer.Whitelist 31 | 32 | abstract class RiemannMetricsConsumer(riemannHost: String, 33 | riemannPort: Integer, 34 | descriptionText: String, 35 | tagList: String, 36 | topic: String, 37 | groupId: String, 38 | zookeeperConnect: String, 39 | zkSessionTimeoutMs: Int = 30000, 40 | readFromStartOfStream: Boolean = true, 41 | stateMatcher: (String, Double) => String = null, 42 | defaultState: String = "info") extends Logging { 43 | val props = new Properties() 44 | props.put("group.id", groupId) 45 | props.put("zookeeper.connect", zookeeperConnect) 46 | props.put("auto.offset.reset", if (readFromStartOfStream) "smallest" else "largest") 47 | props.put("zookeeper.session.timeout.ms", zkSessionTimeoutMs.toString) 48 | 49 | val config = new ConsumerConfig(props) 50 | val connector = Consumer.create(config) 51 | 52 | val filterSpec = new Whitelist(topic) 53 | val mapper = new ObjectMapper(); 54 | 55 | info("Trying to start consumer: topic=%s for zk=%s and groupId=%s".format(topic, zookeeperConnect, groupId)) 56 | val stream = connector.createMessageStreamsByFilter(filterSpec, 1, new StringDecoder(), new StringDecoder()).get(0) 57 | info("Started consumer: topic=%s for zk=%s and groupId=%s".format(topic, zookeeperConnect, groupId)) 58 | 59 | implicit val system = ActorSystem() 60 | implicit val timeout = Timeout(15) 61 | val metricsDestination = riemannConnectAs[Reliable] to new InetSocketAddress(riemannHost, riemannPort) withValues (host("host") 62 | | description(descriptionText) | tags(tagList)) 63 | 64 | def transfer[T](onlyOnce: Boolean = false) 65 | 66 | protected def sendMetric(name: String, metricValue: Double, tagged: String = null) = { 67 | if (tagged == null) 68 | state(detectMetricState(name, metricValue)) | service(name) | metric(metricValue) |>> metricsDestination 69 | else 70 | state(detectMetricState(name, metricValue)) | service(name) | metric(metricValue) | tags(tagged) |>> metricsDestination 71 | } 72 | 73 | protected def detectMetricState(name: String, metricValue: Double): String = { 74 | if (stateMatcher == null) defaultState 75 | else stateMatcher(name, metricValue) 76 | } 77 | 78 | def close() { 79 | connector.shutdown() 80 | } 81 | } -------------------------------------------------------------------------------- /riemann/src/test/scala/ly/stealth/kafka/metrics/CodaHaleMetricsAndRiemannSpec.scala: -------------------------------------------------------------------------------- 1 | package ly.stealth.kafka.metrics 2 | 3 | import java.util.UUID 4 | import com.codahale.metrics.MetricRegistry 5 | import ly.stealth.kafka.riemann.CodaHaleMetricsConsumer 6 | import java.net.InetSocketAddress 7 | import net.benmur.riemann.client._ 8 | import RiemannClient._ 9 | import akka.actor.ActorSystem 10 | import akka.util.Timeout 11 | import scala.concurrent.duration.Duration 12 | import scala.concurrent.Await 13 | import org.specs2.mutable._ 14 | import kafka.utils.Logging 15 | import net.benmur.riemann.client.Query 16 | import java.util.concurrent.TimeUnit 17 | 18 | class CodaHaleMetricsAndRiemannSpec extends Specification with Logging { 19 | 20 | val zkConnection: String = "localhost:2181" 21 | val kafkaConnection: String = "localhost:9092" 22 | val riemannHost: String = "localhost" 23 | val riemannPort: Int = 5555 24 | val registry = new MetricRegistry() 25 | val metricName: String = "test_counter" 26 | val counter = registry.counter(metricName) 27 | val topic = UUID.randomUUID().toString 28 | 29 | "KafkaReporter" should { 30 | "be able to write metrics to Kafka topic" in { 31 | counter.inc(6) 32 | 33 | var success = true 34 | try { 35 | val producer = KafkaReporter.builder(registry, kafkaConnection, topic).build() 36 | producer.report() 37 | } catch { 38 | case e: Exception => success = false 39 | } 40 | 41 | success must beTrue 42 | } 43 | } 44 | 45 | "CodaHaleMetricsConsumer" should { 46 | "be able to transfer metrics from Kafka topic to Riemann" in { 47 | val groupId = UUID.randomUUID().toString 48 | var success = true 49 | try { 50 | val consumer = new CodaHaleMetricsConsumer(riemannHost, riemannPort, "basic description", "codahale", topic, 51 | groupId, zkConnection, 30000) 52 | consumer.transfer(true) 53 | } catch { 54 | case e: Exception => success = false 55 | } 56 | 57 | success must beTrue 58 | } 59 | } 60 | 61 | "Riemann users" should { 62 | "be able to query metrics, written with CodaHaleMetricsConsumer, from Riemann" in { 63 | implicit val system = ActorSystem() 64 | implicit val timeout = Timeout(30, TimeUnit.SECONDS) 65 | 66 | TimeUnit.SECONDS.sleep(15) 67 | 68 | val metricsDestination = riemannConnectAs[Reliable] to new InetSocketAddress(riemannHost, riemannPort) 69 | val future = metricsDestination ask Query("tagged \"codahale\"") 70 | Await.ready(future, Duration.Inf) 71 | val events = future.value.get.get 72 | 73 | events must not(beEmpty) 74 | } 75 | } 76 | } -------------------------------------------------------------------------------- /riemann/src/test/scala/ly/stealth/kafka/metrics/PsutilMetricsAndRiemannSpec.scala: -------------------------------------------------------------------------------- 1 | package ly.stealth.kafka.metrics 2 | 3 | import java.util.UUID 4 | import ly.stealth.kafka.riemann.PsutilMetricsConsumer 5 | import akka.actor.ActorSystem 6 | import akka.util.Timeout 7 | import net.benmur.riemann.client.RiemannClient._ 8 | import java.net.InetSocketAddress 9 | import net.benmur.riemann.client.Query 10 | import scala.concurrent.Await 11 | import scala.concurrent.duration.Duration 12 | import org.specs2.mutable.Specification 13 | import kafka.utils.Logging 14 | import java.util.concurrent.TimeUnit 15 | 16 | class PsutilMetricsAndRiemannSpec extends Specification with Logging { 17 | val zkConnection: String = "localhost:2181" 18 | val kafkaConnection: String = "localhost:9092" 19 | val riemannHost: String = "localhost" 20 | val riemannPort: Int = 5555 21 | 22 | val psutilTopic = "psutil-kafka-topic" 23 | 24 | "PsutilsMetricsConsumer" should { 25 | "be able to transfer metrics from Kafka topic to Riemann" in { 26 | val groupId = UUID.randomUUID().toString 27 | var success = true 28 | try { 29 | val consumer = new PsutilMetricsConsumer(riemannHost, riemannPort, "basic description", "kafka", psutilTopic, groupId, zkConnection, 30000) 30 | consumer.transfer(true) 31 | } catch { 32 | case e: Exception => success = false 33 | } 34 | 35 | success must beTrue 36 | } 37 | } 38 | 39 | "Riemann users" should { 40 | "be able to query metrics, written with PsutilsMetricsConsumer, from Riemann" in { 41 | implicit val system = ActorSystem() 42 | implicit val timeout = Timeout(30, TimeUnit.SECONDS) 43 | 44 | TimeUnit.SECONDS.sleep(5) 45 | 46 | val metricsDestination = riemannConnectAs[Reliable] to new InetSocketAddress(riemannHost, riemannPort) 47 | val cpuFuture = metricsDestination ask Query("tagged \"cpu\"") 48 | val memoryFuture = metricsDestination ask Query("tagged \"memory\"") 49 | val diskFuture = metricsDestination ask Query("tagged \"disk\"") 50 | 51 | Await.ready(cpuFuture, Duration.Inf) 52 | Await.ready(memoryFuture, Duration.Inf) 53 | Await.ready(diskFuture, Duration.Inf) 54 | 55 | val cpuEvents = cpuFuture.value.get.get 56 | val memoryEvents = memoryFuture.value.get.get 57 | val diskEvents = diskFuture.value.get.get 58 | 59 | cpuEvents must not(beEmpty) 60 | memoryEvents must not(beEmpty) 61 | diskEvents must not(beEmpty) 62 | } 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include ':codahale', ':riemann', ':psutil', ':metrics-test', ':yammer', ':statsd', ':graphite', ':psutil-entities', ':psutil-statsd' 2 | 3 | -------------------------------------------------------------------------------- /shutdown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | docker kill psutil 3 | docker kill riemann 4 | docker kill zkserver 5 | docker kill broker1 6 | 7 | docker rm psutil 8 | docker rm riemann 9 | docker rm broker1 10 | docker rm zkserver 11 | -------------------------------------------------------------------------------- /statsd/src/main/scala/ly/stealth/metrics/kafka/statsd/StatsD.scala: -------------------------------------------------------------------------------- 1 | package ly.stealth.metrics.kafka.statsd 2 | 3 | import java.io.IOException 4 | import java.net._ 5 | import java.nio.ByteBuffer 6 | import java.nio.channels.DatagramChannel 7 | import java.util.Random 8 | import org.slf4j.LoggerFactory 9 | import akka.actor._ 10 | 11 | /** 12 | * Client for sending stats to StatsD uses Akka to manage concurrency 13 | * 14 | * @param context The Akka ActorContext 15 | * @param host The statsd host 16 | * @param port The statsd port 17 | * @param multiMetrics If true, multiple stats will be sent in a single UDP packet 18 | * @param packetBufferSize If multiMetrics is true, this is the max buffer size before sending the UDP packet 19 | */ 20 | class StatsD(context: ActorContext = null, 21 | host: String, 22 | port: Int, 23 | multiMetrics: Boolean = true, 24 | packetBufferSize: Int = 1024) { 25 | 26 | private val rand = new Random() 27 | private var actorRef: ActorRef = null 28 | if (context != null) 29 | actorRef = context.actorOf(Props(new StatsDActor(host, port, multiMetrics, packetBufferSize))) 30 | else 31 | actorRef = ActorSystem.create("statsd").actorOf(Props(new StatsDActor(host, port, multiMetrics, packetBufferSize))) 32 | 33 | /** 34 | * Sends timing stats in milliseconds to StatsD 35 | * 36 | * @param key name of the stat 37 | * @param value time in milliseconds 38 | */ 39 | def timing(key: String, value: Long, sampleRate: Double = 1.0) = { 40 | send(key, value.toString, StatsDProtocol.TIMING_METRIC, sampleRate) 41 | } 42 | 43 | /** 44 | * Decrement StatsD counter 45 | * 46 | * @param key name of the stat 47 | * @param magnitude how much to decrement 48 | */ 49 | def decrement(key: String, magnitude: Long = -1, sampleRate: Double = 1.0) = { 50 | increment(key, magnitude, sampleRate) 51 | } 52 | 53 | /** 54 | * Increment StatsD counter 55 | * 56 | * @param key name of the stat 57 | * @param magnitude how much to increment 58 | */ 59 | def increment(key: String, magnitude: Long = 1, sampleRate: Double = 1.0) = { 60 | send(key, magnitude.toString, StatsDProtocol.COUNTER_METRIC, sampleRate) 61 | } 62 | 63 | /** 64 | * StatsD now also supports gauges, arbitrary values, which can be recorded. 65 | * 66 | * @param key name of the stat 67 | * @param value Can be a fixed value or increase or decrease (Ex: "10" "-1" "+5") 68 | */ 69 | def gauge(key: String, value: Double = 1.0, sampleRate: Double = 1.0) = { 70 | send(key, value.toString, StatsDProtocol.GAUGE_METRIC, sampleRate) 71 | } 72 | 73 | /** 74 | * StatsD supports counting unique occurrences of events between flushes, using a Set to store all occurring events. 75 | * 76 | * @param key name of the stat 77 | * @param value value of the set 78 | */ 79 | def set(key: String, value: Long, sampleRate: Double = 1.0) = { 80 | send(key, value.toString, StatsDProtocol.SET_METRIC, sampleRate) 81 | } 82 | 83 | /** 84 | * Checks the sample rate and sends the stat to the actor if it passes 85 | */ 86 | private def send(key: String, value: String, metric: String, sampleRate: Double): Boolean = { 87 | if (sampleRate >= 1 || rand.nextDouble <= sampleRate) { 88 | actorRef ! SendStat(StatsDProtocol.stat(key, value, metric, sampleRate)) 89 | true 90 | } 91 | else { 92 | false 93 | } 94 | } 95 | } 96 | 97 | object StatsDProtocol { 98 | val TIMING_METRIC = "ms" 99 | val COUNTER_METRIC = "c" 100 | val GAUGE_METRIC = "g" 101 | val SET_METRIC = "s" 102 | 103 | /** 104 | * @return Returns a string that conforms to the StatsD protocol: 105 | * KEY:VALUE|METRIC or KEY:VALUE|METRIC|@SAMPLE_RATE 106 | */ 107 | def stat(key: String, value: String, metric: String, sampleRate: Double) = { 108 | val sampleRateString = if (sampleRate < 1) "|@" + sampleRate else "" 109 | key + ":" + value + "|" + metric + sampleRateString 110 | } 111 | } 112 | 113 | /** 114 | * Message for the StatsDActor 115 | */ 116 | private case class SendStat(stat: String) 117 | 118 | /** 119 | * @param host The statsd host 120 | * @param port The statsd port 121 | * @param multiMetrics If true, multiple stats will be sent in a single UDP packet 122 | * @param packetBufferSize If multiMetrics is true, this is the max buffer size before sending the UDP packet 123 | */ 124 | private class StatsDActor(host: String, 125 | port: Int, 126 | multiMetrics: Boolean, 127 | packetBufferSize: Int) extends Actor { 128 | 129 | private val log = LoggerFactory.getLogger(getClass()) 130 | 131 | private val sendBuffer = ByteBuffer.allocate(packetBufferSize) 132 | 133 | private val address = new InetSocketAddress(InetAddress.getByName(host), port) 134 | private val channel = DatagramChannel.open() 135 | 136 | def receive = { 137 | case msg: SendStat => doSend(msg.stat) 138 | case _ => log.error("Unknown message") 139 | } 140 | 141 | override def postStop() = { 142 | //save any remaining data to StatsD 143 | flush 144 | 145 | //Close the channel 146 | if (channel.isOpen()) { 147 | channel.close() 148 | } 149 | 150 | sendBuffer.clear() 151 | } 152 | 153 | private def doSend(stat: String) = { 154 | try { 155 | val data = stat.getBytes("utf-8") 156 | 157 | // If we're going to go past the threshold of the buffer then flush. 158 | // the +1 is for the potential '\n' in multi_metrics below 159 | if (sendBuffer.remaining() < (data.length + 1)) { 160 | flush 161 | } 162 | 163 | // multiple metrics are separated by '\n' 164 | if (sendBuffer.position() > 0) { 165 | sendBuffer.put('\n'.asInstanceOf[Byte]) 166 | } 167 | 168 | // append the data 169 | sendBuffer.put(data) 170 | 171 | if (!multiMetrics) { 172 | flush 173 | } 174 | 175 | } 176 | catch { 177 | case e: IOException => { 178 | log.error("Could not send stat {} to host {}:{}", sendBuffer.toString, address.getHostName(), address.getPort().toString, e) 179 | } 180 | } 181 | } 182 | 183 | private def flush(): Unit = { 184 | try { 185 | val sizeOfBuffer = sendBuffer.position() 186 | 187 | if (sizeOfBuffer <= 0) { 188 | // empty buffer 189 | return 190 | } 191 | 192 | // send and reset the buffer 193 | sendBuffer.flip() 194 | val nbSentBytes = channel.send(sendBuffer, address) 195 | sendBuffer.limit(sendBuffer.capacity()) 196 | sendBuffer.rewind() 197 | 198 | if (sizeOfBuffer != nbSentBytes) { 199 | log.error("Could not send entirely stat {} to host {}:{}. Only sent {} bytes out of {} bytes", sendBuffer.toString(), 200 | address.getHostName(), address.getPort().toString, nbSentBytes.toString, sizeOfBuffer.toString) 201 | } 202 | 203 | } 204 | catch { 205 | case e: IOException => { 206 | log.error("Could not send stat {} to host {}:{}", sendBuffer.toString, address.getHostName(), address.getPort().toString, e) 207 | } 208 | } 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /vagrant/README.md: -------------------------------------------------------------------------------- 1 | # Apache Kafka # 2 | 3 | Using Vagrant to get up and running. 4 | 5 | 1) Install Vagrant [http://www.vagrantup.com/](http://www.vagrantup.com/) 6 | 2) Install Virtual Box [https://www.virtualbox.org/](https://www.virtualbox.org/) 7 | 3) gradle :psutil:installDependencies to install psutil dependencies 8 | 9 | In the main kafka folder 10 | 11 | 1) vagrant up 12 | 13 | once this is done 14 | * Zookeeper will be running 192.168.50.5 15 | * Broker 1 on 192.168.50.10 16 | * Broker 2 on 192.168.50.20 17 | * Broker 3 on 192.168.50.30 18 | 19 | When you are all up and running you will be back at a command brompt. 20 | 21 | If you want you can login to the machines using vagrant ssh but you don't need to. 22 | 23 | You can access the brokers and zookeeper by their IP 24 | 25 | e.g. 26 | 27 | bin/kafka-console-producer.sh --broker-list 192.168.50.10:9092,192.168.50.20:9092,192.168.50.30:9092 --topic sandbox 28 | 29 | bin/kafka-console-consumer.sh --zookeeper 192.168.50.5:2181 --topic sandbox --from-beginning 30 | -------------------------------------------------------------------------------- /vagrant/broker.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | #!/bin/sh -Eux 17 | 18 | # Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR 19 | trap founderror 1 2 3 15 ERR 20 | 21 | founderror() 22 | { 23 | exit 1 24 | } 25 | 26 | exitscript() 27 | { 28 | #remove lock file 29 | #rm $lockfile 30 | exit 0 31 | } 32 | 33 | apt-get -y update 34 | apt-get install -y software-properties-common python-software-properties 35 | add-apt-repository -y ppa:webupd8team/java 36 | apt-get -y update 37 | /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections 38 | apt-get -y install oracle-java7-installer oracle-java7-set-default 39 | 40 | chmod a+rw /opt 41 | cd /opt 42 | ln -s /vagrant kafka 43 | cd kafka 44 | IP=$(ifconfig | grep 'inet addr:'| grep 168 | grep 192|cut -d: -f2 | awk '{ print $1}') 45 | sed 's/broker.id=0/'broker.id=$1'/' /opt/kafka/config/server.properties > /tmp/prop1.tmp 46 | sed 's/#advertised.host.name=/'advertised.host.name=$IP'/' /tmp/prop1.tmp > /tmp/prop2.tmp 47 | sed 's/#host.name=localhost/'host.name=$IP'/' /tmp/prop2.tmp > /tmp/prop3.tmp 48 | sed 's/zookeeper.connect=localhost:2181/'zookeeper.connect=192.168.86.5:2181'/' /tmp/prop3.tmp > /opt/server.properties 49 | 50 | /vagrant/vagrant/kafka.sh #install kafka 51 | 52 | /opt/apache/kafka/bin/kafka-server-start.sh /opt/server.properties 1>> /tmp/broker.log 2>> /tmp/broker.log & 53 | 54 | /vagrant/vagrant/psutil.sh 55 | 56 | exitscript -------------------------------------------------------------------------------- /vagrant/kafka.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | #!/bin/sh -Eux 17 | 18 | # Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR 19 | trap founderror 1 2 3 15 ERR 20 | 21 | founderror() 22 | { 23 | exit 1 24 | } 25 | 26 | exitscript() 27 | { 28 | #remove lock file 29 | #rm $lockfile 30 | exit 0 31 | } 32 | 33 | mkdir -p /opt/apache 34 | cd /opt/apache 35 | version=0.8.1 36 | scala=2.10 37 | release=kafka_$scala-$version 38 | 39 | url=archive.apache.org/dist/kafka 40 | wget https://$url/$version/$release.tgz 41 | wget https://$url/$version/$release.tgz.md5 42 | wget https://$url/$version/$release.tgz.sh1 43 | wget https://$url/$version/$release.tgz.sh2 44 | wget https://$url/$version/$release.tgz.asc 45 | tar -xvf $release.tgz 46 | /vagrant/vagrant/verify.sh $release.tgz 47 | ln -s /opt/apache/$release kafka 48 | exitscript -------------------------------------------------------------------------------- /vagrant/provision.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | #!/bin/sh -Eux 17 | 18 | # Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR 19 | trap founderror 1 2 3 15 ERR 20 | 21 | founderror() 22 | { 23 | exit 1 24 | } 25 | 26 | exitscript() 27 | { 28 | #remove lock file 29 | #rm $lockfile 30 | exit 0 31 | } 32 | 33 | apt-get -y update 34 | apt-get install -y software-properties-common python-software-properties 35 | add-apt-repository -y ppa:webupd8team/java 36 | apt-get -y update 37 | /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections 38 | apt-get -y install oracle-java7-installer oracle-java7-set-default 39 | 40 | exitscript -------------------------------------------------------------------------------- /vagrant/psutil.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | #!/bin/sh -Eux 17 | 18 | # Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR 19 | trap founderror 1 2 3 15 ERR 20 | 21 | founderror() 22 | { 23 | exit 1 24 | } 25 | 26 | exitscript() 27 | { 28 | #remove lock file 29 | #rm $lockfile 30 | exit 0 31 | } 32 | 33 | apt-get install -y python-dev 34 | apt-get install -y python-pip 35 | apt-get install -y git 36 | 37 | pip install psutil 38 | pip install simplejson 39 | 40 | git clone https://github.com/mumrah/kafka-python 41 | pip install ./kafka-python 42 | 43 | exitscript -------------------------------------------------------------------------------- /vagrant/riemann.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | #!/bin/sh -Eux 17 | 18 | # Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR 19 | trap founderror 1 2 3 15 ERR 20 | 21 | founderror() 22 | { 23 | exit 1 24 | } 25 | 26 | exitscript() 27 | { 28 | #remove lock file 29 | #rm $lockfile 30 | exit 0 31 | } 32 | 33 | apt-get -y update 34 | apt-get install -y software-properties-common python-software-properties 35 | add-apt-repository -y ppa:webupd8team/java 36 | apt-get -y update 37 | /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections 38 | apt-get -y install oracle-java7-installer oracle-java7-set-default 39 | 40 | chown -R vagrant /opt 41 | 42 | cd /opt 43 | 44 | wget http://aphyr.com/riemann/riemann-0.2.4.tar.bz2 45 | tar xvfj riemann-0.2.4.tar.bz2 46 | cd riemann-0.2.4 47 | 48 | /opt/riemann-0.2.4/bin/riemann /vagrant/config/riemann.config >> /tmp/riemann.log & 49 | 50 | apt-add-repository -y ppa:brightbox/ruby-ng 51 | apt-get -y update 52 | 53 | apt-get -y install rubygems 54 | apt-get -y install ruby1.9.3 55 | 56 | gem1.9.3 install riemann-client riemann-tools riemann-dash 57 | 58 | chown -R vagrant /var/lib/gems/ 59 | 60 | riemann-dash /vagrant/config/config.rb >> /tmp/riemann-dash.log & 61 | 62 | /vagrant/vagrant/psutil.sh 63 | 64 | python /vagrant/psutil/src/main/python/psutil_producer.py --url 192.168.86.5:9092 --reportInterval 5 --topic psutil-metrics --configLocation /vagrant/psutil/src/main/python/config.json & 65 | 66 | #Sometimes Kafka refuses to create topic, so let's try once more 67 | python /vagrant/psutil/src/main/python/psutil_producer.py --url 192.168.86.5:9092 --reportInterval 5 --topic psutil-metrics --configLocation /vagrant/psutil/src/main/python/config.json & 68 | 69 | exitscript -------------------------------------------------------------------------------- /vagrant/verify.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | #!/bin/sh -Eux 17 | 18 | # Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR 19 | trap founderror 1 2 3 15 ERR 20 | 21 | founderror() 22 | { 23 | exit 1 24 | } 25 | 26 | exitscript() 27 | { 28 | #remove lock file 29 | #rm $lockfile 30 | exit 0 31 | } 32 | 33 | release=$1 34 | 35 | #gpg --verify $release.asc $release 36 | 37 | /vagrant/vagrant/verify_hash.sh $release MD5 md5 38 | /vagrant/vagrant/verify_hash.sh $release SHA1 sh1 39 | /vagrant/vagrant/verify_hash.sh $release SHA256 sh2 40 | 41 | exitscript -------------------------------------------------------------------------------- /vagrant/verify_hash.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | #!/bin/sh -Eux 17 | 18 | # Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR 19 | trap founderror 1 2 3 15 ERR 20 | 21 | founderror() 22 | { 23 | exit 1 24 | } 25 | 26 | exitscript() 27 | { 28 | #remove lock file 29 | #rm $lockfile 30 | exit 0 31 | } 32 | 33 | hs=$(gpg --print-md $2 $1) 34 | hf=$(cat $1.$3) 35 | if [ "$hs" == "$hf" ]; then 36 | echo "$2 ok" 37 | else 38 | echo "error $hs != $hf" 39 | founderror 40 | fi 41 | 42 | exitscript -------------------------------------------------------------------------------- /vagrant/zk.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | #!/bin/sh -Eux 17 | 18 | # Trap non-normal exit signals: 1/HUP, 2/INT, 3/QUIT, 15/TERM, ERR 19 | trap founderror 1 2 3 15 ERR 20 | 21 | founderror() 22 | { 23 | exit 1 24 | } 25 | 26 | exitscript() 27 | { 28 | #remove lock file 29 | #rm $lockfile 30 | exit 0 31 | } 32 | 33 | apt-get -y update 34 | apt-get install -y software-properties-common python-software-properties 35 | add-apt-repository -y ppa:webupd8team/java 36 | apt-get -y update 37 | /bin/echo debconf shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections 38 | apt-get -y install oracle-java7-installer oracle-java7-set-default 39 | 40 | /vagrant/vagrant/kafka.sh #install kafka 41 | 42 | /opt/apache/kafka/bin/zookeeper-server-start.sh /opt/apache/kafka/config/zookeeper.properties 1>> /tmp/zk.log 2>> /tmp/zk.log & 43 | 44 | exitscript -------------------------------------------------------------------------------- /yammer/src/main/java/ly/stealth/kafka/metrics/KafkaBrokerReporter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package ly.stealth.kafka.metrics; 19 | 20 | import com.yammer.metrics.Metrics; 21 | import kafka.metrics.KafkaMetricsConfig; 22 | import kafka.metrics.KafkaMetricsReporter; 23 | import kafka.metrics.KafkaMetricsReporterMBean; 24 | import kafka.producer.ProducerConfig; 25 | import kafka.utils.VerifiableProperties; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.util.concurrent.TimeUnit; 30 | 31 | interface KafkaBrokerReporterMBean extends KafkaMetricsReporterMBean {} 32 | 33 | public class KafkaBrokerReporter implements KafkaMetricsReporter, KafkaBrokerReporterMBean { 34 | private static final Logger log = LoggerFactory.getLogger(KafkaBrokerReporter.class); 35 | private TopicReporter underlying; 36 | private VerifiableProperties props; 37 | private boolean running; 38 | private boolean initialized; 39 | 40 | @Override 41 | public String getMBeanName() { 42 | return "kafka:type=ly.stealth.kafka.metrics.KafkaBrokerReporter"; 43 | } 44 | 45 | synchronized public void init(VerifiableProperties props) { 46 | if (!initialized) { 47 | this.props = props; 48 | props.props().put("metadata.broker.list", String.format("%s:%d", "localhost", props.getInt("port"))); 49 | 50 | final KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props); 51 | 52 | this.underlying = new TopicReporter(Metrics.defaultRegistry(), 53 | new ProducerConfig(props.props()), 54 | "broker%s".format(props.getString("broker.id"))); 55 | initialized = true; 56 | startReporter(metricsConfig.pollingIntervalSecs()); 57 | } 58 | } 59 | 60 | synchronized public void startReporter(long pollingPeriodSecs) { 61 | if (initialized && !running) { 62 | underlying.start(pollingPeriodSecs, TimeUnit.SECONDS); 63 | running = true; 64 | log.info(String.format("Started Kafka Topic metrics reporter with polling period %d seconds", pollingPeriodSecs)); 65 | } 66 | } 67 | 68 | public synchronized void stopReporter() { 69 | if (initialized && running) { 70 | underlying.shutdown(); 71 | running = false; 72 | log.info("Stopped Kafka Topic metrics reporter"); 73 | underlying = new TopicReporter(Metrics.defaultRegistry(), new ProducerConfig(props.props()), String.format("broker%s", props.getString("broker.id"))); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /yammer/src/main/java/ly/stealth/kafka/metrics/TopicReporter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package ly.stealth.kafka.metrics; 19 | 20 | import com.yammer.metrics.core.*; 21 | import com.yammer.metrics.reporting.AbstractPollingReporter; 22 | import com.yammer.metrics.stats.Snapshot; 23 | import kafka.javaapi.producer.Producer; 24 | import kafka.producer.KeyedMessage; 25 | import kafka.producer.ProducerConfig; 26 | 27 | import java.io.UnsupportedEncodingException; 28 | import java.util.HashMap; 29 | import java.util.Map; 30 | import java.util.Set; 31 | import java.util.concurrent.TimeUnit; 32 | 33 | import static java.lang.String.format; 34 | import static java.lang.String.valueOf; 35 | 36 | public class TopicReporter extends AbstractPollingReporter implements MetricProcessor { 37 | private final Map producerMap = new HashMap(); 38 | private final MetricPredicate predicate = MetricPredicate.ALL; 39 | private final Clock clock = Clock.defaultClock(); 40 | private final ProducerConfig producerConfig; 41 | private final String prefix; 42 | 43 | private Long startTime = 0L; 44 | 45 | public TopicReporter(MetricsRegistry metricsRegistry, ProducerConfig producerConfig, String prefix) { 46 | super(metricsRegistry, "kafka-topic-reporter"); 47 | this.producerConfig = producerConfig; 48 | this.prefix = prefix; 49 | } 50 | 51 | public void run() { 52 | final Set> metrics=getMetricsRegistry().allMetrics().entrySet(); 53 | try { 54 | for (Map.Entry entry : metrics) { 55 | final MetricName metricName = entry.getKey(); 56 | final Metric metric = entry.getValue(); 57 | if (predicate.matches(metricName, metric)) { 58 | final Context context = new Context() { 59 | public Producer getProducer() { 60 | return getActualProducer(metricName); 61 | } 62 | }; 63 | metric.processWith(this, entry.getKey(), context); 64 | } 65 | } 66 | } catch(Exception e) { 67 | e.printStackTrace(); 68 | } 69 | } 70 | 71 | public void processMeter(MetricName name, Metered meter, Context context) { 72 | final String header = "# time,count,1 min rate,mean rate,5 min rate,15 min rate"; 73 | final Producer producer = context.getProducer(); 74 | final String topic = "%s-metrics-meter".format(prefix); 75 | final String message = valueOf(meter.count()) + ',' + meter.oneMinuteRate() + ',' + meter.meanRate() + ',' 76 | + meter.fiveMinuteRate() + ',' + meter.fifteenMinuteRate(); 77 | send(producer, header, topic, message); 78 | } 79 | 80 | public void processCounter(MetricName name, Counter counter, Context context) { 81 | final String header = "# time,count"; 82 | final Producer producer = context.getProducer(); 83 | final String topic = "%s-metrics-counter".format(prefix); 84 | final String message = valueOf(counter.count()); 85 | send(producer, header, topic, message); 86 | } 87 | 88 | public void processHistogram(MetricName name, Histogram histogram, Context context) { 89 | final String header = "# time,min,max,mean,median,stddev,95%,99%,99.9%"; 90 | final Producer producer = context.getProducer(); 91 | final Snapshot snapshot = histogram.getSnapshot(); 92 | final String topic="%s-metrics-histogram".format(prefix); 93 | final String message = valueOf(histogram.min()) + ',' + histogram.max() + ',' + histogram.mean() + ',' 94 | + snapshot.getMedian() + ',' + histogram.stdDev() + ',' + snapshot.get95thPercentile() + ',' + snapshot.get99thPercentile() + ',' 95 | + snapshot.get999thPercentile(); 96 | send(producer, header, topic, message); 97 | } 98 | 99 | public void processTimer(MetricName name, Timer timer, Context context) { 100 | final String header = "# time,min,max,mean,median,stddev,95%,99%,99.9%"; 101 | final Producer producer = context.getProducer(); 102 | final Snapshot snapshot = timer.getSnapshot(); 103 | final String topic="%s-metrics-timer".format(prefix); 104 | final String message = valueOf(timer.min()) + ',' + timer.max() + ',' + timer.mean() + ',' + snapshot.getMedian() + ',' 105 | + timer.stdDev() + ',' + snapshot.get95thPercentile() + ',' + snapshot.get99thPercentile() + ',' + snapshot.get999thPercentile(); 106 | send(producer, header, topic, message); 107 | } 108 | 109 | public void processGauge(MetricName name, Gauge gauge, Context context){ 110 | final String header = "# time,finalue"; 111 | final Producer producer = context.getProducer(); 112 | final String topic = "%s-metrics-gauge".format(prefix); 113 | final String message = gauge.value().toString(); 114 | send(producer, header, topic, message); 115 | } 116 | 117 | @Override 118 | public void start(long period, TimeUnit unit) { 119 | this.startTime = clock.time(); 120 | super.start(period, unit); 121 | } 122 | 123 | @Override 124 | public void shutdown() { 125 | try { 126 | super.shutdown(); 127 | } finally { 128 | for (Producer producer : producerMap.values()) { 129 | producer.close(); 130 | } 131 | } 132 | } 133 | 134 | private Producer getActualProducer(MetricName metricName) { 135 | Producer producer; 136 | synchronized(producerMap) { 137 | producer = producerMap.get(metricName); 138 | if (producer == null) { 139 | producer = new Producer(producerConfig); 140 | producerMap.put(metricName, producer); 141 | } 142 | } 143 | return producer; 144 | } 145 | 146 | private void send(Producer producer,String header, String topic, String message) { 147 | final Long time = TimeUnit.MILLISECONDS.toSeconds(clock.time() - startTime); 148 | try { 149 | producer.send(new KeyedMessage(topic, format("%s\n%d,%s", header, time, message).getBytes("UTF-8"))); 150 | } catch (UnsupportedEncodingException e) { 151 | throw new RuntimeException(e); 152 | } 153 | } 154 | } 155 | 156 | interface Context{ 157 | public Producer getProducer(); 158 | } --------------------------------------------------------------------------------