├── .gitignore ├── src └── main │ └── java │ └── com │ └── criteo │ └── kafka │ ├── KafkaGraphiteMetricsReporterMBean.java │ ├── RegexMetricPredicate.java │ ├── KafkaGraphiteMetricsReporter.java │ └── GraphiteReporter.java ├── README.md ├── kafka-graphite.iml ├── pom.xml └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | pom.xml.tag 3 | pom.xml.releaseBackup 4 | pom.xml.next 5 | release.properties 6 | -------------------------------------------------------------------------------- /src/main/java/com/criteo/kafka/KafkaGraphiteMetricsReporterMBean.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Damien Claveau 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | 19 | package com.criteo.kafka; 20 | 21 | import kafka.metrics.KafkaMetricsReporterMBean; 22 | 23 | public interface KafkaGraphiteMetricsReporterMBean extends 24 | KafkaMetricsReporterMBean { 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/criteo/kafka/RegexMetricPredicate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Damien Claveau 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | 19 | package com.criteo.kafka; 20 | 21 | import java.util.regex.Pattern; 22 | 23 | import org.apache.log4j.Logger; 24 | 25 | import com.yammer.metrics.core.Metric; 26 | import com.yammer.metrics.core.MetricName; 27 | import com.yammer.metrics.core.MetricPredicate; 28 | 29 | public class RegexMetricPredicate implements MetricPredicate { 30 | 31 | Pattern pattern = null; 32 | //static Logger LOG = Logger.getLogger(RegexMetricPredicate.class); 33 | 34 | public RegexMetricPredicate(String regex) { 35 | pattern = Pattern.compile(regex); 36 | } 37 | 38 | @Override 39 | public boolean matches(MetricName name, Metric metric) { 40 | boolean ok = pattern.matcher(name.getMBeanName()).matches(); 41 | //LOG(String.format("name: %s - %s", name.getMBeanName(), ok)); 42 | return ok; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Kafka Graphite Metrics Reporter 2 | ============================== 3 | 4 | This is a simple reporter for kafka using the 5 | [GraphiteReporter](http://metrics.codahale.com/manual/graphite/). It works with 6 | kafka 0.8.1.1 version. 7 | 8 | Big thanks to Maxime Brugidou from Criteo who did the initial commit of the Ganglia version, 9 | available here https://github.com/criteo/kafka-ganglia 10 | 11 | Install On Broker 12 | ------------ 13 | 14 | 1. Build the `kafka-graphite-1.1.6.jar` jar using `mvn package`. 15 | 2. Add `kafka-graphite-1.1.6.jar` and `metrics-graphite-2.2.0.jar` to the `libs/` 16 | directory of your kafka broker installation 17 | 3. Configure the broker (see the configuration section below) 18 | 4. Restart the broker 19 | 20 | Configuration 21 | ------------ 22 | 23 | Edit the `server.properties` file of your installation, activate the reporter by setting: 24 | 25 | kafka.metrics.reporters=com.criteo.kafka.KafkaGraphiteMetricsReporter[,kafka.metrics.KafkaCSVMetricsReporter[,....]] 26 | kafka.graphite.metrics.reporter.enabled=true 27 | 28 | Here is a list of default properties used: 29 | 30 | kafka.graphite.metrics.host=localhost 31 | kafka.graphite.metrics.port=8649 32 | kafka.graphite.metrics.group=kafka 33 | # This can be use to filter some metrics from graphite 34 | # since kafka has quite a lot of metrics, it is useful 35 | # if you have many topics/partitions. 36 | # Only metrics matching with the pattern will be written to graphite. 37 | kafka.graphite.metrics.filter.regex="kafka.server":type="BrokerTopicMetrics",.* 38 | 39 | Usage As Lib 40 | ----------- 41 | 42 | Simply build the jar and publish it to your maven internal repository (this 43 | package is not published to any public repositories unfortunately). 44 | -------------------------------------------------------------------------------- /kafka-graphite.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/main/java/com/criteo/kafka/KafkaGraphiteMetricsReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Damien Claveau 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | 19 | package com.criteo.kafka; 20 | 21 | import java.io.IOException; 22 | import java.util.concurrent.ScheduledExecutorService; 23 | import java.util.concurrent.TimeUnit; 24 | 25 | import kafka.metrics.KafkaMetricsConfig; 26 | import kafka.metrics.KafkaMetricsReporter; 27 | import kafka.utils.VerifiableProperties; 28 | 29 | import com.yammer.metrics.Metrics; 30 | import com.yammer.metrics.core.Clock; 31 | import com.yammer.metrics.core.MetricPredicate; 32 | import org.apache.log4j.*; 33 | 34 | public class KafkaGraphiteMetricsReporter implements KafkaMetricsReporter, 35 | KafkaGraphiteMetricsReporterMBean { 36 | 37 | static Logger LOG = Logger.getLogger(KafkaGraphiteMetricsReporter.class); 38 | static String GRAPHITE_DEFAULT_HOST = "localhost"; 39 | static int GRAPHITE_DEFAULT_PORT = 2003; 40 | static String GRAPHITE_DEFAULT_PREFIX = "kafka"; 41 | static boolean HIDE_METER_MEANS_DEFAULT = false; // Export all metrics by default 42 | 43 | String graphiteHost = GRAPHITE_DEFAULT_HOST; 44 | int graphitePort = GRAPHITE_DEFAULT_PORT; 45 | String graphiteGroupPrefix = GRAPHITE_DEFAULT_PREFIX; 46 | boolean hideMetersMeans = HIDE_METER_MEANS_DEFAULT; 47 | 48 | GraphiteReporter reporter = null; 49 | MetricPredicate predicate = MetricPredicate.ALL; 50 | private ScheduledExecutorService executor; 51 | 52 | boolean initialized = false; 53 | boolean running = false; 54 | 55 | @Override 56 | public String getMBeanName() { 57 | return "kafka:type=com.criteo.kafka.KafkaGraphiteMetricsReporter"; 58 | } 59 | 60 | @Override 61 | public synchronized void startReporter(long pollingPeriodSecs) { 62 | if (initialized && !running) { 63 | executor.scheduleAtFixedRate(reporter, pollingPeriodSecs, pollingPeriodSecs, TimeUnit.SECONDS); 64 | running = true; 65 | LOG.info(String.format("Started Kafka Graphite metrics reporter with polling period %d seconds", pollingPeriodSecs)); 66 | } 67 | } 68 | 69 | @Override 70 | public synchronized void stopReporter() { 71 | if (initialized && running) { 72 | executor.shutdown(); 73 | running = false; 74 | LOG.info("Stopped Kafka Graphite metrics reporter"); 75 | reporter = createReporter(graphiteHost, graphitePort, graphiteGroupPrefix, predicate, hideMetersMeans); 76 | } 77 | } 78 | 79 | @Override 80 | public synchronized void init(VerifiableProperties props) { 81 | if (!initialized) { 82 | KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props); 83 | graphiteHost = props.getString("kafka.graphite.metrics.host", GRAPHITE_DEFAULT_HOST); 84 | graphitePort = props.getInt("kafka.graphite.metrics.port", GRAPHITE_DEFAULT_PORT); 85 | graphiteGroupPrefix = props.getString("kafka.graphite.metrics.group", GRAPHITE_DEFAULT_PREFIX); 86 | String regex = props.getString("kafka.graphite.metrics.filter.regex", null); 87 | String logDir = props.getString("kafka.graphite.metrics.logFile", null); 88 | hideMetersMeans = props.getBoolean("kafka.graphite.metrics.hideMetersMeans", false); 89 | 90 | if (logDir != null) { 91 | setLogFile(logDir); 92 | } 93 | 94 | LOG.debug("Initialize GraphiteReporter ["+graphiteHost+","+graphitePort+","+graphiteGroupPrefix+"]"); 95 | 96 | executor = Metrics.defaultRegistry().newScheduledThreadPool(1, "MetricReporter"); 97 | 98 | if (regex != null) 99 | predicate = new RegexMetricPredicate(regex); 100 | else 101 | predicate = MetricPredicate.ALL; 102 | 103 | reporter = createReporter(graphiteHost, graphitePort, graphiteGroupPrefix, predicate, hideMetersMeans); 104 | 105 | if (props.getBoolean("kafka.graphite.metrics.reporter.enabled", false)) { 106 | initialized = true; 107 | startReporter(metricsConfig.pollingIntervalSecs()); 108 | LOG.debug("GraphiteReporter started."); 109 | } 110 | } 111 | } 112 | 113 | private GraphiteReporter createReporter(String graphiteHost, int graphitePort, String graphiteGroupPrefix, 114 | MetricPredicate predicate, boolean hideMetersMeans){ 115 | GraphiteReporter reporter = null; 116 | 117 | try { 118 | reporter = new GraphiteReporter( 119 | Metrics.defaultRegistry(), 120 | graphiteGroupPrefix, 121 | predicate, 122 | new GraphiteReporter.DefaultSocketProvider(graphiteHost, graphitePort), 123 | Clock.defaultClock()); 124 | reporter.setHideMetersMeans(hideMetersMeans); 125 | } catch (IOException e) { 126 | LOG.error("Unable to initialize GraphiteReporter", e); 127 | } 128 | 129 | return reporter; 130 | } 131 | 132 | private void setLogFile(String logDir){ 133 | RollingFileAppender fa = new RollingFileAppender(); 134 | fa.setName("GraphiteReporter"); 135 | fa.setFile(logDir); 136 | fa.setLayout(new PatternLayout("[%d] %-4r [%t] %-5p %c %x - %m%n")); 137 | fa.setThreshold(Level.INFO); 138 | fa.setMaxFileSize("256MB"); 139 | fa.setMaxBackupIndex(2); 140 | fa.setAppend(true); 141 | fa.activateOptions(); 142 | 143 | LogManager.getLogger("com.criteo").removeAllAppenders(); 144 | LogManager.getLogger("com.criteo").addAppender(fa); 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 4.0.0 7 | com.criteo.kafka 8 | kafka-graphite 9 | jar 10 | 1.1.6 11 | Kafka metrics output to Graphite 12 | http://maven.criteo 13 | 14 | 15 | 3.0.4 16 | 17 | 18 | 19 | 20 | org.apache.kafka 21 | kafka_2.10 22 | 0.8.1.1 23 | 24 | 25 | com.yammer.metrics 26 | metrics-graphite 27 | 2.2.0 28 | 29 | 30 | 31 | 32 | org.apache.zookeeper 33 | zookeeper 34 | 3.3.4 35 | 36 | 37 | log4j 38 | log4j 39 | 40 | 41 | jline 42 | jline 43 | 44 | 45 | 46 | 47 | org.scala-lang 48 | scala-library 49 | 2.9.2 50 | 51 | 52 | log4j 53 | log4j 54 | 1.2.15 55 | 56 | 57 | javax.jms 58 | jms 59 | 60 | 61 | com.sun.jmx 62 | jmxri 63 | 64 | 65 | com.sun.jdmk 66 | jmxtools 67 | 68 | 69 | 70 | 71 | net.sf.jopt-simple 72 | jopt-simple 73 | 3.2 74 | 75 | 76 | org.slf4j 77 | slf4j-simple 78 | 1.6.4 79 | 80 | 81 | org.scala-lang 82 | scala-compiler 83 | 2.9.2 84 | 85 | 86 | com.101tec 87 | zkclient 88 | 0.3 89 | 90 | 91 | org.xerial.snappy 92 | snappy-java 93 | 1.0.4.1 94 | 95 | 96 | com.yammer.metrics 97 | metrics-core 98 | 2.2.0 99 | 100 | 101 | com.yammer.metrics 102 | metrics-annotation 103 | 2.2.0 104 | 105 | 106 | org.easymock 107 | easymock 108 | 3.0 109 | test 110 | 111 | 112 | junit 113 | junit 114 | 4.11 115 | 116 | 117 | org.scalatest 118 | scalatest_2.9.2 119 | 1.8 120 | test 121 | 122 | 123 | 124 | 125 | 126 | criteo.releases 127 | Criteo Releases 128 | http://maven/content/repositories/criteo.releases 129 | 130 | 131 | criteo.snapshots 132 | Criteo Snapshots 133 | http://maven/content/repositories/criteo.snapshots 134 | 135 | 136 | 137 | 138 | UTF-8 139 | uber 140 | 141 | 142 | 143 | 144 | 145 | org.apache.maven.plugins 146 | maven-compiler-plugin 147 | 148 | 1.6 149 | 1.6 150 | 151 | 152 | 153 | 154 | 155 | org.apache.maven.plugins 156 | maven-eclipse-plugin 157 | 2.9 158 | 159 | true 160 | true 161 | 162 | 163 | 164 | 165 | 166 | org.apache.maven.plugins 167 | maven-source-plugin 168 | 2.2.1 169 | 170 | 171 | attach-sources 172 | 173 | jar 174 | 175 | 176 | 177 | 178 | 179 | org.apache.maven.plugins 180 | maven-javadoc-plugin 181 | 2.9 182 | 183 | 184 | org.umlgraph.doclet.UmlGraphDoc 185 | 186 | org.umlgraph 187 | doclet 188 | 5.1 189 | 190 | -views 191 | true 192 | 193 | 194 | 195 | attach-javadocs 196 | 197 | jar 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | org.apache.maven.plugins 207 | maven-shade-plugin 208 | 2.0 209 | 210 | 211 | package 212 | 213 | shade 214 | 215 | 216 | false 217 | true 218 | ${shaded.name} 219 | 220 | 221 | *:* 222 | 223 | 224 | true 225 | 226 | 227 | *:* 228 | 229 | META-INF/*.SF 230 | META-INF/*.DSA 231 | META-INF/*.RSA 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | org.apache.maven.plugins 243 | maven-jar-plugin 244 | 2.4 245 | 246 | 247 | true 248 | 249 | true 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /src/main/java/com/criteo/kafka/GraphiteReporter.java: -------------------------------------------------------------------------------- 1 | package com.criteo.kafka; 2 | 3 | import com.yammer.metrics.Metrics; 4 | import com.yammer.metrics.core.*; 5 | import com.yammer.metrics.reporting.AbstractPollingReporter; 6 | import com.yammer.metrics.reporting.SocketProvider; 7 | import com.yammer.metrics.stats.Snapshot; 8 | import com.yammer.metrics.core.MetricPredicate; 9 | import org.apache.log4j.Logger; 10 | 11 | import java.io.BufferedWriter; 12 | import java.io.IOException; 13 | import java.io.OutputStreamWriter; 14 | import java.io.Writer; 15 | import java.lang.Thread.State; 16 | import java.net.Socket; 17 | import java.util.Locale; 18 | import java.util.Map.Entry; 19 | import java.util.SortedMap; 20 | import java.util.concurrent.TimeUnit; 21 | 22 | 23 | /** 24 | * A simple reporter which sends out application metrics to a Graphite 25 | * server periodically. 26 | */ 27 | public class GraphiteReporter extends AbstractPollingReporter implements MetricProcessor { 28 | private static final Logger LOG = Logger.getLogger(GraphiteReporter.class); 29 | protected final String prefix; 30 | protected final MetricPredicate predicate; 31 | protected final Locale locale = Locale.US; 32 | protected final Clock clock; 33 | protected final SocketProvider socketProvider; 34 | protected final VirtualMachineMetrics vm; 35 | protected Writer writer; 36 | public boolean printVMMetrics = true; 37 | 38 | private boolean hideMetersMeans = false; // Export all data by default 39 | 40 | /** 41 | * Enables the graphite reporter to send data for the default metrics registry to graphite 42 | * server with the specified period. 43 | * 44 | * @param period the period between successive outputs 45 | * @param unit the time unit of {@code period} 46 | * @param host the host name of graphite server (carbon-cache agent) 47 | * @param port the port number on which the graphite server is listening 48 | */ 49 | public static void enable(long period, TimeUnit unit, String host, int port) { 50 | enable(Metrics.defaultRegistry(), period, unit, host, port); 51 | } 52 | 53 | /** 54 | * Enables the graphite reporter to send data for the given metrics registry to graphite server 55 | * with the specified period. 56 | * 57 | * @param metricsRegistry the metrics registry 58 | * @param period the period between successive outputs 59 | * @param unit the time unit of {@code period} 60 | * @param host the host name of graphite server (carbon-cache agent) 61 | * @param port the port number on which the graphite server is listening 62 | */ 63 | public static void enable(MetricsRegistry metricsRegistry, long period, TimeUnit unit, String host, int port) { 64 | enable(metricsRegistry, period, unit, host, port, null); 65 | } 66 | 67 | /** 68 | * Enables the graphite reporter to send data to graphite server with the specified period. 69 | * 70 | * @param period the period between successive outputs 71 | * @param unit the time unit of {@code period} 72 | * @param host the host name of graphite server (carbon-cache agent) 73 | * @param port the port number on which the graphite server is listening 74 | * @param prefix the string which is prepended to all metric names 75 | */ 76 | public static void enable(long period, TimeUnit unit, String host, int port, String prefix) { 77 | enable(Metrics.defaultRegistry(), period, unit, host, port, prefix); 78 | } 79 | 80 | /** 81 | * Enables the graphite reporter to send data to graphite server with the specified period. 82 | * 83 | * @param metricsRegistry the metrics registry 84 | * @param period the period between successive outputs 85 | * @param unit the time unit of {@code period} 86 | * @param host the host name of graphite server (carbon-cache agent) 87 | * @param port the port number on which the graphite server is listening 88 | * @param prefix the string which is prepended to all metric names 89 | */ 90 | public static void enable(MetricsRegistry metricsRegistry, long period, TimeUnit unit, String host, int port, String prefix) { 91 | enable(metricsRegistry, period, unit, host, port, prefix, MetricPredicate.ALL); 92 | } 93 | 94 | /** 95 | * Enables the graphite reporter to send data to graphite server with the specified period. 96 | * 97 | * @param metricsRegistry the metrics registry 98 | * @param period the period between successive outputs 99 | * @param unit the time unit of {@code period} 100 | * @param host the host name of graphite server (carbon-cache agent) 101 | * @param port the port number on which the graphite server is listening 102 | * @param prefix the string which is prepended to all metric names 103 | * @param predicate filters metrics to be reported 104 | */ 105 | public static void enable(MetricsRegistry metricsRegistry, long period, TimeUnit unit, String host, int port, String prefix, MetricPredicate predicate) { 106 | try { 107 | final com.yammer.metrics.reporting.GraphiteReporter reporter = new com.yammer.metrics.reporting.GraphiteReporter(metricsRegistry, 108 | prefix, 109 | predicate, 110 | new DefaultSocketProvider(host, 111 | port), 112 | Clock.defaultClock()); 113 | reporter.start(period, unit); 114 | } catch (Exception e) { 115 | LOG.error("Error creating/starting Graphite reporter:", e); 116 | } 117 | } 118 | 119 | /** 120 | * Creates a new {@link com.yammer.metrics.reporting.GraphiteReporter}. 121 | * 122 | * @param host is graphite server 123 | * @param port is port on which graphite server is running 124 | * @param prefix is prepended to all names reported to graphite 125 | * @throws IOException if there is an error connecting to the Graphite server 126 | */ 127 | public GraphiteReporter(String host, int port, String prefix) throws IOException { 128 | this(Metrics.defaultRegistry(), host, port, prefix); 129 | } 130 | 131 | /** 132 | * Creates a new {@link com.yammer.metrics.reporting.GraphiteReporter}. 133 | * 134 | * @param metricsRegistry the metrics registry 135 | * @param host is graphite server 136 | * @param port is port on which graphite server is running 137 | * @param prefix is prepended to all names reported to graphite 138 | * @throws IOException if there is an error connecting to the Graphite server 139 | */ 140 | public GraphiteReporter(MetricsRegistry metricsRegistry, String host, int port, String prefix) throws IOException { 141 | this(metricsRegistry, 142 | prefix, 143 | MetricPredicate.ALL, 144 | new DefaultSocketProvider(host, port), 145 | Clock.defaultClock()); 146 | } 147 | 148 | /** 149 | * Creates a new {@link com.yammer.metrics.reporting.GraphiteReporter}. 150 | * 151 | * @param metricsRegistry the metrics registry 152 | * @param prefix is prepended to all names reported to graphite 153 | * @param predicate filters metrics to be reported 154 | * @param socketProvider a {@link SocketProvider} instance 155 | * @param clock a {@link Clock} instance 156 | * @throws IOException if there is an error connecting to the Graphite server 157 | */ 158 | public GraphiteReporter(MetricsRegistry metricsRegistry, String prefix, MetricPredicate predicate, SocketProvider socketProvider, Clock clock) throws IOException { 159 | this(metricsRegistry, prefix, predicate, socketProvider, clock, 160 | VirtualMachineMetrics.getInstance()); 161 | } 162 | 163 | /** 164 | * Creates a new {@link com.yammer.metrics.reporting.GraphiteReporter}. 165 | * 166 | * @param metricsRegistry the metrics registry 167 | * @param prefix is prepended to all names reported to graphite 168 | * @param predicate filters metrics to be reported 169 | * @param socketProvider a {@link SocketProvider} instance 170 | * @param clock a {@link Clock} instance 171 | * @param vm a {@link VirtualMachineMetrics} instance 172 | * @throws IOException if there is an error connecting to the Graphite server 173 | */ 174 | public GraphiteReporter(MetricsRegistry metricsRegistry, String prefix, MetricPredicate predicate, SocketProvider socketProvider, Clock clock, VirtualMachineMetrics vm) throws IOException { 175 | this(metricsRegistry, prefix, predicate, socketProvider, clock, vm, "graphite-reporter"); 176 | } 177 | 178 | /** 179 | * Creates a new {@link com.yammer.metrics.reporting.GraphiteReporter}. 180 | * 181 | * @param metricsRegistry the metrics registry 182 | * @param prefix is prepended to all names reported to graphite 183 | * @param predicate filters metrics to be reported 184 | * @param socketProvider a {@link SocketProvider} instance 185 | * @param clock a {@link Clock} instance 186 | * @param vm a {@link VirtualMachineMetrics} instance 187 | * @throws IOException if there is an error connecting to the Graphite server 188 | */ 189 | public GraphiteReporter(MetricsRegistry metricsRegistry, String prefix, MetricPredicate predicate, SocketProvider socketProvider, Clock clock, VirtualMachineMetrics vm, String name) throws IOException { 190 | super(metricsRegistry, name); 191 | this.socketProvider = socketProvider; 192 | this.vm = vm; 193 | 194 | this.clock = clock; 195 | 196 | if (prefix != null) { 197 | // Pre-append the "." so that we don't need to make anything conditional later. 198 | this.prefix = prefix + "."; 199 | } else { 200 | this.prefix = ""; 201 | } 202 | this.predicate = predicate; 203 | } 204 | 205 | public void setHideMetersMeans(boolean value){ 206 | this.hideMetersMeans = value; 207 | } 208 | 209 | @Override 210 | public void run() { 211 | Socket socket = null; 212 | int counter = 0; 213 | try { 214 | socket = this.socketProvider.get(); 215 | writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream())); 216 | 217 | final long epoch = clock.time() / 1000; 218 | if (this.printVMMetrics) { 219 | printVmMetrics(epoch); 220 | } 221 | counter = printRegularMetrics(epoch); 222 | LOG.info(Integer.toString(counter) + " metrics sent"); 223 | writer.flush(); 224 | } catch (Exception e) { 225 | if (LOG.isDebugEnabled()) { 226 | LOG.debug("Error writing to Graphite", e); 227 | } else { 228 | LOG.warn("Error writing to Graphite: ", e); 229 | } 230 | if (writer != null) { 231 | try { 232 | writer.flush(); 233 | } catch (IOException e1) { 234 | LOG.error("Error while flushing writer:", e1); 235 | } 236 | } 237 | } finally { 238 | if (socket != null) { 239 | try { 240 | socket.close(); 241 | } catch (IOException e) { 242 | LOG.error("Error while closing socket:", e); 243 | } 244 | } 245 | writer = null; 246 | } 247 | } 248 | 249 | protected int printRegularMetrics(final Long epoch) { 250 | int counter = 0; 251 | for (Entry> entry : getMetricsRegistry().groupedMetrics( 252 | predicate).entrySet()) { 253 | for (Entry subEntry : entry.getValue().entrySet()) { 254 | final Metric metric = subEntry.getValue(); 255 | if (metric != null) { 256 | try { 257 | metric.processWith(this, subEntry.getKey(), epoch); 258 | counter++; 259 | } catch (Exception ignored) { 260 | LOG.error("Error printing regular metrics:", ignored); 261 | } 262 | } 263 | } 264 | } 265 | return counter; 266 | } 267 | 268 | protected void sendInt(long timestamp, String name, String valueName, long value) { 269 | sendToGraphite(timestamp, name, valueName + " " + String.format(locale, "%d", value)); 270 | } 271 | 272 | protected void sendFloat(long timestamp, String name, String valueName, double value) { 273 | sendToGraphite(timestamp, name, valueName + " " + String.format(locale, "%2.2f", value)); 274 | } 275 | 276 | protected void sendObjToGraphite(long timestamp, String name, String valueName, Object value) { 277 | sendToGraphite(timestamp, name, valueName + " " + String.format(locale, "%s", value)); 278 | } 279 | 280 | protected void sendToGraphite(long timestamp, String name, String value) { 281 | try { 282 | if (!prefix.isEmpty()) { 283 | writer.write(prefix); 284 | } 285 | writer.write(sanitizeString(name)); 286 | writer.write('.'); 287 | writer.write(value); 288 | writer.write(' '); 289 | writer.write(Long.toString(timestamp)); 290 | writer.write('\n'); 291 | writer.flush(); 292 | } catch (IOException e) { 293 | LOG.error("Error sending to Graphite:", e); 294 | } 295 | } 296 | 297 | protected String sanitizeName(MetricName name) { 298 | final StringBuilder sb = new StringBuilder() 299 | .append(name.getGroup()) 300 | .append('.') 301 | .append(name.getType()) 302 | .append('.'); 303 | if (name.hasScope()) { 304 | sb.append(name.getScope()) 305 | .append('.'); 306 | } 307 | return sb.append(name.getName()).toString(); 308 | } 309 | 310 | protected String sanitizeString(String s) { 311 | return s.replace(' ', '-'); 312 | } 313 | 314 | @Override 315 | public void processGauge(MetricName name, Gauge gauge, Long epoch) throws IOException { 316 | sendObjToGraphite(epoch, sanitizeName(name), "value", gauge.value()); 317 | } 318 | 319 | @Override 320 | public void processCounter(MetricName name, Counter counter, Long epoch) throws IOException { 321 | sendInt(epoch, sanitizeName(name), "count", counter.count()); 322 | } 323 | 324 | @Override 325 | public void processMeter(MetricName name, Metered meter, Long epoch) throws IOException { 326 | final String sanitizedName = sanitizeName(name); 327 | sendInt(epoch, sanitizedName, "count", meter.count()); 328 | if (!hideMetersMeans) { 329 | sendFloat(epoch, sanitizedName, "meanRate", meter.meanRate()); 330 | sendFloat(epoch, sanitizedName, "1MinuteRate", meter.oneMinuteRate()); 331 | sendFloat(epoch, sanitizedName, "5MinuteRate", meter.fiveMinuteRate()); 332 | sendFloat(epoch, sanitizedName, "15MinuteRate", meter.fifteenMinuteRate()); 333 | } 334 | } 335 | 336 | @Override 337 | public void processHistogram(MetricName name, Histogram histogram, Long epoch) throws IOException { 338 | final String sanitizedName = sanitizeName(name); 339 | sendSummarizable(epoch, sanitizedName, histogram); 340 | sendSampling(epoch, sanitizedName, histogram); 341 | } 342 | 343 | @Override 344 | public void processTimer(MetricName name, Timer timer, Long epoch) throws IOException { 345 | processMeter(name, timer, epoch); 346 | final String sanitizedName = sanitizeName(name); 347 | sendSummarizable(epoch, sanitizedName, timer); 348 | sendSampling(epoch, sanitizedName, timer); 349 | } 350 | 351 | protected void sendSummarizable(long epoch, String sanitizedName, Summarizable metric) throws IOException { 352 | sendFloat(epoch, sanitizedName, "min", metric.min()); 353 | sendFloat(epoch, sanitizedName, "max", metric.max()); 354 | sendFloat(epoch, sanitizedName, "mean", metric.mean()); 355 | sendFloat(epoch, sanitizedName, "stddev", metric.stdDev()); 356 | } 357 | 358 | protected void sendSampling(long epoch, String sanitizedName, Sampling metric) throws IOException { 359 | final Snapshot snapshot = metric.getSnapshot(); 360 | sendFloat(epoch, sanitizedName, "median", snapshot.getMedian()); 361 | sendFloat(epoch, sanitizedName, "75percentile", snapshot.get75thPercentile()); 362 | sendFloat(epoch, sanitizedName, "95percentile", snapshot.get95thPercentile()); 363 | sendFloat(epoch, sanitizedName, "98percentile", snapshot.get98thPercentile()); 364 | sendFloat(epoch, sanitizedName, "99percentile", snapshot.get99thPercentile()); 365 | sendFloat(epoch, sanitizedName, "999percentile", snapshot.get999thPercentile()); 366 | } 367 | 368 | protected void printVmMetrics(long epoch) { 369 | sendFloat(epoch, "jvm.memory", "heap_usage", vm.heapUsage()); 370 | sendFloat(epoch, "jvm.memory", "non_heap_usage", vm.nonHeapUsage()); 371 | for (Entry pool : vm.memoryPoolUsage().entrySet()) { 372 | sendFloat(epoch, "jvm.memory.memory_pool_usages", sanitizeString(pool.getKey()), pool.getValue()); 373 | } 374 | 375 | sendInt(epoch, "jvm", "daemon_thread_count", vm.daemonThreadCount()); 376 | sendInt(epoch, "jvm", "thread_count", vm.threadCount()); 377 | sendInt(epoch, "jvm", "uptime", vm.uptime()); 378 | sendFloat(epoch, "jvm", "fd_usage", vm.fileDescriptorUsage()); 379 | 380 | for (Entry entry : vm.threadStatePercentages().entrySet()) { 381 | sendFloat(epoch, "jvm.thread-states", entry.getKey().toString().toLowerCase(), entry.getValue()); 382 | } 383 | 384 | for (Entry entry : vm.garbageCollectors().entrySet()) { 385 | final String name = "jvm.gc." + sanitizeString(entry.getKey()); 386 | sendInt(epoch, name, "time", entry.getValue().getTime(TimeUnit.MILLISECONDS)); 387 | sendInt(epoch, name, "runs", entry.getValue().getRuns()); 388 | } 389 | } 390 | 391 | public static class DefaultSocketProvider implements SocketProvider { 392 | 393 | private final String host; 394 | private final int port; 395 | 396 | public DefaultSocketProvider(String host, int port) { 397 | this.host = host; 398 | this.port = port; 399 | 400 | } 401 | 402 | @Override 403 | public Socket get() throws Exception { 404 | return new Socket(this.host, this.port); 405 | } 406 | 407 | } 408 | } 409 | --------------------------------------------------------------------------------