├── .gitignore
├── README.md
├── pom.xml
└── src
    └── main
        └── java
            └── com
                └── criteo
                    └── kafka
                        ├── KafkaGangliaMetricsReporter.java
                        ├── KafkaGangliaMetricsReporterMBean.java
                        └── RegexMetricPredicate.java


/.gitignore:
--------------------------------------------------------------------------------
1 | /.project
2 | /.classpath
3 | /target/
4 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Kafka Ganglia Metrics Reporter
 2 | ==============================
 3 | 
 4 | Warning: this repository is not used by Criteo anymore. If you are using it and are interested in taking ownership, let us know by opening an issue.
 5 | 
 6 | This is a simple reporter for kafka using the 
 7 | [GangliaReporter](http://metrics.codahale.com/manual/ganglia/). It works with 
 8 | kafka 0.8 beta version.
 9 | 
10 | Install On Broker
11 | ------------
12 | 
13 | 1. Build the `kafka-ganglia-1.0.0.jar` jar using `mvn package`.
14 | 2. Add `kafka-ganglia-1.0.0.jar` and `metrics-ganglia-2.2.0.jar` to the `libs/` 
15 |    directory of your kafka broker installation
16 | 3. Configure the broker (see the configuration section below)
17 | 4. Restart the broker
18 | 
19 | Configuration
20 | ------------
21 | 
22 | Edit the `server.properties` file of your installation, activate the reporter by setting:
23 | 
24 |     kafka.metrics.reporters=com.criteo.kafka.KafkaGangliaMetricsReporter[,kafka.metrics.KafkaCSVMetricsReporter[,....]]
25 |     kafka.ganglia.metrics.reporter.enabled=true
26 | 
27 | Here is a list of default properties used:
28 | 
29 |     kafka.ganglia.metrics.host=localhost
30 |     kafka.ganglia.metrics.port=8649
31 |     kafka.ganglia.metrics.group=kafka
32 |     # This can be use to exclude some metrics from ganglia 
33 |     # since kafka has quite a lot of metrics, it is useful
34 |     # if you have many topics/partitions.
35 |     kafka.ganglia.metrics.exclude.regex=<not set>
36 | 
37 | Usage As Lib
38 | -----------
39 | 
40 | Simply build the jar and publish it to your maven internal repository (this 
41 | package is not published to any public repositories unfortunately).
42 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
  3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
  5 |                              http://maven.apache.org/xsd/maven-4.0.0.xsd">
  6 |   <modelVersion>4.0.0</modelVersion>
  7 |   <groupId>com.criteo.kafka</groupId>
  8 |   <artifactId>kafka-ganglia</artifactId>
  9 |   <packaging>jar</packaging>
 10 |   <version>1.0.0</version>
 11 |   <name>Kafka metrics output to Ganglia</name>
 12 |   <url>http://maven.criteo</url>
 13 | 
 14 |   <prerequisites>
 15 |     <maven>3.0.4</maven>
 16 |   </prerequisites>
 17 | 
 18 |   <dependencies>
 19 |     <dependency>
 20 |       <groupId>org.apache.kafka</groupId>
 21 |       <artifactId>kafka_2.10</artifactId>
 22 |       <version>0.8.0</version>
 23 |     </dependency>
 24 |     <dependency>
 25 |       <groupId>com.yammer.metrics</groupId>
 26 |       <artifactId>metrics-ganglia</artifactId>
 27 |       <version>2.2.0</version>
 28 |     </dependency>
 29 | 
 30 |     <!-- Fix lame kafka pom -->
 31 |     <dependency>
 32 |       <groupId>org.apache.zookeeper</groupId>
 33 |       <artifactId>zookeeper</artifactId>
 34 |       <version>3.3.4</version>
 35 |       <exclusions>
 36 |         <exclusion>
 37 |           <groupId>log4j</groupId>
 38 |           <artifactId>log4j</artifactId>
 39 |         </exclusion>
 40 |         <exclusion>
 41 |           <groupId>jline</groupId>
 42 |           <artifactId>jline</artifactId>
 43 |         </exclusion>
 44 |       </exclusions>
 45 |     </dependency>
 46 |     <dependency>
 47 |       <groupId>org.scala-lang</groupId>
 48 |       <artifactId>scala-library</artifactId>
 49 |       <version>2.10.7</version>
 50 |     </dependency>
 51 |     <dependency>
 52 |       <groupId>log4j</groupId>
 53 |       <artifactId>log4j</artifactId>
 54 |       <version>1.2.15</version>
 55 |       <exclusions>
 56 |         <exclusion>
 57 |           <groupId>javax.jms</groupId>
 58 |           <artifactId>jms</artifactId>
 59 |         </exclusion>
 60 |         <exclusion>
 61 |             <groupId>com.sun.jmx</groupId>
 62 |             <artifactId>jmxri</artifactId>
 63 |         </exclusion>
 64 |         <exclusion>
 65 |             <groupId>com.sun.jdmk</groupId>
 66 |             <artifactId>jmxtools</artifactId>
 67 |         </exclusion>
 68 |       </exclusions>
 69 |     </dependency>
 70 |     <dependency>
 71 |       <groupId>net.sf.jopt-simple</groupId>
 72 |       <artifactId>jopt-simple</artifactId>
 73 |       <version>3.2</version>
 74 |     </dependency>
 75 |     <dependency>
 76 |       <groupId>org.slf4j</groupId>
 77 |       <artifactId>slf4j-simple</artifactId>
 78 |       <version>1.6.4</version>
 79 |     </dependency>
 80 |     <dependency>
 81 |       <groupId>org.scala-lang</groupId>
 82 |       <artifactId>scala-compiler</artifactId>
 83 |       <version>2.10.7</version>
 84 |     </dependency>
 85 |     <dependency>
 86 |       <groupId>com.101tec</groupId>
 87 |       <artifactId>zkclient</artifactId>
 88 |       <version>0.3</version>
 89 |     </dependency>
 90 |     <dependency>
 91 |       <groupId>org.xerial.snappy</groupId>
 92 |       <artifactId>snappy-java</artifactId>
 93 |       <version>1.0.4.1</version>
 94 |     </dependency>
 95 |     <dependency>
 96 |       <groupId>com.yammer.metrics</groupId>
 97 |       <artifactId>metrics-core</artifactId>
 98 |       <version>2.2.0</version>
 99 |     </dependency>
100 |     <dependency>
101 |       <groupId>com.yammer.metrics</groupId>
102 |       <artifactId>metrics-annotation</artifactId>
103 |       <version>2.2.0</version>
104 |     </dependency>
105 |     <dependency>
106 |       <groupId>org.easymock</groupId>
107 |       <artifactId>easymock</artifactId>
108 |       <version>3.0</version>
109 |       <scope>test</scope>
110 |     </dependency>
111 |     <dependency>
112 |       <groupId>junit</groupId>
113 |       <artifactId>junit</artifactId>
114 |       <version>4.11</version>
115 |     </dependency>
116 |     <dependency>
117 |       <groupId>org.scalatest</groupId>
118 |       <artifactId>scalatest_2.9.2</artifactId>
119 |       <version>1.8</version>
120 |       <scope>test</scope>
121 |     </dependency>
122 |   </dependencies>
123 | 
124 |   <distributionManagement>
125 |     <repository>
126 |       <id>criteo.releases</id>
127 |       <name>Criteo Releases</name>
128 |       <url>http://maven/content/repositories/criteo.releases</url>
129 |     </repository>
130 |     <snapshotRepository>
131 |       <id>criteo.snapshots</id>
132 |       <name>Criteo Snapshots</name>
133 |       <url>http://maven/content/repositories/criteo.snapshots</url>
134 |     </snapshotRepository>
135 |   </distributionManagement>
136 | 
137 |   <properties>
138 |     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
139 |     <shaded.name>uber</shaded.name>
140 |   </properties>
141 | 
142 |   <build>
143 |     <plugins>
144 |       <plugin>
145 |         <groupId>org.apache.maven.plugins</groupId>
146 |         <artifactId>maven-compiler-plugin</artifactId>
147 |         <configuration>
148 |           <source>1.6</source>
149 |           <target>1.6</target>
150 |         </configuration>
151 |       </plugin>
152 | 
153 |       <!-- Download sources and javadocs for eclipse -->
154 |       <plugin>
155 |         <groupId>org.apache.maven.plugins</groupId>
156 |         <artifactId>maven-eclipse-plugin</artifactId>
157 |         <version>2.9</version>
158 |         <configuration>
159 |           <downloadSources>true</downloadSources>
160 |           <downloadJavadocs>true</downloadJavadocs>
161 |         </configuration>
162 |       </plugin>
163 | 
164 |       <!-- Create jar of sources and javadocs -->
165 |       <plugin>
166 |         <groupId>org.apache.maven.plugins</groupId>
167 |         <artifactId>maven-source-plugin</artifactId>
168 |         <version>2.2.1</version>
169 |         <executions>
170 |           <execution>
171 |             <id>attach-sources</id>
172 |             <goals>
173 |               <goal>jar</goal>
174 |             </goals>
175 |           </execution>
176 |         </executions>
177 |       </plugin>
178 |       <plugin>
179 |         <groupId>org.apache.maven.plugins</groupId>
180 |         <artifactId>maven-javadoc-plugin</artifactId>
181 |         <version>2.9</version>
182 |         <configuration>
183 |           <!-- Add nice UML graph -->
184 |           <doclet>org.umlgraph.doclet.UmlGraphDoc</doclet>
185 |           <docletArtifact>
186 |             <groupId>org.umlgraph</groupId>
187 |             <artifactId>doclet</artifactId>
188 |             <version>5.1</version>
189 |           </docletArtifact>
190 |           <additionalparam>-views</additionalparam>
191 |           <useStandardDocletOptions>true</useStandardDocletOptions>
192 |         </configuration>
193 |         <executions>
194 |           <execution>
195 |             <id>attach-javadocs</id>
196 |             <goals>
197 |               <goal>jar</goal>
198 |             </goals>
199 |           </execution>
200 |         </executions>
201 |       </plugin>
202 | 
203 |       <!-- Build an ÜberJar -->
204 |       <!-- Uncomment to have a standalone jar -->
205 |       <plugin>
206 |         <groupId>org.apache.maven.plugins</groupId>
207 |         <artifactId>maven-shade-plugin</artifactId>
208 |         <version>2.0</version>
209 |         <executions>
210 |           <execution>
211 |             <phase>package</phase>
212 |             <goals>
213 |               <goal>shade</goal>
214 |             </goals>
215 |             <configuration>
216 |               <createDependencyReducedPom>false</createDependencyReducedPom>
217 |               <shadedArtifactAttached>true</shadedArtifactAttached>
218 |               <shadedClassifierName>${shaded.name}</shadedClassifierName>
219 |               <artifactSet>
220 |                 <includes>
221 |                   <include>*:*</include>
222 |                 </includes>
223 |               </artifactSet>
224 |               <minimizeJar>true</minimizeJar>
225 |               <filters>
226 |                 <filter>
227 |                   <artifact>*:*</artifact>
228 |                   <excludes>
229 |                     <exclude>META-INF/*.SF</exclude>
230 |                     <exclude>META-INF/*.DSA</exclude>
231 |                     <exclude>META-INF/*.RSA</exclude>
232 |                   </excludes>
233 |                 </filter>
234 |               </filters>
235 |             </configuration>
236 |           </execution>
237 |         </executions>
238 |       </plugin>
239 | 
240 |      <!-- Add infos in the manifest -->
241 |       <plugin>
242 |         <groupId>org.apache.maven.plugins</groupId>
243 |         <artifactId>maven-jar-plugin</artifactId>
244 |         <version>2.4</version>
245 |         <configuration>
246 |           <archive>
247 |             <index>true</index>
248 |             <manifest>
249 |               <addDefaultImplementationEntries>true
250 |                 </addDefaultImplementationEntries>
251 |             </manifest>
252 |           </archive>
253 |         </configuration>
254 |       </plugin>
255 |     </plugins>
256 |   </build>
257 | 
258 | </project>
259 | 


--------------------------------------------------------------------------------
/src/main/java/com/criteo/kafka/KafkaGangliaMetricsReporter.java:
--------------------------------------------------------------------------------
 1 | package com.criteo.kafka;
 2 | 
 3 | import java.io.IOException;
 4 | import java.util.concurrent.TimeUnit;
 5 | 
 6 | import org.apache.log4j.Logger;
 7 | 
 8 | import com.yammer.metrics.Metrics;
 9 | import com.yammer.metrics.core.Metric;
10 | import com.yammer.metrics.core.MetricName;
11 | import com.yammer.metrics.core.MetricPredicate;
12 | import com.yammer.metrics.reporting.GangliaReporter;
13 | 
14 | import kafka.metrics.KafkaMetricsConfig;
15 | import kafka.metrics.KafkaMetricsReporter;
16 | import kafka.metrics.KafkaMetricsReporterMBean;
17 | import kafka.utils.VerifiableProperties;
18 | 
19 | public class KafkaGangliaMetricsReporter implements KafkaMetricsReporter,
20 | 	KafkaGangliaMetricsReporterMBean {
21 | 
22 | 	static Logger LOG = Logger.getLogger(KafkaGangliaMetricsReporter.class);
23 | 	static String GANGLIA_DEFAULT_HOST = "localhost";
24 | 	static int GANGLIA_DEFAULT_PORT = 8649;
25 | 	static String GANGLIA_DEFAULT_PREFIX = "kafka";
26 | 	
27 | 	boolean initialized = false;
28 | 	boolean running = false;
29 | 	GangliaReporter reporter = null;
30 |     String gangliaHost = GANGLIA_DEFAULT_HOST;
31 |     int gangliaPort = GANGLIA_DEFAULT_PORT;
32 |     String gangliaGroupPrefix = GANGLIA_DEFAULT_PREFIX;
33 |     MetricPredicate predicate = MetricPredicate.ALL;
34 | 
35 | 	@Override
36 | 	public String getMBeanName() {
37 | 		return "kafka:type=com.criteo.kafka.KafkaGangliaMetricsReporter";
38 | 	}
39 | 
40 | 	@Override
41 | 	public synchronized void startReporter(long pollingPeriodSecs) {
42 | 		if (initialized && !running) {
43 | 			reporter.start(pollingPeriodSecs, TimeUnit.SECONDS);
44 | 			running = true;
45 | 			LOG.info(String.format("Started Kafka Ganglia metrics reporter with polling period %d seconds", pollingPeriodSecs));
46 | 		}
47 | 	}
48 | 
49 | 	@Override
50 | 	public synchronized void stopReporter() {
51 | 		if (initialized && running) {
52 | 			reporter.shutdown();
53 | 			running = false;
54 | 			LOG.info("Stopped Kafka Ganglia metrics reporter");
55 |             try {
56 |             	reporter = new GangliaReporter(
57 |             			Metrics.defaultRegistry(),
58 |             			gangliaHost,
59 |             			gangliaPort,
60 |             			gangliaGroupPrefix,
61 |             			predicate
62 |             			);
63 |             } catch (IOException e) {
64 |             	LOG.error("Unable to initialize GangliaReporter", e);
65 |             }
66 | 		}
67 | 	}
68 | 
69 | 	@Override
70 | 	public synchronized void init(VerifiableProperties props) {
71 | 		if (!initialized) {
72 | 			KafkaMetricsConfig metricsConfig = new KafkaMetricsConfig(props);
73 |             gangliaHost = props.getString("kafka.ganglia.metrics.host", GANGLIA_DEFAULT_HOST);
74 |             gangliaPort = props.getInt("kafka.ganglia.metrics.port", GANGLIA_DEFAULT_PORT);
75 |             gangliaGroupPrefix = props.getString("kafka.ganglia.metrics.group", GANGLIA_DEFAULT_PREFIX);
76 |             String regex = props.getString("kafka.ganglia.metrics.exclude.regex", null);
77 |             if (regex != null) {
78 |             	predicate = new RegexMetricPredicate(regex);
79 |             }
80 |             try {
81 |             	reporter = new GangliaReporter(
82 |             			Metrics.defaultRegistry(),
83 |             			gangliaHost,
84 |             			gangliaPort,
85 |             			gangliaGroupPrefix,
86 |             			predicate
87 |             			);
88 |             } catch (IOException e) {
89 |             	LOG.error("Unable to initialize GangliaReporter", e);
90 |             }
91 |             if (props.getBoolean("kafka.ganglia.metrics.reporter.enabled", false)) {
92 |             	initialized = true;
93 |             	startReporter(metricsConfig.pollingIntervalSecs());
94 |             }
95 |         }
96 | 	}
97 | }
98 | 


--------------------------------------------------------------------------------
/src/main/java/com/criteo/kafka/KafkaGangliaMetricsReporterMBean.java:
--------------------------------------------------------------------------------
1 | package com.criteo.kafka;
2 | 
3 | import kafka.metrics.KafkaMetricsReporterMBean;
4 | 
5 | public interface KafkaGangliaMetricsReporterMBean extends
6 | 		KafkaMetricsReporterMBean {
7 | 
8 | }
9 | 


--------------------------------------------------------------------------------
/src/main/java/com/criteo/kafka/RegexMetricPredicate.java:
--------------------------------------------------------------------------------
 1 | package com.criteo.kafka;
 2 | 
 3 | import java.util.regex.Pattern;
 4 | 
 5 | import org.apache.log4j.Logger;
 6 | 
 7 | import com.yammer.metrics.core.Metric;
 8 | import com.yammer.metrics.core.MetricName;
 9 | import com.yammer.metrics.core.MetricPredicate;
10 | 
11 | public class RegexMetricPredicate implements MetricPredicate {
12 | 
13 | 	Pattern pattern = null;
14 | 	//static Logger LOG = Logger.getLogger(RegexMetricPredicate.class);
15 | 	
16 | 	public RegexMetricPredicate(String regex) {
17 | 		pattern = Pattern.compile(regex);
18 | 	}
19 | 	
20 | 	@Override
21 | 	public boolean matches(MetricName name, Metric metric) {
22 | 		boolean ok = !pattern.matcher(name.getName()).matches();
23 | 		//LOG.info(String.format("name: %s - %s", name.getName(), ok));
24 | 		return ok;
25 | 	}
26 | 
27 | }
28 | 


--------------------------------------------------------------------------------