├── .gitignore
├── README.md
├── pom.xml
└── src
    ├── main
        └── java
        │   └── com
        │       └── github
        │           └── ptgoetz
        │               └── logback
        │                   └── kafka
        │                       ├── KafkaAppender.java
        │                       └── formatter
        │                           ├── Formatter.java
        │                           ├── JsonFormatter.java
        │                           └── MessageFormatter.java
    └── test
        ├── java
            └── com
            │   └── github
            │       └── ptgoetz
            │           └── logback
            │               └── kafka
            │                   └── formatter
            │                       ├── JsonFormatterTest.java
            │                       ├── MockLoggingEvent.java
            │                       └── SimpleApp.java
        └── resources
            ├── logback-formatter.xml
            └── logback.xml


/.gitignore:
--------------------------------------------------------------------------------
1 | target/
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # logback-kafka
 2 | 
 3 | 
 4 | Logback appenders for logging data to Apache Kafka
 5 | 
 6 | 
 7 | ## Maven Dependency
 8 | To use logback-kafka in your project add to following to your pom.xml:
 9 | 
10 | ```xml
11 | <dependency>
12 |     <groupId>com.github.ptgoetz</groupId>
13 |     <artifactId>logback-kafka</artifactId>
14 |     <version>0.1.0</version>
15 | </dependency>
16 | ```
17 | 
18 | ## Configuration
19 | 
20 | To configure your application to log to kafka, add an appender entry in your logback configuration file, and specify 
21 | a zookeeper host string, and kafka topic name to log to.
22 | 
23 | 
24 | ```xml
25 | <?xml version="1.0" encoding="UTF-8" ?>
26 | <configuration>
27 |     <appender name="KAFKA"
28 |         class="com.github.ptgoetz.logback.kafka.KafkaAppender">
29 |         <topic>mytopic</topic>
30 |         <zookeeperHost>localhost:2181</zookeeperHost>
31 |     </appender>
32 |     <root level="debug">
33 |         <appender-ref ref="KAFKA" />
34 |     </root>
35 | </configuration>
36 | ```
37 | 
38 | ## Overriding Default Behavior
39 | By default, the Kafka appender will simply write the received log message to the kafka queue. You can override this 
40 | behavior by specifying a custom formatter class:
41 | 
42 | ```xml
43 | <?xml version="1.0" encoding="UTF-8" ?>
44 | <configuration>
45 |     <appender name="KAFKA"
46 |         class="com.github.ptgoetz.logback.kafka.KafkaAppender">
47 |         <topic>foo</topic>
48 |         <zookeeperHost>localhost:2181</zookeeperHost>
49 |         <!-- specify a custom formatter -->
50 |         <formatter class="com.github.ptgoetz.logback.kafka.formatter.JsonFormatter">
51 |             <!-- 
52 |             Whether we expect the log message to be JSON encoded or not.
53 |             If set to "false", the log message will be treated as a string, 
54 |             and wrapped in quotes. Otherwise it will be treated as a parseable
55 |             JSON object.
56 |             -->
57 |             <expectJson>true</expectJson>
58 |         </formatter>
59 |     </appender>
60 |     <root level="debug">
61 |         <appender-ref ref="KAFKA" />
62 |     </root>
63 | </configuration>
64 | ```
65 | 
66 | 
67 | 
68 | Formatters simply need to implement the `com.github.ptgoetz.logback.kafka.formatter.Formatter` interface:
69 | 
70 | ```java
71 | package com.github.ptgoetz.logback.kafka.formatter;
72 | 
73 | import ch.qos.logback.classic.spi.ILoggingEvent;
74 | 
75 | public interface Formatter {
76 |     String format(ILoggingEvent event);
77 | }
78 | ```
79 | 
80 | You can find the `ch.qos.logback.classic.spi.ILoggingEvent` javadoc [here](http://logback.qos.ch/apidocs/ch/qos/logback/classic/spi/ILoggingEvent.html).
81 | 
82 | 
83 | 
84 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
 1 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 2 | 	<modelVersion>4.0.0</modelVersion>
 3 | 
 4 | 	<groupId>com.github.ptgoetz</groupId>
 5 | 	<artifactId>logback-kafka</artifactId>
 6 | 	<version>0.1.1-SNAPSHOT</version>
 7 | 	<packaging>jar</packaging>
 8 | 
 9 | 	<parent>
10 | 		<groupId>org.sonatype.oss</groupId>
11 | 		<artifactId>oss-parent</artifactId>
12 | 		<version>7</version>
13 | 	</parent>
14 | 
15 | 	<name>logback-kafka</name>
16 | 	<url>http://maven.apache.org</url>
17 | 
18 | 	<licenses>
19 | 		<license>
20 | 			<name>Eclipse Public License - v 1.0</name>
21 | 			<url>http://www.eclipse.org/legal/epl-v10.html</url>
22 | 			<distribution>repo</distribution>
23 | 		</license>
24 | 	</licenses>
25 | 	<scm>
26 | 		<connection>scm:git:git@github.com:ptgoetz/logback-kafka.git</connection>
27 | 		<developerConnection>scm:git:git@github.com:ptgoetz/logback-kafka.git</developerConnection>
28 | 		<url>:git@github.com:ptgoetz/logback-kafka.git</url>
29 | 	</scm>
30 | 
31 | 	<developers>
32 | 		<developer>
33 | 			<id>ptgoetz</id>
34 | 			<name>P. Taylor Goetz</name>
35 | 			<email>ptgoetz@gmail.com</email>
36 | 		</developer>
37 | 	</developers>
38 | 
39 | 	<properties>
40 | 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
41 | 	</properties>
42 | 
43 | 	<dependencies>
44 | 		<dependency>
45 | 			<groupId>junit</groupId>
46 | 			<artifactId>junit</artifactId>
47 | 			<version>4.10</version>
48 | 			<scope>test</scope>
49 | 		</dependency>
50 | 		<dependency>
51 | 			<groupId>ch.qos.logback</groupId>
52 | 			<artifactId>logback-classic</artifactId>
53 | 			<version>1.0.13</version>
54 | 		</dependency>
55 | 		<dependency>
56 | 			<groupId>kafka</groupId>
57 | 			<artifactId>kafka</artifactId>
58 | 			<version>0.7.1</version>
59 | 		</dependency>
60 | 		<dependency>
61 | 			<groupId>org.scala-lang</groupId>
62 | 			<artifactId>scala-library</artifactId>
63 | 			<version>2.8.0</version>
64 | 		</dependency>
65 | 		<dependency>
66 | 			<groupId>com.github.sgroschupf</groupId>
67 | 			<artifactId>zkclient</artifactId>
68 | 			<version>0.1</version>
69 | 		</dependency>
70 | 		<dependency>
71 | 			<artifactId>slf4j-api</artifactId>
72 | 			<groupId>org.slf4j</groupId>
73 | 			<version>1.7.5</version>
74 | 		</dependency>
75 | 	</dependencies>
76 | </project>
77 | 


--------------------------------------------------------------------------------
/src/main/java/com/github/ptgoetz/logback/kafka/KafkaAppender.java:
--------------------------------------------------------------------------------
 1 | package com.github.ptgoetz.logback.kafka;
 2 | 
 3 | import java.util.Properties;
 4 | 
 5 | import kafka.javaapi.producer.Producer;
 6 | import kafka.javaapi.producer.ProducerData;
 7 | import kafka.producer.ProducerConfig;
 8 | import ch.qos.logback.classic.spi.ILoggingEvent;
 9 | import ch.qos.logback.core.AppenderBase;
10 | 
11 | import com.github.ptgoetz.logback.kafka.formatter.Formatter;
12 | import com.github.ptgoetz.logback.kafka.formatter.MessageFormatter;
13 | 
14 | public class KafkaAppender extends AppenderBase<ILoggingEvent> {
15 | 
16 |     private String topic;
17 |     private String zookeeperHost;
18 |     private Producer<String, String> producer;
19 |     private Formatter formatter;
20 | 
21 |     public String getTopic() {
22 |         return topic;
23 |     }
24 | 
25 |     public void setTopic(String topic) {
26 |         this.topic = topic;
27 |     }
28 | 
29 |     public String getZookeeperHost() {
30 |         return zookeeperHost;
31 |     }
32 | 
33 |     public void setZookeeperHost(String zookeeperHost) {
34 |         this.zookeeperHost = zookeeperHost;
35 |     }
36 | 
37 |     public Formatter getFormatter() {
38 |         return formatter;
39 |     }
40 | 
41 |     public void setFormatter(Formatter formatter) {
42 |         this.formatter = formatter;
43 |     }
44 | 
45 |     @Override
46 |     public void start() {
47 |         if (this.formatter == null) {
48 |             this.formatter = new MessageFormatter();
49 |         }
50 |         super.start();
51 |         Properties props = new Properties();
52 |         props.put("zk.connect", this.zookeeperHost);
53 |         props.put("serializer.class", "kafka.serializer.StringEncoder");
54 |         ProducerConfig config = new ProducerConfig(props);
55 |         this.producer = new Producer<String, String>(config);
56 |     }
57 | 
58 |     @Override
59 |     public void stop() {
60 |         super.stop();
61 |         this.producer.close();
62 |     }
63 | 
64 |     @Override
65 |     protected void append(ILoggingEvent event) {
66 |         String payload = this.formatter.format(event);
67 |         ProducerData<String, String> data = new ProducerData<String, String>(this.topic, payload);
68 |         this.producer.send(data);
69 |     }
70 | 
71 | }
72 | 


--------------------------------------------------------------------------------
/src/main/java/com/github/ptgoetz/logback/kafka/formatter/Formatter.java:
--------------------------------------------------------------------------------
1 | package com.github.ptgoetz.logback.kafka.formatter;
2 | 
3 | import ch.qos.logback.classic.spi.ILoggingEvent;
4 | 
5 | public interface Formatter {
6 |     String format(ILoggingEvent event);
7 | }
8 | 


--------------------------------------------------------------------------------
/src/main/java/com/github/ptgoetz/logback/kafka/formatter/JsonFormatter.java:
--------------------------------------------------------------------------------
 1 | package com.github.ptgoetz.logback.kafka.formatter;
 2 | 
 3 | import ch.qos.logback.classic.spi.ILoggingEvent;
 4 | 
 5 | public class JsonFormatter implements Formatter {
 6 |     private static final String QUOTE = "\"";
 7 |     private static final String COLON = ":";
 8 |     private static final String COMMA = ",";
 9 | 
10 |     private boolean expectJson = false;
11 | 
12 |     public String format(ILoggingEvent event) {
13 |         StringBuilder sb = new StringBuilder();
14 |         sb.append("{");
15 |         fieldName("level", sb);
16 |         quote(event.getLevel().levelStr, sb);
17 |         sb.append(COMMA);
18 |         fieldName("logger", sb);
19 |         quote(event.getLoggerName(), sb);
20 |         sb.append(COMMA);
21 |         fieldName("timestamp", sb);
22 |         sb.append(event.getTimeStamp());
23 |         sb.append(COMMA);
24 |         fieldName("message", sb);
25 |         if (this.expectJson) {
26 |             sb.append(event.getFormattedMessage());
27 |         } else {
28 |             quote(event.getFormattedMessage(), sb);
29 |         }
30 | 
31 |         sb.append("}");
32 |         return sb.toString();
33 |     }
34 | 
35 |     private static void fieldName(String name, StringBuilder sb) {
36 |         quote(name, sb);
37 |         sb.append(COLON);
38 |     }
39 | 
40 |     private static void quote(String value, StringBuilder sb) {
41 |         sb.append(QUOTE);
42 |         sb.append(value);
43 |         sb.append(QUOTE);
44 |     }
45 | 
46 |     public boolean isExpectJson() {
47 |         return expectJson;
48 |     }
49 | 
50 |     public void setExpectJson(boolean expectJson) {
51 |         this.expectJson = expectJson;
52 |     }
53 | }
54 | 


--------------------------------------------------------------------------------
/src/main/java/com/github/ptgoetz/logback/kafka/formatter/MessageFormatter.java:
--------------------------------------------------------------------------------
 1 | package com.github.ptgoetz.logback.kafka.formatter;
 2 | 
 3 | import ch.qos.logback.classic.spi.ILoggingEvent;
 4 | 
 5 | /**
 6 |  * 
 7 |  * Formatter implementation that simply returns the logback message.
 8 |  * 
 9 |  * @author tgoetz
10 |  * 
11 |  */
12 | public class MessageFormatter implements Formatter {
13 | 
14 |     public String format(ILoggingEvent event) {
15 |         return event.getFormattedMessage();
16 |     }
17 | 
18 | }
19 | 


--------------------------------------------------------------------------------
/src/test/java/com/github/ptgoetz/logback/kafka/formatter/JsonFormatterTest.java:
--------------------------------------------------------------------------------
 1 | package com.github.ptgoetz.logback.kafka.formatter;
 2 | 
 3 | import org.junit.Test;
 4 | 
 5 | import static junit.framework.Assert.*;
 6 | import junit.framework.TestCase;
 7 | 
 8 | public class JsonFormatterTest extends TestCase {
 9 | 
10 |     @Test
11 |     public void testJsonFormat() {
12 |         String nonJsonMessage = "{\"level\":\"INFO\",\"logger\":\"test\",\"timestamp\":1370918376296,\"message\":\"foobar\"}";
13 |         String jsonMessage = "{\"level\":\"INFO\",\"logger\":\"test\",\"timestamp\":1370918376296,\"message\":{\"foo\":\"bar\"}}";
14 | 
15 |         // non-JSON
16 |         MockLoggingEvent event = new MockLoggingEvent(false);
17 |         JsonFormatter formatter = new JsonFormatter();
18 |         formatter.setExpectJson(false);
19 |         String json = formatter.format(event);
20 |         assertEquals(nonJsonMessage, json);
21 | 
22 |         // JSON
23 |         event = new MockLoggingEvent(true);
24 |         formatter.setExpectJson(true);
25 |         json = formatter.format(event);
26 |         assertEquals(jsonMessage, json);
27 |     }
28 | }
29 | 


--------------------------------------------------------------------------------
/src/test/java/com/github/ptgoetz/logback/kafka/formatter/MockLoggingEvent.java:
--------------------------------------------------------------------------------
 1 | package com.github.ptgoetz.logback.kafka.formatter;
 2 | 
 3 | import java.util.Map;
 4 | 
 5 | import org.slf4j.Marker;
 6 | 
 7 | import ch.qos.logback.classic.Level;
 8 | import ch.qos.logback.classic.spi.ILoggingEvent;
 9 | import ch.qos.logback.classic.spi.IThrowableProxy;
10 | import ch.qos.logback.classic.spi.LoggerContextVO;
11 | 
12 | public class MockLoggingEvent implements ILoggingEvent {
13 |     private boolean jsonMessage = false;
14 | 
15 |     public MockLoggingEvent(boolean jsonMessage) {
16 |         this.jsonMessage = jsonMessage;
17 |     }
18 | 
19 |     public Object[] getArgumentArray() {
20 |         // TODO Auto-generated method stub
21 |         return null;
22 |     }
23 | 
24 |     public StackTraceElement[] getCallerData() {
25 |         // TODO Auto-generated method stub
26 |         return null;
27 |     }
28 | 
29 |     public String getFormattedMessage() {
30 |         if (this.jsonMessage) {
31 |             return "{\"foo\":\"bar\"}";
32 |         } else {
33 |             return "foobar";
34 |         }
35 |     }
36 | 
37 |     public Level getLevel() {
38 |         return Level.INFO;
39 |     }
40 | 
41 |     public LoggerContextVO getLoggerContextVO() {
42 |         // TODO Auto-generated method stub
43 |         return null;
44 |     }
45 | 
46 |     public String getLoggerName() {
47 |         // TODO Auto-generated method stub
48 |         return "test";
49 |     }
50 | 
51 |     public Map<String, String> getMDCPropertyMap() {
52 |         // TODO Auto-generated method stub
53 |         return null;
54 |     }
55 | 
56 |     public Marker getMarker() {
57 |         // TODO Auto-generated method stub
58 |         return null;
59 |     }
60 | 
61 |     public Map<String, String> getMdc() {
62 |         // TODO Auto-generated method stub
63 |         return null;
64 |     }
65 | 
66 |     public String getMessage() {
67 |         // TODO Auto-generated method stub
68 |         return null;
69 |     }
70 | 
71 |     public String getThreadName() {
72 |         // TODO Auto-generated method stub
73 |         return null;
74 |     }
75 | 
76 |     public IThrowableProxy getThrowableProxy() {
77 |         // TODO Auto-generated method stub
78 |         return null;
79 |     }
80 | 
81 |     public long getTimeStamp() {
82 |         // TODO Auto-generated method stub
83 |         return 1370918376296L;
84 |     }
85 | 
86 |     public boolean hasCallerData() {
87 |         // TODO Auto-generated method stub
88 |         return false;
89 |     }
90 | 
91 |     public void prepareForDeferredProcessing() {
92 |         // TODO Auto-generated method stub
93 | 
94 |     }
95 | 
96 | }
97 | 


--------------------------------------------------------------------------------
/src/test/java/com/github/ptgoetz/logback/kafka/formatter/SimpleApp.java:
--------------------------------------------------------------------------------
 1 | package com.github.ptgoetz.logback.kafka.formatter;
 2 | 
 3 | import org.slf4j.Logger;
 4 | import org.slf4j.LoggerFactory;
 5 | 
 6 | public class SimpleApp {
 7 |     public static final Logger LOG = LoggerFactory.getLogger(SimpleApp.class);
 8 | 
 9 |     public static void main(String[] args) {
10 |         LOG.info("Hello {}", "World!");
11 |     }
12 | }
13 | 


--------------------------------------------------------------------------------
/src/test/resources/logback-formatter.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8" ?>
 2 | <configuration>
 3 | 	<appender name="KAFKA"
 4 | 		class="com.github.ptgoetz.logback.kafka.KafkaAppender">
 5 | 		<topic>foo</topic>
 6 | 		<zookeeperHost>localhost:2181</zookeeperHost>
 7 | 		<!-- specify a custom formatter -->
 8 | 		<formatter class="com.github.ptgoetz.logback.kafka.formatter.JsonFormatter">
 9 | 		    <!-- 
10 | 		    Whether we expect the log message to be JSON encoded or not.
11 | 		    If set to "false", the log message will be treated as a string, 
12 | 		    and wrapped in quotes. Otherwise it will be treated as a parseable
13 | 		    JSON object.
14 | 		    -->
15 | 			<expectJson>true</expectJson>
16 | 		</formatter>
17 | 	</appender>
18 | 	<root level="debug">
19 | 		<appender-ref ref="KAFKA" />
20 | 	</root>
21 | </configuration>


--------------------------------------------------------------------------------
/src/test/resources/logback.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8" ?>
 2 | <configuration>
 3 | 	<appender name="KAFKA"
 4 | 		class="com.github.ptgoetz.logback.kafka.KafkaAppender">
 5 | 		<topic>mytopic</topic>
 6 | 		<zookeeperHost>localhost:2181</zookeeperHost>
 7 | 	</appender>
 8 | 	<root level="debug">
 9 | 		<appender-ref ref="KAFKA" />
10 | 	</root>
11 | </configuration>


--------------------------------------------------------------------------------