├── .gitignore
├── jzmq-2.1.0.jar
├── src
    ├── main
    │   └── java
    │   │   └── org
    │   │       └── tlrx
    │   │           └── logback
    │   │               └── appender
    │   │                   ├── ZMQSocketOutputStream.java
    │   │                   └── ZMQSocketAppender.java
    └── test
    │   ├── resources
    │       └── logback-test.xml
    │   └── java
    │       └── org
    │           └── tlrx
    │               └── logback
    │                   └── appender
    │                       └── ZMQSocketAppenderTest.java
├── pom.xml
└── README.textile


/.gitignore:
--------------------------------------------------------------------------------
1 | slf4j-logback-zeromq.iml
2 | .idea/
3 | target/
4 | 


--------------------------------------------------------------------------------
/jzmq-2.1.0.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tlrx/slf4j-logback-zeromq/master/jzmq-2.1.0.jar


--------------------------------------------------------------------------------
/src/main/java/org/tlrx/logback/appender/ZMQSocketOutputStream.java:
--------------------------------------------------------------------------------
 1 | package org.tlrx.logback.appender;
 2 | 
 3 | import org.zeromq.ZMQ;
 4 | 
 5 | import java.io.IOException;
 6 | import java.io.OutputStream;
 7 | import java.nio.ByteBuffer;
 8 | 
 9 | /**
10 |  * @author tlrx
11 |  */
12 | public class ZMQSocketOutputStream extends OutputStream {
13 | 
14 |     private final ZMQ.Socket socket;
15 | 
16 |     public ZMQSocketOutputStream(ZMQ.Socket socket) {
17 |         this.socket = socket;
18 |     }
19 | 
20 |     @Override
21 |     public void write(int i) throws IOException {
22 |         ByteBuffer b = ByteBuffer.allocate(4);
23 |         b.putInt(i);
24 |         socket.send(b.array(), 0);
25 |     }
26 | 
27 |     @Override
28 |     public void write(byte[] bytes) throws IOException {
29 |         socket.send(bytes, 0);
30 |     }
31 | 
32 |     @Override
33 |     public void close() throws IOException {
34 |         socket.close();
35 |     }
36 | }
37 | 


--------------------------------------------------------------------------------
/src/test/resources/logback-test.xml:
--------------------------------------------------------------------------------
 1 | <configuration>
 2 | 
 3 |     <!--Example appender for Elasticsearch zeromq transport plugin
 4 |     <appender name="zeromq-connect" class="org.tlrx.logback.appender.ZMQSocketAppender">
 5 |         <type>XREQ</type>
 6 |         <connect>tcp://localhost:9700</connect>
 7 |         <encoder>
 8 |             <pattern>POST|/logs/log/|{"origin":"my_app", "level":"%-5level", "message":"%msg", "thread":"%thread", "date":"%d{HH:mm:ss.SSS}", "logger":"%logger{26}", "exception":"%ex"}</pattern>
 9 |         </encoder>
10 |     </appender>
11 |      -->
12 | 
13 |     <appender name="zeromq-bind" class="org.tlrx.logback.appender.ZMQSocketAppender">
14 |         <type>PUB</type>
15 |         <bind>tcp://*:9797</bind>
16 |         <encoder>
17 |             <pattern>%-5level %logger{26} - %msg</pattern>
18 |         </encoder>
19 |     </appender>
20 | 
21 |     <root level="DEBUG">
22 |         <appender-ref ref="zeromq-bind"/>
23 |     </root>
24 | </configuration>


--------------------------------------------------------------------------------
/src/test/java/org/tlrx/logback/appender/ZMQSocketAppenderTest.java:
--------------------------------------------------------------------------------
 1 | package org.tlrx.logback.appender;
 2 | 
 3 | import junit.framework.Assert;
 4 | import junit.framework.TestCase;
 5 | import org.junit.After;
 6 | import org.junit.Before;
 7 | import org.junit.Test;
 8 | import org.slf4j.Logger;
 9 | import org.slf4j.LoggerFactory;
10 | import org.zeromq.ZMQ;
11 | 
12 | /**
13 |  * @author tlrx
14 |  */
15 | public class ZMQSocketAppenderTest extends TestCase {
16 | 
17 |     private static final String CONNECT = "tcp://127.0.0.1:9797";
18 | 
19 |     ZMQ.Context context;
20 |     ZMQ.Socket subscriber;
21 | 
22 |     @Before
23 |     public void setUp() {
24 |         context = ZMQ.context(1);
25 |         subscriber = context.socket(ZMQ.SUB);
26 |         subscriber.connect(CONNECT);
27 |         subscriber.subscribe("ERROR".getBytes());
28 |     }
29 | 
30 |     @After
31 |     public void tearDown() {
32 |         subscriber.close();
33 |         context.term();
34 |     }
35 | 
36 |     private void receiveAndAssert(String expected) {
37 | 
38 |         byte[] rec = subscriber.recv(ZMQ.NOBLOCK);
39 | 
40 |         if (rec != null) {
41 |             String received = new String(rec);
42 |             assertEquals(expected, received);
43 |         } else if (expected != null) {
44 |             Assert.fail("Nothing received!");
45 |         }
46 |     }
47 | 
48 | 
49 |     @Test
50 |     public void test() {
51 | 
52 |         // Creates a logger (and bind a PUB socket)
53 |         Logger logger = LoggerFactory.getLogger(ZMQSocketAppenderTest.class);
54 | 
55 |         for (int count = 0; count < 10; count++) {
56 | 
57 |             if ((count % 2) == 1) {
58 |                 logger.error("Log message bar {}", count);
59 |             } else {
60 |                 logger.info("Log message foo {}", count);
61 |             }
62 | 
63 |             try {
64 |                 Thread.sleep(500);
65 |             } catch (InterruptedException e) {
66 |                 Assert.fail();
67 |             }
68 |         }
69 | 
70 |         receiveAndAssert("ERROR o.t.l.a.ZMQSocketAppenderTest - Log message bar 1");
71 |         receiveAndAssert("ERROR o.t.l.a.ZMQSocketAppenderTest - Log message bar 3");
72 |         receiveAndAssert("ERROR o.t.l.a.ZMQSocketAppenderTest - Log message bar 5");
73 |         receiveAndAssert("ERROR o.t.l.a.ZMQSocketAppenderTest - Log message bar 7");
74 |         receiveAndAssert("ERROR o.t.l.a.ZMQSocketAppenderTest - Log message bar 9");
75 |     }
76 | }
77 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
 1 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 2 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 3 |     <modelVersion>4.0.0</modelVersion>
 4 | 
 5 |     <groupId>org.tlrx</groupId>
 6 |     <artifactId>slf4j-logback-zeromq</artifactId>
 7 |     <version>0.0.1-SNAPSHOT</version>
 8 |     <packaging>jar</packaging>
 9 | 
10 |     <name>ZeroMQ logger for SLF4J and Logback</name>
11 | 
12 |     <properties>
13 |         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
14 |     </properties>
15 | 
16 |     <developers>
17 |         <developer>
18 |             <name>tlrx</name>
19 |             <email>tlrx.dev@gmail.com</email>
20 |             <url>http://twitter.com/tlrx</url>
21 |         </developer>
22 |     </developers>
23 | 
24 |     <licenses>
25 |       <license>
26 |         <name>The Apache Software License, Version 2.0</name>
27 |         <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
28 |         <distribution>repo</distribution>
29 |         <comments>A business-friendly OSS license</comments>
30 |       </license>
31 |     </licenses>
32 | 
33 |     <dependencies>
34 |         <!--Only need Logback Core to compile -->
35 |         <dependency>
36 |             <groupId>ch.qos.logback</groupId>
37 |             <artifactId>logback-core</artifactId>
38 |             <version>1.0.0</version>
39 |             <scope>compile</scope>
40 |         </dependency>
41 |         <!-- ZeroMQ Java Binding lib -->
42 |         <dependency>
43 |             <groupId>org.elasticsearch.zeromq</groupId>
44 |             <artifactId>jzmq</artifactId>
45 |             <version>2.1.0</version>
46 |             <scope>system</scope>
47 |             <systemPath>${project.basedir}/jzmq-2.1.0.jar</systemPath>
48 |         </dependency>
49 |         <!-- Test dependencies -->
50 |         <dependency>
51 |             <groupId>junit</groupId>
52 |             <artifactId>junit</artifactId>
53 |             <version>4.10</version>
54 |             <scope>test</scope>
55 |         </dependency>
56 |         <!--Need Logback Classic to be run -->
57 |         <dependency>
58 |             <groupId>ch.qos.logback</groupId>
59 |             <artifactId>logback-classic</artifactId>
60 |             <version>1.0.0</version>
61 |             <scope>test</scope>
62 |         </dependency>
63 |     </dependencies>
64 |     <build>
65 |         <plugins>
66 |             <plugin>
67 |                 <groupId>org.apache.maven.plugins</groupId>
68 |                 <artifactId>maven-compiler-plugin</artifactId>
69 |                 <version>2.3.2</version>
70 |                 <configuration>
71 |                     <source>1.6</source>
72 |                     <target>1.6</target>
73 |                 </configuration>
74 |             </plugin>
75 |         </plugins>
76 |     </build>
77 | </project>
78 | 


--------------------------------------------------------------------------------
/src/main/java/org/tlrx/logback/appender/ZMQSocketAppender.java:
--------------------------------------------------------------------------------
  1 | package org.tlrx.logback.appender;
  2 | 
  3 | import ch.qos.logback.core.OutputStreamAppender;
  4 | import org.zeromq.ZMQ;
  5 | 
  6 | /**
  7 |  * A ØMQ socket appender for Logback
  8 |  *
  9 |  * @author tlrx
 10 |  */
 11 | public class ZMQSocketAppender<E> extends OutputStreamAppender<E> {
 12 | 
 13 |     private static final ZMQ.Context context = ZMQ.context(1);
 14 | 
 15 |     private String type;
 16 |     private String bind;
 17 |     private String connect;
 18 |     private long highWaterMark = 10000L;
 19 | 
 20 |     private enum METHOD {
 21 |         BIND,
 22 |         CONNECT
 23 |     }
 24 | 
 25 |     private ZMQSocketOutputStream outputStream;
 26 | 
 27 |     public String getBind() {
 28 |         return bind;
 29 |     }
 30 | 
 31 |     public void setBind(String bind) {
 32 |         this.bind = bind;
 33 |     }
 34 | 
 35 |     public String getConnect() {
 36 |         return connect;
 37 |     }
 38 | 
 39 |     public void setConnect(String connect) {
 40 |         this.connect = connect;
 41 |     }
 42 | 
 43 |     public String getType() {
 44 |         return type;
 45 |     }
 46 | 
 47 |     public void setType(String type) {
 48 |         this.type = type;
 49 |     }
 50 | 
 51 |     public long getHighWaterMark() {
 52 |         return highWaterMark;
 53 |     }
 54 | 
 55 |     public void setHighWaterMark(long highWaterMark) {
 56 |         this.highWaterMark = highWaterMark;
 57 |     }
 58 | 
 59 |     @Override
 60 |     public void start() {
 61 | 
 62 |         boolean error = false;
 63 | 
 64 |         // Determines the socket type
 65 |         int socketType;
 66 | 
 67 |         if ("sub".equalsIgnoreCase(type)) {
 68 |             socketType = ZMQ.SUB;
 69 |         } else if ("req".equalsIgnoreCase(type)) {
 70 |             socketType = ZMQ.REQ;
 71 |         } else if ("xreq".equalsIgnoreCase(type)) {
 72 |             socketType = ZMQ.XREQ;
 73 |         } else if ("pub".equalsIgnoreCase(type)) {
 74 |             socketType = ZMQ.PUB;
 75 |         } else {
 76 |             addWarn("[" + type + "] should be one of [REQ, XREQ, SUB]" + ", using default ØMQ socket type, PUB by default.");
 77 |             socketType = ZMQ.PUB;
 78 |         }
 79 | 
 80 |         // Determines the socket connection method (bind or connect)
 81 |         METHOD method = METHOD.BIND;
 82 |         String address = null;
 83 | 
 84 |         switch (socketType) {
 85 |             case ZMQ.SUB:
 86 |                 // Sub sockets can connect or bind
 87 |                 if (connect != null) {
 88 |                     method = METHOD.CONNECT;
 89 |                     address = connect;
 90 |                 } else if (bind != null) {
 91 |                     method = METHOD.BIND;
 92 |                     address = bind;
 93 |                 } else {
 94 |                     addError("Either <connect> or <bind> must not be null for SUB sockets.");
 95 |                     error = true;
 96 |                 }
 97 |                 break;
 98 | 
 99 |             case ZMQ.PUB:
100 |                 // Pub sockets can connect or bind
101 |                 if (bind != null) {
102 |                     method = METHOD.BIND;
103 |                     address = bind;
104 |                 } else if (connect != null) {
105 |                     method = METHOD.CONNECT;
106 |                     address = connect;
107 |                 } else {
108 |                     addError("Either <connect> or <bind> must not be null for PUB sockets.");
109 |                     error = true;
110 |                 }
111 |                 break;
112 | 
113 |             case ZMQ.REQ:
114 |                 // Req sockets can  only connect
115 |                 if (connect != null) {
116 |                     method = METHOD.CONNECT;
117 |                     address = connect;
118 |                 } else {
119 |                     addError("<connect> must not be null for REQ sockets.");
120 |                     error = true;
121 |                 }
122 |                 break;
123 | 
124 |             case ZMQ.XREQ:
125 |                 // Req sockets can  only connect
126 |                 if (connect != null) {
127 |                     method = METHOD.CONNECT;
128 |                     address = connect;
129 |                 } else {
130 |                     addError("<connect> must not be null for XREQ sockets.");
131 |                     error = true;
132 |                 }
133 |                 break;
134 | 
135 |             default:
136 |                 // Should not happen
137 |                 addError("Unexpected error, please check your configuration");
138 |                 error = true;
139 |                 break;
140 |         }
141 | 
142 |         if(highWaterMark<0){
143 |             addError("<highWaterMark> must not be >= 0");
144 |         }
145 | 
146 |         if (!error) {
147 |             // Creates the zmq socket
148 |             ZMQ.Socket socket = context.socket(socketType);
149 | 
150 |             socket.setHWM(highWaterMark);
151 | 
152 |             if (method == METHOD.CONNECT) {
153 |                 socket.connect(address);
154 |             } else {
155 |                 socket.bind(address);
156 |             }
157 | 
158 |             // Set the socket as an OutputStream
159 |             outputStream = new ZMQSocketOutputStream(socket);
160 |             setOutputStream(outputStream);
161 | 
162 |             super.start();
163 |         } else {
164 |             addError("Socket " + getName() + " has configuration errors and is not started!");
165 |         }
166 |     }
167 | 
168 |     @Override
169 |     public void stop() {
170 |         super.stop();
171 |     }
172 | }
173 | 


--------------------------------------------------------------------------------
/README.textile:
--------------------------------------------------------------------------------
  1 | h1. Logback appender for ØMQ message transport layer
  2 | 
  3 | This project must be used with the "Logback":http://logback.qos.ch/ project in order to produce log messages over "ZeroMQ":http://www.zeromq.org/ messaging library.
  4 | 
  5 | This way, the application logs can be sent to remote systems / can be receive by remote systems in any language supported by ØMQ.
  6 | 
  7 | 
  8 | h2. Installation
  9 | 
 10 | h3. Requirements
 11 | 
 12 | Before installing and using this appender with Logback, you need to install ZeroMQ Java Binding library (2.1) on your system. This binding uses native library to work. 
 13 | 
 14 | The "ZeroMQ":http//www.zeromq.org/ website is a great place to start installing the libraries, specially the "Java binding page":http://www.zeromq.org/bindings:java
 15 | 
 16 | If you want to develop or modify this plugin, I encourage you to read the "ØMQ - The Guide":http://zguide.zeromq.org/page:all which is very well documented.
 17 | 
 18 | You must download slf4j-api.jar, logback-core.jar and logback-classic.jar on their respective project pages "SLF4J":http://www.slf4j.org/ and "Logback":http://logback.qos.ch/ .
 19 | 
 20 | 
 21 | h3. Installation
 22 | 
 23 | Put the following jar in the classpath of your Java application:
 24 | 
 25 | <pre>
 26 | slf4j-logback-zeromq-0.0.1.jar
 27 | </pre>
 28 | 
 29 | You must also put in the classpath the ØMQ Java binding compiled for your system:
 30 | 
 31 | <pre>
 32 | jzmq-2.1.0.jar
 33 | </pre>
 34 | 
 35 | Finally, put the SLF4J and Logback jars in the classpath:
 36 | 
 37 | <pre>
 38 | slf4j-api-1.6.4.jar
 39 | logback-core-1.0.0.jar
 40 | logback-classic-1.0.0.jar
 41 | </pre>
 42 | 
 43 | 
 44 | h3. Configuration
 45 | 
 46 | The configuration of the ØMQ appender is done in the usual "Logback configuration file":http://logback.qos.ch/manual/configuration.html
 47 | 
 48 | The appender must be declared in the logback*.xml file:
 49 | 
 50 | <pre>
 51 | 	<appender name="zeromq-bind" class="org.tlrx.logback.appender.ZMQSocketAppender">
 52 | 		<!-- Define the ØMQ socket type -->
 53 | 		<type>PUB</type>
 54 | 		<!-- Only one of bind+connect must be declared -->
 55 | 		<bind>tcp://*:9797</bind>
 56 | 		<connect>tcp:localhost:9700</connect>
 57 | 		<encoder>
 58 | 			<!-- Message format -->
 59 | 			<pattern>%-5level %logger{26} - %msg</pattern>
 60 | 		</encoder>
 61 | 	</appender>
 62 | </pre>
 63 | 
 64 | Where:
 65 | 
 66 | |_. Property            |_. Description|
 67 | | @<type>@              | Define the ØMQ socket type, value can be: *PUB*, *SUB*, *REQ*,  *XREQ*. The default value is set to PUB |
 68 | | @<bind>@              | The ØMQ socket will connect to the given address using "bind()":http://api.zeromq.org/2-1:zmq-bind method. Can be used only if socket is PUB or SUB type |
 69 | | @<connect>@           | The ØMQ socket will connect to the given address using "connect()":http://api.zeromq.org/2-1:zmq-connect method  |
 70 | | @<pattern>@		| The pattern defines the format of the ØMQ message that will be sent by the appender. The format used the same PatternLayout of the default Logback "ConsoleAppender":http://logback.qos.ch/manual/layouts.html#ClassicPatternLayout|
 71 | | @<highWaterMark>@     | A high water mark for the socket, after which messages will be discarded (if they're not sent). This prevents infinite buffering (and ergo infinite RAM use) if the thing you are sending to is down. Defaults to 10000 messages |
 72 | 
 73 | 
 74 | h2. Examples of use
 75 | 
 76 | h3. Publish log messages over TCP
 77 | 
 78 | h4. Description
 79 | 
 80 | In this example, the log messages will be encapsualted in a ØMQ message and published by the appender on a given address.
 81 | 
 82 | Remote applications (subscribers) can connect to the ØMQ publisher socket and start receiving messages. The subscribers can also filter by log level the messages they want to receive.
 83 | 
 84 | h4. Configuration
 85 | 
 86 | The configuration of the Logback appender defines a Publisher socket:
 87 | 
 88 | <pre>
 89 |  <appender name="zeromq-bind" class="org.tlrx.logback.appender.ZMQSocketAppender">
 90 |         <type>PUB</type>
 91 |         <bind>tcp://*:9797</bind>
 92 |         <encoder>
 93 |             <pattern>%-5level %logger{26} - %msg</pattern>
 94 |         </encoder>
 95 |     </appender>
 96 | </pre>
 97 | 
 98 | h4. At runtime
 99 | 
100 | Once started, the appender sends log messages like this:
101 | 
102 | <pre>
103 | DEBUG org.eclipse.jetty.util.log - doSelect true
104 | </pre>
105 | 
106 | Remote application can now connect to the publisher address to receive log messages. The @ZMQSocketAppenderTest@ class gives an example of how to connect to the publisher and receive only ERROR messages.
107 | 
108 | If no subscribers is connected, the publisher will drop the message (this is a default ØMQ behavior).
109 | 
110 | 
111 | h3. Index log messages in "Elasticsearch":http://www.elasticsearch.org/
112 | 
113 | h4. Description
114 | 
115 | In this example, the ØMQ appender is configured to send log messages to the "ØMQ transport plugin for Elasticsearch":https://github.com/tlrx/transport-zeromq 
116 | 
117 | This way, it is possible to index and search log messages with Elasticsearch features.
118 | 
119 | h4. Configuration
120 | 
121 | The configuration of the Logback appender defines a XREQ socket:
122 | 
123 | <pre>
124 | <configuration>
125 | ...
126 |     <appender name="zeromq-connect" class="org.tlrx.logback.appender.ZMQSocketAppender">
127 |         <type>XREQ</type>
128 |         <connect>tcp://localhost:9700</connect>
129 |         <encoder>
130 |             <pattern>POST|/logs/log/|{"origin":"my_app", "level":"%-5level", "message":"%msg", "thread":"%thread", "date":"%d{HH:mm:ss.SSS}", "logger":"%logger{26}", "exception":"%ex"}</pattern>
131 |         </encoder>
132 |     </appender>
133 | ...
134 | </configuration>
135 | 
136 | </pre>
137 | 
138 | The ØMQ transport plugin for Elasticsearch listen by default on the @<tcp://*:9700>@ address.
139 | 
140 | In this configuration, the pattern for log messages reflects the format expected by the Elasticsearch plugin.
141 | 
142 | h4. At runtime
143 | 
144 | Once started, the appender sends log messages to the Elasticsearch plugin, which index them in the *logs* index.
145 | 
146 |  
147 | 


--------------------------------------------------------------------------------