├── .gitignore
├── src
    ├── test
    │   ├── java
    │   │   └── .gitkeep
    │   └── resources
    │   │   └── .gitkeep
    └── main
    │   ├── resources
    │       ├── .gitkeep
    │       └── log4j.properties
    │   └── java
    │       └── com
    │           └── m2mci
    │               └── mqttKafkaBridge
    │                   ├── CommandLineParser.java
    │                   └── Bridge.java
├── .settings
    ├── org.eclipse.m2e.core.prefs
    ├── org.eclipse.core.resources.prefs
    └── org.eclipse.jdt.core.prefs
├── .project
├── .classpath
├── README.md
└── pom.xml


/.gitignore:
--------------------------------------------------------------------------------
1 | target/


--------------------------------------------------------------------------------
/src/test/java/.gitkeep:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/src/main/resources/.gitkeep:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/src/test/resources/.gitkeep:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/.settings/org.eclipse.m2e.core.prefs:
--------------------------------------------------------------------------------
1 | activeProfiles=
2 | eclipse.preferences.version=1
3 | resolveWorkspaceProjects=false
4 | version=1
5 | 


--------------------------------------------------------------------------------
/.settings/org.eclipse.core.resources.prefs:
--------------------------------------------------------------------------------
1 | eclipse.preferences.version=1
2 | encoding//src/main/java=UTF-8
3 | encoding//src/main/resources=UTF-8
4 | encoding//src/test/java=UTF-8
5 | encoding//src/test/resources=UTF-8
6 | encoding/<project>=UTF-8
7 | 


--------------------------------------------------------------------------------
/.settings/org.eclipse.jdt.core.prefs:
--------------------------------------------------------------------------------
1 | eclipse.preferences.version=1
2 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
3 | org.eclipse.jdt.core.compiler.compliance=1.7
4 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
5 | org.eclipse.jdt.core.compiler.source=1.7
6 | 


--------------------------------------------------------------------------------
/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | # Set root logger level to INFO and its only appender to A1.
2 | log4j.rootLogger=INFO, A1
3 | 
4 | # A1 is set to be a ConsoleAppender.
5 | log4j.appender.A1=org.apache.log4j.ConsoleAppender
6 | 
7 | # A1 uses PatternLayout.
8 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout
9 | log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n


--------------------------------------------------------------------------------
/.project:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <projectDescription>
 3 | 	<name>mqttKafkaBridge</name>
 4 | 	<comment></comment>
 5 | 	<projects>
 6 | 	</projects>
 7 | 	<buildSpec>
 8 | 		<buildCommand>
 9 | 			<name>org.eclipse.jdt.core.javabuilder</name>
10 | 			<arguments>
11 | 			</arguments>
12 | 		</buildCommand>
13 | 		<buildCommand>
14 | 			<name>org.eclipse.m2e.core.maven2Builder</name>
15 | 			<arguments>
16 | 			</arguments>
17 | 		</buildCommand>
18 | 	</buildSpec>
19 | 	<natures>
20 | 		<nature>org.eclipse.jdt.core.javanature</nature>
21 | 		<nature>org.eclipse.m2e.core.maven2Nature</nature>
22 | 	</natures>
23 | </projectDescription>
24 | 


--------------------------------------------------------------------------------
/.classpath:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <classpath>
 3 | 	<classpathentry kind="src" output="target/classes" path="src/main/java">
 4 | 		<attributes>
 5 | 			<attribute name="optional" value="true"/>
 6 | 			<attribute name="maven.pomderived" value="true"/>
 7 | 		</attributes>
 8 | 	</classpathentry>
 9 | 	<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
10 | 		<attributes>
11 | 			<attribute name="maven.pomderived" value="true"/>
12 | 		</attributes>
13 | 	</classpathentry>
14 | 	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
15 | 		<attributes>
16 | 			<attribute name="optional" value="true"/>
17 | 			<attribute name="maven.pomderived" value="true"/>
18 | 		</attributes>
19 | 	</classpathentry>
20 | 	<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
21 | 		<attributes>
22 | 			<attribute name="maven.pomderived" value="true"/>
23 | 		</attributes>
24 | 	</classpathentry>
25 | 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
26 | 		<attributes>
27 | 			<attribute name="maven.pomderived" value="true"/>
28 | 		</attributes>
29 | 	</classpathentry>
30 | 	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
31 | 		<attributes>
32 | 			<attribute name="maven.pomderived" value="true"/>
33 | 		</attributes>
34 | 	</classpathentry>
35 | 	<classpathentry kind="output" path="target/classes"/>
36 | </classpath>
37 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # mqttKafkaBridge
 2 | 
 3 | Bridge which consumes MQTT messages and republishes them on Kafka on the same topic.
 4 | 
 5 | ## Usage
 6 | 
 7 |     $ java -jar mqttKafkaBridge.jar [options...]
 8 | 
 9 | Where `options` are:
10 | 
11 |     --help (-h)               : Show help
12 |     --id VAL                  : MQTT Client ID
13 |     --topics VAL              : MQTT topic filters (comma-separated)
14 |     --uri VAL                 : MQTT Server URI
15 |     --brokerlist (-b) VAL     : Kafka Broker List (comma-separated)
16 | 
17 | If you don't specify any command-line options, it uses the following defaults:
18 | 
19 |     id:      mqttKafkaBridge
20 |     topics:  '#' (all topics)
21 |     uri:     tcp://localhost:1883
22 |     b:       localhost:9092
23 | 
24 | ***Note***: you can't run more than one bridge using the default settings, since two clients cannot connect to the same MQTT server with the same client ID. Additionally, you will get multiple messages published to Kafka for each message published to MQTT. If you wish to run multiple instances, you'll need to divide up the topics among the instances, and make sure to give them different IDs.
25 | 
26 | ## Logging
27 | `mqttKafkaBridge` uses [log4j](http://logging.apache.org/log4j/2.x/) for logging, as do the [Paho](http://www.eclipse.org/paho/) and [Kafka](http://kafka.apache.org/) libraries it uses. There is a default `log4j.properties` file packaged with the jar which simply prints all messages of level `INFO` or greater to the console. If you want to customize logging, simply create your own `log4j.properties` file, and start up `mqttKafkaBridge` as follows:
28 | 
29 |     $ java -Dlog4j.configuration=file:///path/to/log4j.properties -jar mqttKafkaBridge.jar [options...]
30 | 
31 | 


--------------------------------------------------------------------------------
/src/main/java/com/m2mci/mqttKafkaBridge/CommandLineParser.java:
--------------------------------------------------------------------------------
 1 | package com.m2mci.mqttKafkaBridge;
 2 | 
 3 | import java.io.OutputStream;
 4 | import java.io.PrintStream;
 5 | 
 6 | import org.kohsuke.args4j.CmdLineException;
 7 | import org.kohsuke.args4j.CmdLineParser;
 8 | import org.kohsuke.args4j.Option;
 9 | 
10 | public class CommandLineParser {
11 | 	private static final String ALL_MQTT_TOPICS = "#";
12 | 	private static final String DEFAULT_BROKER_LIST = "localhost:9092";
13 | 	private static final String DEFAULT_MQTT_SERVER_URI = "tcp://localhost:1883";
14 | 
15 | 	@Option(name="--id", usage="MQTT Client ID")
16 | 	private String clientId = "mqttKafkaBridge";
17 | 
18 | 	@Option(name="--uri", usage="MQTT Server URI")
19 | 	private String serverURI = DEFAULT_MQTT_SERVER_URI;
20 | 
21 | 	@Option(name="--brokerlist", aliases="-b", usage="Broker list (comma-separated)")
22 | 	private String brokerList = DEFAULT_BROKER_LIST;
23 | 	
24 | 	@Option(name="--topics", usage="MQTT topic filters (comma-separated)")
25 | 	private String mqttTopicFilters = ALL_MQTT_TOPICS;
26 | 	
27 | 	@Option(name="--help", aliases="-h", usage="Show help")
28 | 	private boolean showHelp = false;
29 | 	
30 | 	private CmdLineParser parser = new CmdLineParser(this);
31 | 	
32 | 	public String getClientId() {
33 | 		return clientId;
34 | 	}
35 | 
36 | 	public String getServerURI() {
37 | 		return serverURI;
38 | 	}
39 | 
40 | 	public String getBrokerList() {
41 | 		return brokerList;
42 | 	}
43 | 
44 | 	public String[] getMqttTopicFilters() {
45 | 		return mqttTopicFilters.split(",");
46 | 	}
47 | 
48 | 	public void parse(String[] args) throws CmdLineException {
49 | 		parser.parseArgument(args);
50 | 		if (showHelp) {
51 | 			printUsage(System.out);
52 | 			System.exit(0);
53 | 		}
54 | 	}
55 | 
56 | 	public void printUsage(OutputStream out) {
57 | 		PrintStream stream = new PrintStream(out);
58 | 		stream.println("java " + Bridge.class.getName() + " [options...]");
59 | 		parser.printUsage(out);
60 | 	}
61 | }
62 | 


--------------------------------------------------------------------------------
/src/main/java/com/m2mci/mqttKafkaBridge/Bridge.java:
--------------------------------------------------------------------------------
 1 | package com.m2mci.mqttKafkaBridge;
 2 | 
 3 | import java.util.Properties;
 4 | 
 5 | import kafka.javaapi.producer.Producer;
 6 | import kafka.producer.KeyedMessage;
 7 | import kafka.producer.ProducerConfig;
 8 | 
 9 | import org.apache.log4j.Logger;
10 | import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
11 | import org.eclipse.paho.client.mqttv3.IMqttToken;
12 | import org.eclipse.paho.client.mqttv3.MqttAsyncClient;
13 | import org.eclipse.paho.client.mqttv3.MqttCallback;
14 | import org.eclipse.paho.client.mqttv3.MqttException;
15 | import org.eclipse.paho.client.mqttv3.MqttMessage;
16 | import org.kohsuke.args4j.CmdLineException;
17 | 
18 | public class Bridge implements MqttCallback {
19 | 	private Logger logger = Logger.getLogger(this.getClass().getName());
20 | 	private MqttAsyncClient mqtt;
21 | 	private Producer<String, String> kafkaProducer;
22 | 	
23 | 	private void connect(String serverURI, String clientId, String brokerList) throws MqttException {
24 | 		mqtt = new MqttAsyncClient(serverURI, clientId);
25 | 		mqtt.setCallback(this);
26 | 		IMqttToken token = mqtt.connect();
27 | 		Properties props = new Properties();
28 |                 props.put("metadata.broker.list", brokerList);
29 | 		props.put("serializer.class", "kafka.serializer.StringEncoder");
30 | 		ProducerConfig config = new ProducerConfig(props);
31 | 		kafkaProducer = new Producer<String, String>(config);
32 | 		token.waitForCompletion();
33 | 		logger.info("Connected to MQTT and Kafka");
34 | 	}
35 | 
36 | 	private void reconnect() throws MqttException {
37 | 		IMqttToken token = mqtt.connect();
38 | 		token.waitForCompletion();
39 | 	}
40 | 	
41 | 	private void subscribe(String[] mqttTopicFilters) throws MqttException {
42 | 		int[] qos = new int[mqttTopicFilters.length];
43 | 		for (int i = 0; i < qos.length; ++i) {
44 | 			qos[i] = 0;
45 | 		}
46 | 		mqtt.subscribe(mqttTopicFilters, qos);
47 | 	}
48 | 
49 | 	@Override
50 | 	public void connectionLost(Throwable cause) {
51 | 		logger.warn("Lost connection to MQTT server", cause);
52 | 		while (true) {
53 | 			try {
54 | 				logger.info("Attempting to reconnect to MQTT server");
55 | 				reconnect();
56 | 				logger.info("Reconnected to MQTT server, resuming");
57 | 				return;
58 | 			} catch (MqttException e) {
59 | 				logger.warn("Reconnect failed, retrying in 10 seconds", e);
60 | 			}
61 | 			try {
62 | 				Thread.sleep(10000);
63 | 			} catch (InterruptedException e) {
64 | 			}
65 | 		}
66 | 	}
67 | 
68 | 	@Override
69 | 	public void deliveryComplete(IMqttDeliveryToken token) {
70 | 		// TODO Auto-generated method stub
71 | 		
72 | 	}
73 | 
74 | 	@Override
75 | 	public void messageArrived(String topic, MqttMessage message) throws Exception {
76 | 		byte[] payload = message.getPayload();
77 |                 kafkaProducer.send(new KeyedMessage<String, String>(topic, new String(payload)));
78 | 	}
79 | 
80 | 	/**
81 | 	 * @param args
82 | 	 */
83 | 	public static void main(String[] args) {
84 | 		CommandLineParser parser = null;
85 | 		try {
86 | 			parser = new CommandLineParser();
87 | 			parser.parse(args);
88 | 			Bridge bridge = new Bridge();
89 | 			bridge.connect(parser.getServerURI(), parser.getClientId(), parser.getBrokerList());
90 | 			bridge.subscribe(parser.getMqttTopicFilters());
91 | 		} catch (MqttException e) {
92 | 			e.printStackTrace(System.err);
93 | 		} catch (CmdLineException e) {
94 | 			System.err.println(e.getMessage());
95 | 			parser.printUsage(System.err);
96 | 		}
97 | 	}
98 | }
99 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  3 | 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  4 | 	<modelVersion>4.0.0</modelVersion>
  5 | 	<groupId>com.m2mci</groupId>
  6 | 	<artifactId>mqttKafkaBridge</artifactId>
  7 | 	<version>0.1.0</version>
  8 | 
  9 | 	<!-- This should point to the location of the remote git repository modify 
 10 | 		this as necessary. This is needed for doing releases -->
 11 | 	<scm>
 12 | 	<developerConnection>
 13 |             scm:git:https://bitbucket.org/m2mconnections/mqttKafkaBridge.git
 14 |         </developerConnection>
 15 | 	</scm>
 16 | 
 17 |         <repositories>
 18 |            <repository>
 19 |               <id>Eclipse Paho Repo</id>
 20 |               <url>https://repo.eclipse.org/content/repositories/paho-releases/</url>
 21 |            </repository>
 22 |         </repositories>
 23 | 
 24 | 	<!-- Dependencies -->
 25 | 	<dependencies>
 26 | 		<dependency>
 27 | 			<groupId>junit</groupId>
 28 | 			<artifactId>junit</artifactId>
 29 | 			<version>4.11</version>
 30 | 		</dependency>
 31 | 		<dependency>
 32 | 			<groupId>org.mockito</groupId>
 33 | 			<artifactId>mockito-core</artifactId>
 34 | 			<version>1.9.5</version>
 35 | 		</dependency>
 36 |                 <dependency>
 37 |                         <groupId>org.eclipse.paho</groupId>
 38 |                         <artifactId>org.eclipse.paho.client.mqttv3</artifactId>
 39 |                         <version>1.0.2</version>
 40 |                 </dependency>
 41 |                 <dependency>
 42 |                         <groupId>org.apache.kafka</groupId>
 43 |                         <artifactId>kafka_2.10</artifactId>
 44 |                         <version>0.8.2.1</version>
 45 |                 </dependency>
 46 | 		<dependency>
 47 | 			<groupId>org.scala-lang</groupId>
 48 | 			<artifactId>scala-library</artifactId>
 49 | 			<version>2.10.4</version>
 50 | 		</dependency>
 51 | 		<dependency>
 52 | 			<groupId>args4j</groupId>
 53 | 			<artifactId>args4j</artifactId>
 54 | 			<version>2.0.25</version>
 55 | 		</dependency>
 56 | 		<dependency>
 57 | 			<groupId>com.101tec</groupId>
 58 | 			<artifactId>zkclient</artifactId>
 59 | 			<version>0.3</version>
 60 | 		</dependency>
 61 | 	</dependencies>
 62 | 
 63 | 	<!-- Plugins -->
 64 | 	<build>
 65 | 		<plugins>
 66 | 			<!-- Set the compiler plugin - override default JDK version -->
 67 | 			<plugin>
 68 | 				<groupId>org.apache.maven.plugins</groupId>
 69 | 				<artifactId>maven-compiler-plugin</artifactId>
 70 | 				<version>3.1</version>
 71 | 				<configuration>
 72 | 					<source>1.7</source>
 73 | 					<target>1.7</target>
 74 | 				</configuration>
 75 | 			</plugin>
 76 | 			<!-- These plugins are needed for doing releases -->
 77 | 			<plugin>
 78 | 				<groupId>org.apache.maven.plugins</groupId>
 79 | 				<artifactId>maven-release-plugin</artifactId>
 80 | 				<version>2.4.1</version>
 81 | 			</plugin>
 82 | 			<plugin>
 83 | 				<groupId>org.apache.maven.plugins</groupId>
 84 | 				<artifactId>maven-javadoc-plugin</artifactId>
 85 | 				<version>2.9</version>
 86 | 			</plugin>
 87 | 			<plugin>
 88 | 				<groupId>org.apache.maven.plugins</groupId>
 89 | 				<artifactId>maven-deploy-plugin</artifactId>
 90 | 				<version>2.7</version>
 91 | 			</plugin>
 92 | 			<plugin>
 93 | 				<groupId>org.apache.maven.plugins</groupId>
 94 | 				<artifactId>maven-source-plugin</artifactId>
 95 | 				<version>2.2.1</version>
 96 | 			</plugin>
 97 | 			<!-- Code coverage plugin -->
 98 | 			<plugin>
 99 | 				<groupId>org.jacoco</groupId>
100 | 				<artifactId>jacoco-maven-plugin</artifactId>
101 | 				<version>0.6.2.201302030002</version>
102 | 			</plugin>
103 |                         <plugin>
104 |                                 <artifactId>maven-assembly-plugin</artifactId>
105 |                                 <version>2.5.4</version>
106 |                                 <configuration>
107 |                                      <archive>
108 |                                          <manifest>
109 |                                                 <mainClass>com.m2mci.mqttKafkaBridge.Bridge</mainClass>
110 |                                          </manifest>
111 |                                     </archive>
112 |                                     <descriptorRefs>
113 |                                           <descriptorRef>jar-with-dependencies</descriptorRef>
114 |                                     </descriptorRefs>
115 |                                 </configuration>
116 | 			</plugin>
117 | 		</plugins>
118 | 	</build>
119 | 
120 | 	<!-- Properties -->
121 | 	<properties>
122 | 		<!-- Set the source and output encodings, to avoid warnings about the encodings 
123 | 			not being set -->
124 | 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
125 | 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
126 | 	</properties>
127 | </project>
128 | 


--------------------------------------------------------------------------------