├── .classpath
├── .gitignore
├── .project
├── .settings
    ├── org.eclipse.core.resources.prefs
    ├── org.eclipse.jdt.core.prefs
    └── org.eclipse.m2e.core.prefs
├── README.md
├── pom.xml
└── src
    ├── main
        ├── java
        │   └── com
        │   │   └── m2mci
        │   │       └── mqttKafkaBridge
        │   │           ├── Bridge.java
        │   │           └── CommandLineParser.java
        └── resources
        │   ├── .gitkeep
        │   └── log4j.properties
    └── test
        ├── java
            └── .gitkeep
        └── resources
            └── .gitkeep


/.classpath:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <classpath>
 3 | 	<classpathentry kind="src" output="target/classes" path="src/main/java">
 4 | 		<attributes>
 5 | 			<attribute name="optional" value="true"/>
 6 | 			<attribute name="maven.pomderived" value="true"/>
 7 | 		</attributes>
 8 | 	</classpathentry>
 9 | 	<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
10 | 		<attributes>
11 | 			<attribute name="maven.pomderived" value="true"/>
12 | 		</attributes>
13 | 	</classpathentry>
14 | 	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
15 | 		<attributes>
16 | 			<attribute name="optional" value="true"/>
17 | 			<attribute name="maven.pomderived" value="true"/>
18 | 		</attributes>
19 | 	</classpathentry>
20 | 	<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
21 | 		<attributes>
22 | 			<attribute name="maven.pomderived" value="true"/>
23 | 		</attributes>
24 | 	</classpathentry>
25 | 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
26 | 		<attributes>
27 | 			<attribute name="maven.pomderived" value="true"/>
28 | 		</attributes>
29 | 	</classpathentry>
30 | 	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
31 | 		<attributes>
32 | 			<attribute name="maven.pomderived" value="true"/>
33 | 		</attributes>
34 | 	</classpathentry>
35 | 	<classpathentry kind="output" path="target/classes"/>
36 | </classpath>
37 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | target/


--------------------------------------------------------------------------------
/.project:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <projectDescription>
 3 | 	<name>mqttKafkaBridge</name>
 4 | 	<comment></comment>
 5 | 	<projects>
 6 | 	</projects>
 7 | 	<buildSpec>
 8 | 		<buildCommand>
 9 | 			<name>org.eclipse.jdt.core.javabuilder</name>
10 | 			<arguments>
11 | 			</arguments>
12 | 		</buildCommand>
13 | 		<buildCommand>
14 | 			<name>org.eclipse.m2e.core.maven2Builder</name>
15 | 			<arguments>
16 | 			</arguments>
17 | 		</buildCommand>
18 | 	</buildSpec>
19 | 	<natures>
20 | 		<nature>org.eclipse.jdt.core.javanature</nature>
21 | 		<nature>org.eclipse.m2e.core.maven2Nature</nature>
22 | 	</natures>
23 | </projectDescription>
24 | 


--------------------------------------------------------------------------------
/.settings/org.eclipse.core.resources.prefs:
--------------------------------------------------------------------------------
1 | eclipse.preferences.version=1
2 | encoding//src/main/java=UTF-8
3 | encoding//src/main/resources=UTF-8
4 | encoding//src/test/java=UTF-8
5 | encoding//src/test/resources=UTF-8
6 | encoding/<project>=UTF-8
7 | 


--------------------------------------------------------------------------------
/.settings/org.eclipse.jdt.core.prefs:
--------------------------------------------------------------------------------
1 | eclipse.preferences.version=1
2 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
3 | org.eclipse.jdt.core.compiler.compliance=1.7
4 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
5 | org.eclipse.jdt.core.compiler.source=1.7
6 | 


--------------------------------------------------------------------------------
/.settings/org.eclipse.m2e.core.prefs:
--------------------------------------------------------------------------------
1 | activeProfiles=
2 | eclipse.preferences.version=1
3 | resolveWorkspaceProjects=false
4 | version=1
5 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # mqttKafkaBridge
 2 | 
 3 | Bridge which consumes MQTT messages and republishes them on Kafka on the same topic.
 4 | 
 5 | ## Usage
 6 | 
 7 |     $ java -jar mqttKafkaBridge.jar [options...]
 8 | 
 9 | Where `options` are:
10 | 
11 |     --help (-h)  : Show help
12 |     --id VAL     : MQTT Client ID
13 |     --topics VAL : MQTT topic filters (comma-separated)
14 |     --uri VAL    : MQTT Server URI
15 |     --zk VAL     : Zookeeper connect string
16 | 
17 | If you don't specify any command-line options, it uses the following defaults:
18 | 
19 |     id:      mqttKafkaBridge
20 |     topics:  '#' (all topics)
21 |     uri:     tcp://localhost:1883
22 |     zk:      localhost:2181
23 | 
24 | ***Note***: you can't run more than one bridge using the default settings, since two clients cannot connect to the same MQTT server with the same client ID. Additionally, you will get multiple messages published to Kafka for each message published to MQTT. If you wish to run multiple instances, you'll need to divide up the topics among the instances, and make sure to give them different IDs.
25 | 
26 | ## Logging
27 | `mqttKafkaBridge` uses [log4j](http://logging.apache.org/log4j/2.x/) for logging, as do the [Paho](http://www.eclipse.org/paho/) and [Kafka](http://kafka.apache.org/) libraries it uses. There is a default `log4j.properties` file packaged with the jar which simply prints all messages of level `INFO` or greater to the console. If you want to customize logging, simply create your own `log4j.properties` file, and start up `mqttKafkaBridge` as follows:
28 | 
29 |     $ java -Dlog4j.configuration=file:///path/to/log4j.properties -jar mqttKafkaBridge.jar [options...]
30 | 
31 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  3 | 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  4 | 	<modelVersion>4.0.0</modelVersion>
  5 | 	<groupId>com.m2mci</groupId>
  6 | 	<artifactId>mqttKafkaBridge</artifactId>
  7 | 	<version>0.1.0</version>
  8 | 
  9 | 	<!-- This should point to the location of the remote git repository modify 
 10 | 		this as necessary. This is needed for doing releases -->
 11 | 	<scm>
 12 | 		<developerConnection>
 13 | 			scm:git:https://bitbucket.org/m2mconnections/mqttKafkaBridge.git
 14 | 		</developerConnection>
 15 | 	</scm>
 16 | 
 17 | 	<repositories>
 18 | 		<!-- Maven Central -->
 19 | 		<repository>
 20 | 			<id>maven-central</id>
 21 | 			<name>Maven Central Repo</name>
 22 | 			<url>https://repo.maven.apache.org/maven2</url>
 23 | 		</repository>
 24 | 		<!-- Repository for the specified MQTT version -->
 25 | 		<repository>
 26 | 			<id>eclipse-paho</id>
 27 | 			<name>Eclipse Paho Repo</name>
 28 | 			<url>https://repo.eclipse.org/content/repositories/paho-releases/</url>
 29 | 		</repository>
 30 | 		<!-- Repository for the specified Kafka version -->
 31 | 		<repository>
 32 | 			<id>apache-kafka</id>
 33 | 			<name>Apache Kafka Repo</name>
 34 | 			<url>http://clojars.org/repo/</url>
 35 | 		</repository>
 36 | 		<repository>
 37 | 			<id>mvnrepository</id>
 38 | 			<name>mvnrepository</name>
 39 | 			<url>http://www.mvnrepository.com</url>
 40 | 		</repository>
 41 | 	</repositories>
 42 | 
 43 | 	<!-- Dependencies -->
 44 | 	<dependencies>
 45 | 		<dependency>
 46 | 			<groupId>junit</groupId>
 47 | 			<artifactId>junit</artifactId>
 48 | 			<version>4.11</version>
 49 | 		</dependency>
 50 | 		<dependency>
 51 | 			<groupId>org.mockito</groupId>
 52 | 			<artifactId>mockito-core</artifactId>
 53 | 			<version>1.9.5</version>
 54 | 		</dependency>
 55 | 		<dependency>
 56 | 			<groupId>org.eclipse.paho</groupId>
 57 | 			<artifactId>mqtt-client</artifactId>
 58 | 			<version>0.4.0</version>
 59 | 		</dependency>
 60 | 		<dependency>
 61 | 			<groupId>org.apache.kafka</groupId>
 62 | 			<artifactId>kafka</artifactId>
 63 | 			<version>0.7.2</version>
 64 | 		</dependency>
 65 | 		<dependency>
 66 | 			<groupId>org.scala-lang</groupId>
 67 | 			<artifactId>scala-library</artifactId>
 68 | 			<version>2.8.2</version>
 69 | 		</dependency>
 70 | 		<dependency>
 71 | 			<groupId>args4j</groupId>
 72 | 			<artifactId>args4j</artifactId>
 73 | 			<version>2.0.25</version>
 74 | 		</dependency>
 75 | 		<dependency>
 76 | 			<groupId>com.101tec</groupId>
 77 | 			<artifactId>zkclient</artifactId>
 78 | 			<version>0.3</version>
 79 | 		</dependency>
 80 | 	</dependencies>
 81 | 
 82 | 	<!-- Plugins -->
 83 | 	<build>
 84 | 		<plugins>
 85 | 			<!-- Set the compiler plugin - override default JDK version -->
 86 | 			<plugin>
 87 | 				<groupId>org.apache.maven.plugins</groupId>
 88 | 				<artifactId>maven-compiler-plugin</artifactId>
 89 | 				<version>3.1</version>
 90 | 				<configuration>
 91 | 					<source>1.7</source>
 92 | 					<target>1.7</target>
 93 | 				</configuration>
 94 | 			</plugin>
 95 | 			<!-- These plugins are needed for doing releases -->
 96 | 			<plugin>
 97 | 				<groupId>org.apache.maven.plugins</groupId>
 98 | 				<artifactId>maven-release-plugin</artifactId>
 99 | 				<version>2.4.1</version>
100 | 			</plugin>
101 | 			<plugin>
102 | 				<groupId>org.apache.maven.plugins</groupId>
103 | 				<artifactId>maven-javadoc-plugin</artifactId>
104 | 				<version>2.9</version>
105 | 			</plugin>
106 | 			<plugin>
107 | 				<groupId>org.apache.maven.plugins</groupId>
108 | 				<artifactId>maven-deploy-plugin</artifactId>
109 | 				<version>2.7</version>
110 | 			</plugin>
111 | 			<plugin>
112 | 				<groupId>org.apache.maven.plugins</groupId>
113 | 				<artifactId>maven-source-plugin</artifactId>
114 | 				<version>2.2.1</version>
115 | 			</plugin>
116 | 			<!-- Code coverage plugin -->
117 | 			<plugin>
118 | 				<groupId>org.jacoco</groupId>
119 | 				<artifactId>jacoco-maven-plugin</artifactId>
120 | 				<version>0.6.2.201302030002</version>
121 | 			</plugin>
122 | 			<plugin>
123 | 				<artifactId>maven-assembly-plugin</artifactId>
124 | 				<configuration>
125 | 					<archive>
126 | 						<manifest>
127 | 							<mainClass>com.m2mci.mqttKafkaBridge.Bridge</mainClass>
128 | 						</manifest>
129 | 					</archive>
130 | 					<descriptorRefs>
131 | 						<descriptorRef>jar-with-dependencies</descriptorRef>
132 | 					</descriptorRefs>
133 | 				</configuration>
134 | 			</plugin>
135 | 		</plugins>
136 | 	</build>
137 | 
138 | 	<!-- Used for doing releases and pushing artifacts to Artifactory -->
139 | 	<distributionManagement>
140 | 		<snapshotRepository>
141 | 			<id>m2m-snapshot</id>
142 | 			<name>m2m-artifactory1-snapshots</name>
143 | 			<url>http://24.14.16.189:8081/artifactory/libs-snapshot-local</url>
144 | 		</snapshotRepository>
145 | 		<repository>
146 | 			<id>m2m-release</id>
147 | 			<name>m2m-artifactory1-releases</name>
148 | 			<url>http://24.14.16.189:8081/artifactory/libs-release-local</url>
149 | 		</repository>
150 | 	</distributionManagement>
151 | 
152 | 	<!-- Properties -->
153 | 	<properties>
154 | 		<!-- Set the source and output encodings, to avoid warnings about the encodings 
155 | 			not being set -->
156 | 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
157 | 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
158 | 	</properties>
159 | </project>
160 | 


--------------------------------------------------------------------------------
/src/main/java/com/m2mci/mqttKafkaBridge/Bridge.java:
--------------------------------------------------------------------------------
  1 | package com.m2mci.mqttKafkaBridge;
  2 | 
  3 | import java.util.Properties;
  4 | 
  5 | import kafka.javaapi.producer.Producer;
  6 | import kafka.javaapi.producer.ProducerData;
  7 | import kafka.message.Message;
  8 | import kafka.producer.ProducerConfig;
  9 | 
 10 | import org.apache.log4j.Logger;
 11 | import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
 12 | import org.eclipse.paho.client.mqttv3.IMqttToken;
 13 | import org.eclipse.paho.client.mqttv3.MqttAsyncClient;
 14 | import org.eclipse.paho.client.mqttv3.MqttCallback;
 15 | import org.eclipse.paho.client.mqttv3.MqttException;
 16 | import org.eclipse.paho.client.mqttv3.MqttMessage;
 17 | import org.kohsuke.args4j.CmdLineException;
 18 | 
 19 | public class Bridge implements MqttCallback {
 20 | 	private Logger logger = Logger.getLogger(this.getClass().getName());
 21 | 	private MqttAsyncClient mqtt;
 22 | 	private Producer<String, Message> kafkaProducer;
 23 | 	
 24 | 	private void connect(String serverURI, String clientId, String zkConnect) throws MqttException {
 25 | 		mqtt = new MqttAsyncClient(serverURI, clientId);
 26 | 		mqtt.setCallback(this);
 27 | 		IMqttToken token = mqtt.connect();
 28 | 		Properties props = new Properties();
 29 | 		props.put("zk.connect", zkConnect);
 30 | 		props.put("serializer.class", "kafka.serializer.DefaultEncoder");
 31 | 		ProducerConfig config = new ProducerConfig(props);
 32 | 		kafkaProducer = new Producer<String, Message>(config);
 33 | 		token.waitForCompletion();
 34 | 		logger.info("Connected to MQTT and Kafka");
 35 | 	}
 36 | 
 37 | 	private void reconnect() throws MqttException {
 38 | 		IMqttToken token = mqtt.connect();
 39 | 		token.waitForCompletion();
 40 | 	}
 41 | 	
 42 | 	private void subscribe(String[] mqttTopicFilters) throws MqttException {
 43 | 		int[] qos = new int[mqttTopicFilters.length];
 44 | 		for (int i = 0; i < qos.length; ++i) {
 45 | 			qos[i] = 0;
 46 | 		}
 47 | 		mqtt.subscribe(mqttTopicFilters, qos);
 48 | 	}
 49 | 
 50 | 	@Override
 51 | 	public void connectionLost(Throwable cause) {
 52 | 		logger.warn("Lost connection to MQTT server", cause);
 53 | 		while (true) {
 54 | 			try {
 55 | 				logger.info("Attempting to reconnect to MQTT server");
 56 | 				reconnect();
 57 | 				logger.info("Reconnected to MQTT server, resuming");
 58 | 				return;
 59 | 			} catch (MqttException e) {
 60 | 				logger.warn("Reconnect failed, retrying in 10 seconds", e);
 61 | 			}
 62 | 			try {
 63 | 				Thread.sleep(10000);
 64 | 			} catch (InterruptedException e) {
 65 | 			}
 66 | 		}
 67 | 	}
 68 | 
 69 | 	@Override
 70 | 	public void deliveryComplete(IMqttDeliveryToken token) {
 71 | 		// TODO Auto-generated method stub
 72 | 		
 73 | 	}
 74 | 
 75 | 	@Override
 76 | 	public void messageArrived(String topic, MqttMessage message) throws Exception {
 77 | 		byte[] payload = message.getPayload();
 78 | 		ProducerData<String, Message> data = new ProducerData<String, Message>(topic, new Message(payload));
 79 | 		kafkaProducer.send(data);
 80 | 	}
 81 | 
 82 | 	/**
 83 | 	 * @param args
 84 | 	 */
 85 | 	public static void main(String[] args) {
 86 | 		CommandLineParser parser = null;
 87 | 		try {
 88 | 			parser = new CommandLineParser();
 89 | 			parser.parse(args);
 90 | 			Bridge bridge = new Bridge();
 91 | 			bridge.connect(parser.getServerURI(), parser.getClientId(), parser.getZkConnect());
 92 | 			bridge.subscribe(parser.getMqttTopicFilters());
 93 | 		} catch (MqttException e) {
 94 | 			e.printStackTrace(System.err);
 95 | 		} catch (CmdLineException e) {
 96 | 			System.err.println(e.getMessage());
 97 | 			parser.printUsage(System.err);
 98 | 		}
 99 | 	}
100 | }
101 | 


--------------------------------------------------------------------------------
/src/main/java/com/m2mci/mqttKafkaBridge/CommandLineParser.java:
--------------------------------------------------------------------------------
 1 | package com.m2mci.mqttKafkaBridge;
 2 | 
 3 | import java.io.OutputStream;
 4 | import java.io.PrintStream;
 5 | 
 6 | import org.kohsuke.args4j.CmdLineException;
 7 | import org.kohsuke.args4j.CmdLineParser;
 8 | import org.kohsuke.args4j.Option;
 9 | 
10 | public class CommandLineParser {
11 | 	private static final String ALL_MQTT_TOPICS = "#";
12 | 	private static final String DEFAULT_ZOOKEEPER_CONNECT = "localhost:2181";
13 | 	private static final String DEFAULT_MQTT_SERVER_URI = "tcp://localhost:1883";
14 | 
15 | 	@Option(name="--id", usage="MQTT Client ID")
16 | 	private String clientId = "mqttKafkaBridge";
17 | 
18 | 	@Option(name="--uri", usage="MQTT Server URI")
19 | 	private String serverURI = DEFAULT_MQTT_SERVER_URI;
20 | 
21 | 	@Option(name="--zk", usage="Zookeeper connect string")
22 | 	private String zkConnect = DEFAULT_ZOOKEEPER_CONNECT;
23 | 	
24 | 	@Option(name="--topics", usage="MQTT topic filters (comma-separated)")
25 | 	private String mqttTopicFilters = ALL_MQTT_TOPICS;
26 | 	
27 | 	@Option(name="--help", aliases="-h", usage="Show help")
28 | 	private boolean showHelp = false;
29 | 	
30 | 	private CmdLineParser parser = new CmdLineParser(this);
31 | 	
32 | 	public String getClientId() {
33 | 		return clientId;
34 | 	}
35 | 
36 | 	public String getServerURI() {
37 | 		return serverURI;
38 | 	}
39 | 
40 | 	public String getZkConnect() {
41 | 		return zkConnect;
42 | 	}
43 | 
44 | 	public String[] getMqttTopicFilters() {
45 | 		return mqttTopicFilters.split(",");
46 | 	}
47 | 
48 | 	public void parse(String[] args) throws CmdLineException {
49 | 		parser.parseArgument(args);
50 | 		if (showHelp) {
51 | 			printUsage(System.out);
52 | 			System.exit(0);
53 | 		}
54 | 	}
55 | 
56 | 	public void printUsage(OutputStream out) {
57 | 		PrintStream stream = new PrintStream(out);
58 | 		stream.println("java " + Bridge.class.getName() + " [options...]");
59 | 		parser.printUsage(out);
60 | 	}
61 | }
62 | 


--------------------------------------------------------------------------------
/src/main/resources/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jacklund/mqttKafkaBridge/f11a33759c1bba27f95d2e6ec417d9a8867c6094/src/main/resources/.gitkeep


--------------------------------------------------------------------------------
/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | # Set root logger level to INFO and its only appender to A1.
2 | log4j.rootLogger=INFO, A1
3 | 
4 | # A1 is set to be a ConsoleAppender.
5 | log4j.appender.A1=org.apache.log4j.ConsoleAppender
6 | 
7 | # A1 uses PatternLayout.
8 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout
9 | log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n


--------------------------------------------------------------------------------
/src/test/java/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jacklund/mqttKafkaBridge/f11a33759c1bba27f95d2e6ec417d9a8867c6094/src/test/java/.gitkeep


--------------------------------------------------------------------------------
/src/test/resources/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/jacklund/mqttKafkaBridge/f11a33759c1bba27f95d2e6ec417d9a8867c6094/src/test/resources/.gitkeep


--------------------------------------------------------------------------------