├── _config.yml ├── src └── main │ ├── resources │ ├── kafka-connect-mqtt-version.properties │ ├── loadSinkConnector.sh │ └── loadSourceConnector.sh │ ├── assembly │ └── package.xml │ └── java │ └── be │ └── jovacon │ └── kafka │ └── connect │ ├── Version.java │ ├── MQTTSinkConverter.java │ ├── MQTTSourceConnector.java │ ├── MQTTSinkConnector.java │ ├── MQTTSourceConverter.java │ ├── MQTTSinkTask.java │ ├── config │ ├── MQTTSinkConnectorConfig.java │ └── MQTTSourceConnectorConfig.java │ └── MQTTSourceTask.java ├── pom.xml └── README.md /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /src/main/resources/kafka-connect-mqtt-version.properties: -------------------------------------------------------------------------------- 1 | version=${project.version} -------------------------------------------------------------------------------- /src/main/resources/loadSinkConnector.sh: -------------------------------------------------------------------------------- 1 | curl -X POST \ 2 | http://>:8083/connectors \ 3 | -H 'Content-Type: application/json' \ 4 | -d '{ "name": "mqtt-sink-connector", 5 | "config": 6 | { 7 | "connector.class":"be.jovacon.kafka.connect.MQTTSinkConnector", 8 | "mqtt.topic":"my_mqtt_topic", 9 | "topics":"my_kafka_topic", 10 | "mqtt.clientID":"my_client_id", 11 | "mqtt.broker":"tcp://127.0.0.1:1883", 12 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 13 | "key.converter.schemas.enable":"false", 14 | "value.converter":"org.apache.kafka.connect.storage.StringConverter", 15 | "value.converter.schemas.enable":"false" 16 | } 17 | }' -------------------------------------------------------------------------------- /src/main/resources/loadSourceConnector.sh: -------------------------------------------------------------------------------- 1 | curl -X POST \ 2 | http://:8083/connectors \ 3 | -H 'Content-Type: application/json' \ 4 | -d '{ "name": "mqtt-source-connector", 5 | "config": 6 | { 7 | "connector.class":"be.jovacon.kafka.connect.MQTTSourceConnector", 8 | "mqtt.topic":"my_mqtt_topic", 9 | "kafka.topic":"my_kafka_topic", 10 | "mqtt.clientID":"my_client_id", 11 | "mqtt.broker":"tcp://127.0.0.1:1883", 12 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 13 | "key.converter.schemas.enable":"false", 14 | "value.converter":"org.apache.kafka.connect.storage.StringConverter", 15 | "value.converter.schemas.enable":"false" 16 | } 17 | }' -------------------------------------------------------------------------------- /src/main/assembly/package.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | package 7 | 8 | dir 9 | 10 | false 11 | 12 | 13 | kafka-connect-mqtt/ 14 | true 15 | true 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/main/java/be/jovacon/kafka/connect/Version.java: -------------------------------------------------------------------------------- 1 | 2 | 3 | package be.jovacon.kafka.connect; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.Properties; 9 | 10 | public class Version { 11 | private static final Logger log = LoggerFactory.getLogger(Version.class); 12 | private static String version = "unknown"; 13 | 14 | static { 15 | try { 16 | Properties props = new Properties(); 17 | props.load(Version.class.getResourceAsStream("/kafka-connect-mqtt-version.properties")); 18 | version = props.getProperty("version", version).trim(); 19 | } catch (Exception e) { 20 | log.warn("Error while loading version:", e); 21 | } 22 | } 23 | 24 | public static String getVersion() { 25 | return version; 26 | } 27 | 28 | public static void main(String[] args) { 29 | System.out.println(Version.getVersion()); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/be/jovacon/kafka/connect/MQTTSinkConverter.java: -------------------------------------------------------------------------------- 1 | package be.jovacon.kafka.connect; 2 | 3 | import be.jovacon.kafka.connect.config.MQTTSinkConnectorConfig; 4 | import be.jovacon.kafka.connect.config.MQTTSourceConnectorConfig; 5 | import org.apache.kafka.connect.sink.SinkRecord; 6 | import org.eclipse.paho.client.mqttv3.MqttMessage; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * Converts a Kafka message to a MQTT Message 12 | */ 13 | public class MQTTSinkConverter { 14 | 15 | private MQTTSinkConnectorConfig mqttSinkConnectorConfig; 16 | 17 | private Logger log = LoggerFactory.getLogger(MQTTSinkConverter.class); 18 | 19 | public MQTTSinkConverter(MQTTSinkConnectorConfig mqttSinkConnectorConfig) { 20 | this.mqttSinkConnectorConfig = mqttSinkConnectorConfig; 21 | } 22 | 23 | protected MqttMessage convert(SinkRecord sinkRecord) { 24 | log.trace("Converting Kafka message"); 25 | 26 | MqttMessage mqttMessage = new MqttMessage(); 27 | mqttMessage.setPayload(((String)sinkRecord.value()).getBytes()); 28 | mqttMessage.setQos(this.mqttSinkConnectorConfig.getInt(MQTTSourceConnectorConfig.MQTT_QOS)); 29 | log.trace("Result MQTTMessage: " + mqttMessage); 30 | return mqttMessage; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/be/jovacon/kafka/connect/MQTTSourceConnector.java: -------------------------------------------------------------------------------- 1 | package be.jovacon.kafka.connect; 2 | 3 | import be.jovacon.kafka.connect.config.MQTTSourceConnectorConfig; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | import org.apache.kafka.connect.connector.Task; 6 | import org.apache.kafka.connect.source.SourceConnector; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.*; 11 | 12 | /** 13 | * Implementation of the Kafka Connect Source connector 14 | */ 15 | public class MQTTSourceConnector extends SourceConnector { 16 | 17 | private static final Logger log = LoggerFactory.getLogger(MQTTSourceConnector.class); 18 | private MQTTSourceConnectorConfig mqttSourceConnectorConfig; 19 | private Map configProps; 20 | 21 | public void start(Map map) { 22 | this.mqttSourceConnectorConfig = new MQTTSourceConnectorConfig(map); 23 | this.configProps = Collections.unmodifiableMap(map); 24 | } 25 | 26 | public Class taskClass() { 27 | return MQTTSourceTask.class; 28 | } 29 | 30 | public List> taskConfigs(int maxTasks) { 31 | log.debug("Enter taskconfigs"); 32 | if (maxTasks > 1) { 33 | log.info("maxTasks is " + maxTasks + ". MaxTasks > 1 is not supported in this connector."); 34 | } 35 | List> taskConfigs = new ArrayList<>(1); 36 | taskConfigs.add(new HashMap<>(configProps)); 37 | 38 | log.debug("Taskconfigs: " + taskConfigs); 39 | return taskConfigs; 40 | } 41 | 42 | public void stop() { 43 | 44 | } 45 | 46 | public ConfigDef config() { 47 | return MQTTSourceConnectorConfig.configDef(); 48 | } 49 | 50 | public String version() { 51 | return Version.getVersion(); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/be/jovacon/kafka/connect/MQTTSinkConnector.java: -------------------------------------------------------------------------------- 1 | package be.jovacon.kafka.connect; 2 | 3 | import be.jovacon.kafka.connect.config.MQTTSinkConnectorConfig; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | import org.apache.kafka.connect.connector.Task; 6 | import org.apache.kafka.connect.sink.SinkConnector; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.*; 11 | 12 | /** 13 | * Implementation of the Kafka Connect Sink connector 14 | */ 15 | public class MQTTSinkConnector extends SinkConnector { 16 | 17 | private static final Logger log = LoggerFactory.getLogger(MQTTSinkConnector.class); 18 | private MQTTSinkConnectorConfig mqttSinkConnectorConfig; 19 | private Map configProps; 20 | 21 | @Override 22 | public void start(Map map) { 23 | this.mqttSinkConnectorConfig = new MQTTSinkConnectorConfig(map); 24 | this.configProps = Collections.unmodifiableMap(map); 25 | } 26 | 27 | @Override 28 | public Class taskClass() { 29 | return MQTTSinkTask.class; 30 | } 31 | 32 | @Override 33 | public List> taskConfigs(int maxTasks) { 34 | log.debug("Enter taskconfigs"); 35 | if (maxTasks > 1) { 36 | log.info("maxTasks is " + maxTasks + ". MaxTasks > 1 is not supported in this connector."); 37 | } 38 | List> taskConfigs = new ArrayList<>(1); 39 | taskConfigs.add(new HashMap<>(configProps)); 40 | 41 | log.debug("Taskconfigs: " + taskConfigs); 42 | return taskConfigs; 43 | } 44 | 45 | @Override 46 | public void stop() { 47 | 48 | } 49 | 50 | @Override 51 | public ConfigDef config() { 52 | return MQTTSinkConnectorConfig.configDef(); 53 | } 54 | 55 | @Override 56 | public String version() { 57 | return Version.getVersion(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/be/jovacon/kafka/connect/MQTTSourceConverter.java: -------------------------------------------------------------------------------- 1 | package be.jovacon.kafka.connect; 2 | 3 | import be.jovacon.kafka.connect.config.MQTTSourceConnectorConfig; 4 | import org.apache.kafka.connect.data.Schema; 5 | import org.apache.kafka.connect.header.ConnectHeaders; 6 | import org.apache.kafka.connect.source.SourceRecord; 7 | import org.eclipse.paho.client.mqttv3.MqttMessage; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.HashMap; 12 | 13 | /** 14 | * Converts a MQTT message to a Kafka message 15 | */ 16 | public class MQTTSourceConverter { 17 | 18 | private MQTTSourceConnectorConfig mqttSourceConnectorConfig; 19 | 20 | private Logger log = LoggerFactory.getLogger(MQTTSourceConverter.class); 21 | 22 | public MQTTSourceConverter(MQTTSourceConnectorConfig mqttSourceConnectorConfig) { 23 | this.mqttSourceConnectorConfig = mqttSourceConnectorConfig; 24 | } 25 | 26 | protected SourceRecord convert(String topic, MqttMessage mqttMessage) { 27 | log.debug("Converting MQTT message: " + mqttMessage); 28 | // Kafka 2.3 29 | ConnectHeaders headers = new ConnectHeaders(); 30 | headers.addInt("mqtt.message.id", mqttMessage.getId()); 31 | headers.addInt("mqtt.message.qos", mqttMessage.getQos()); 32 | headers.addBoolean("mqtt.message.duplicate", mqttMessage.isDuplicate()); 33 | 34 | // Kafka 1.0 35 | /*SourceRecord sourceRecord = new SourceRecord( 36 | new HashMap<>(), 37 | new HashMap<>(), 38 | this.mqttSourceConnectorConfig.getString(MQTTSourceConnectorConfig.KAFKA_TOPIC), 39 | Schema.STRING_SCHEMA, 40 | new String(mqttMessage.getPayload())); 41 | */ 42 | // Kafka 2.3 43 | SourceRecord sourceRecord = new SourceRecord(new HashMap<>(), 44 | new HashMap<>(), 45 | this.mqttSourceConnectorConfig.getString(MQTTSourceConnectorConfig.KAFKA_TOPIC), 46 | (Integer) null, 47 | Schema.STRING_SCHEMA, 48 | topic, 49 | Schema.STRING_SCHEMA, 50 | new String(mqttMessage.getPayload()), 51 | System.currentTimeMillis(), 52 | headers); 53 | log.debug("Converted MQTT Message: " + sourceRecord); 54 | return sourceRecord; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 4.0.0 8 | 9 | be.jovacon 10 | kafka-connect-mqtt 11 | jar 12 | kafka.connect.mqtt 13 | 1.1.0 14 | 15 | 16 | 17 | org.apache.kafka 18 | connect-api 19 | 2.3.0 20 | provided 21 | 22 | 23 | com.github.jcustenborder.kafka.connect 24 | connect-utils 25 | 0.4.156 26 | 27 | 28 | org.apache.logging.log4j 29 | log4j-core 30 | 2.11.2 31 | 32 | 33 | org.eclipse.paho 34 | org.eclipse.paho.client.mqttv3 35 | 1.2.2 36 | 37 | 38 | 39 | 40 | 41 | 42 | org.apache.maven.plugins 43 | maven-resources-plugin 44 | 45 | 46 | 47 | src/main/resources 48 | true 49 | 50 | 51 | 52 | 53 | 54 | maven-assembly-plugin 55 | 56 | 57 | src/main/assembly/package.xml 58 | 59 | false 60 | 61 | 62 | 63 | make-assembly 64 | package 65 | 66 | single 67 | 68 | 69 | 70 | 71 | 72 | org.apache.maven.plugins 73 | maven-compiler-plugin 74 | 75 | 8 76 | 8 77 | 78 | 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /src/main/java/be/jovacon/kafka/connect/MQTTSinkTask.java: -------------------------------------------------------------------------------- 1 | package be.jovacon.kafka.connect; 2 | 3 | import be.jovacon.kafka.connect.config.MQTTSinkConnectorConfig; 4 | import be.jovacon.kafka.connect.config.MQTTSourceConnectorConfig; 5 | import org.apache.kafka.connect.errors.ConnectException; 6 | import org.apache.kafka.connect.sink.SinkRecord; 7 | import org.apache.kafka.connect.sink.SinkTask; 8 | import org.eclipse.paho.client.mqttv3.*; 9 | import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.util.Collection; 14 | import java.util.Iterator; 15 | import java.util.Map; 16 | 17 | /** 18 | * Implementation of the Kafka Connect Sink task 19 | */ 20 | public class MQTTSinkTask extends SinkTask { 21 | 22 | private Logger log = LoggerFactory.getLogger(MQTTSinkTask.class); 23 | private MQTTSinkConnectorConfig config; 24 | private MQTTSinkConverter mqttSinkConverter; 25 | 26 | private IMqttClient mqttClient; 27 | 28 | @Override 29 | public String version() { 30 | return Version.getVersion(); 31 | } 32 | 33 | @Override 34 | public void start(Map map) { 35 | config = new MQTTSinkConnectorConfig(map); 36 | mqttSinkConverter = new MQTTSinkConverter(config); 37 | try { 38 | mqttClient = new MqttClient(config.getString(MQTTSinkConnectorConfig.BROKER), config.getString(MQTTSinkConnectorConfig.CLIENTID), new MemoryPersistence()); 39 | 40 | log.info("Connecting to MQTT Broker " + config.getString(MQTTSourceConnectorConfig.BROKER)); 41 | connect(mqttClient); 42 | log.info("Connected to MQTT Broker. This connector publishes to the " + this.config.getString(MQTTSinkConnectorConfig.MQTT_TOPIC) + " topic"); 43 | 44 | } 45 | catch (MqttException e) { 46 | throw new ConnectException(e); 47 | } 48 | } 49 | 50 | private void connect(IMqttClient mqttClient) throws MqttException{ 51 | MqttConnectOptions connOpts = new MqttConnectOptions(); 52 | connOpts.setCleanSession(config.getBoolean(MQTTSinkConnectorConfig.MQTT_CLEANSESSION)); 53 | connOpts.setKeepAliveInterval(config.getInt(MQTTSinkConnectorConfig.MQTT_KEEPALIVEINTERVAL)); 54 | connOpts.setConnectionTimeout(config.getInt(MQTTSinkConnectorConfig.MQTT_CONNECTIONTIMEOUT)); 55 | connOpts.setAutomaticReconnect(config.getBoolean(MQTTSinkConnectorConfig.MQTT_ARC)); 56 | 57 | if (!config.getString(MQTTSinkConnectorConfig.MQTT_USERNAME).equals("") && !config.getPassword(MQTTSinkConnectorConfig.MQTT_PASSWORD).equals("")) { 58 | connOpts.setUserName(config.getString(MQTTSinkConnectorConfig.MQTT_USERNAME)); 59 | connOpts.setPassword(config.getPassword(MQTTSinkConnectorConfig.MQTT_PASSWORD).value().toCharArray()); 60 | } 61 | 62 | log.debug("MQTT Connection properties: " + connOpts); 63 | 64 | mqttClient.connect(connOpts); 65 | } 66 | 67 | @Override 68 | public void put(Collection collection) { 69 | try { 70 | for (Iterator iterator = collection.iterator(); iterator.hasNext(); ) { 71 | SinkRecord sinkRecord = iterator.next(); 72 | log.debug("Received message with offset " + sinkRecord.kafkaOffset()); 73 | MqttMessage mqttMessage = mqttSinkConverter.convert(sinkRecord); 74 | if (!mqttClient.isConnected()) mqttClient.connect(); 75 | log.debug("Publishing message to topic " + this.config.getString(MQTTSinkConnectorConfig.MQTT_TOPIC) + " with payload " + new String(mqttMessage.getPayload())); 76 | mqttClient.publish(this.config.getString(MQTTSinkConnectorConfig.MQTT_TOPIC), mqttMessage); 77 | } 78 | } catch (MqttException e) { 79 | throw new ConnectException(e); 80 | } 81 | 82 | } 83 | 84 | @Override 85 | public void stop() { 86 | if (mqttClient.isConnected()) { 87 | try { 88 | log.debug("Disconnecting from MQTT Broker " + config.getString(MQTTSinkConnectorConfig.BROKER)); 89 | mqttClient.disconnect(); 90 | } catch (MqttException mqttException) { 91 | log.error("Exception thrown while disconnecting client.", mqttException); 92 | } 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-connect-mqtt 2 | 3 | This repo contains a MQTT Source and Sink Connector for Apache Kafka. It is tested with Kafka 2+. 4 | 5 | Using the Source connector you can subscribe to a MQTT topic and write these messages to a Kafka topic. 6 | 7 | The Sink connector works the other way around. 8 | 9 | Note: 10 | * SSL connections are not supported at the moment 11 | * The connector works only with a single task. Settings maxTasks > 1 has no effect. 12 | 13 | ## Building the connector 14 | 15 | To build the connector, you must have the following installed: 16 | 17 | * Java 8 or later 18 | * Maven 19 | * GIT 20 | 21 | Clone the repository with the following command: 22 | ``` 23 | git clone https://github.com/johanvandevenne/kafka-connect-mqtt.git 24 | ``` 25 | Change directory into the `kafka-connect-mqtt` directory 26 | ``` 27 | cd kafka-connect-mqtt 28 | ``` 29 | Build the connector using Maven 30 | ``` 31 | mvn clean install 32 | ``` 33 | ## Installing the connector 34 | 35 | ### Prerequisites 36 | 37 | You must have Kafka 2+ installed 38 | 39 | 40 | ### Installing 41 | 42 | * Copy the folder `/target/kafka-connect-mqtt-1.0-0-package/share/kafka-connect-mqtt` to your Kafka Connect plugin path 43 | * Restart Kafka Connect 44 | * Check if the connector has been loaded succesfully 45 | 46 | ``` 47 | http://:8083/connector-plugins 48 | ``` 49 | If you see these entries, the connector has been installed succesfully 50 | 51 | ``` 52 | { 53 | "class": "MQTTSinkConnector", 54 | "type": "sink", 55 | "version": "1.0.0" 56 | }, 57 | { 58 | "class": "MQTTSourceConnector", 59 | "type": "source", 60 | "version": "1.0.0" 61 | }, 62 | ``` 63 | 64 | ## Configuring the Source connector 65 | 66 | The MQTT Source connector subscribes to a Topic on a MQTT Broker and sends the messages to a Kafka topic. 67 | 68 | Here is a basic configuration example: 69 | ``` 70 | curl -X POST \ 71 | http://:8083/connectors \ 72 | -H 'Content-Type: application/json' \ 73 | -d '{ "name": "mqtt-source-connector", 74 | "config": 75 | { 76 | "connector.class":"be.jovacon.kafka.connect.MQTTSourceConnector", 77 | "mqtt.topic":"my_mqtt_topic", 78 | "kafka.topic":"my_kafka_topic", 79 | "mqtt.clientID":"my_client_id", 80 | "mqtt.broker":"tcp://127.0.0.1:1883", 81 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 82 | "key.converter.schemas.enable":false, 83 | "value.converter":"org.apache.kafka.connect.storage.StringConverter", 84 | "value.converter.schemas.enable":false 85 | } 86 | }' 87 | ``` 88 | ### Optional Configuration options 89 | * `mqtt.qos` (optional): 0 – At most Once, 1 – At Least Once, 2 – Exactly Once 90 | * `mqtt.automaticReconnect` (optional)(default: true): Should the client automatically reconnect in case of connection failures 91 | * `mqtt.keepAliveInterval` (optional)(default: 60 seconds) 92 | * `mqtt.cleanSession` (optional)(default: true): Controls the state after disconnecting the client from the broker. 93 | * `mqtt.connectionTimeout` (optional)(default: 30 seconds) 94 | * `mqtt.username` (optional): Username to connect to MQTT broker 95 | * `mqtt.password` (optional): Password to connect to MQTT broker 96 | 97 | ## Configuring the Sink connector 98 | 99 | The MQTT Sink Connector reads messages from a Kafka topic and publishes them to a MQTT topic. 100 | 101 | Here is a basic configuration example: 102 | ``` 103 | curl -X POST \ 104 | http://>:8083/connectors \ 105 | -H 'Content-Type: application/json' \ 106 | -d '{ "name": "mqtt-sink-connector", 107 | "config": 108 | { 109 | "connector.class":"be.jovacon.kafka.connect.MQTTSinkConnector", 110 | "mqtt.topic":"my_mqtt_topic", 111 | "topics":"my_kafka_topic", 112 | "mqtt.clientID":"my_client_id", 113 | "mqtt.broker":"tcp://127.0.0.1:1883", 114 | "key.converter":"org.apache.kafka.connect.storage.StringConverter", 115 | "key.converter.schemas.enable":false, 116 | "value.converter":"org.apache.kafka.connect.storage.StringConverter", 117 | "value.converter.schemas.enable":false 118 | } 119 | }' 120 | ``` 121 | 122 | ### Optional Configuration options 123 | * `mqtt.qos` (optional): 0 – At most Once, 1 – At Least Once, 2 – Exactly Once 124 | * `mqtt.automaticReconnect` (optional)(default: true): Should the client automatically reconnect in case of connection failures 125 | * `mqtt.keepAliveInterval` (optional)(default: 60 seconds) 126 | * `mqtt.cleanSession` (optional)(default: true): Controls the state after disconnecting the client from the broker. 127 | * `mqtt.connectionTimeout` (optional)(default: 30 seconds) 128 | * `mqtt.username` (optional): Username to connect to MQTT broker 129 | * `mqtt.password` (optional): Password to connect to MQTT broker 130 | 131 | 132 | ## Authors 133 | 134 | * **Johan Vandevenne** - *Initial work* 135 | -------------------------------------------------------------------------------- /src/main/java/be/jovacon/kafka/connect/config/MQTTSinkConnectorConfig.java: -------------------------------------------------------------------------------- 1 | package be.jovacon.kafka.connect.config; 2 | 3 | import org.apache.kafka.common.config.AbstractConfig; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | 6 | import java.util.Map; 7 | 8 | public class MQTTSinkConnectorConfig extends AbstractConfig { 9 | 10 | public static final String BROKER = "mqtt.broker"; 11 | public static final String BROKER_DOC = "Host and port of the MQTT broker, eg: tcp://192.168.1.1:1883"; 12 | 13 | public static final String CLIENTID = "mqtt.clientID"; 14 | public static final String CLIENTID_DOC = "clientID"; 15 | 16 | public static final String MQTT_TOPIC = "mqtt.topic"; 17 | public static final String MQTT_TOPIC_DOC = "List of topic names to subscribe to"; 18 | 19 | public static final String MQTT_ARC = "mqtt.automaticReconnect"; 20 | public static final String MQTT_ARC_DOC = "set Automatic reconnect, default true"; 21 | 22 | public static final String MQTT_KEEPALIVEINTERVAL = "mqtt.keepAliveInterval"; 23 | public static final String MQTT_KEEPALIVEINTERVAL_DOC = "set the keepalive interval, default is 60 seconds"; 24 | 25 | public static final String MQTT_CLEANSESSION = "mqtt.cleanSession"; 26 | public static final String MQTT_CLEANSESSION_DOC = "Sets whether the client and server should remember state across restarts and reconnects, default is true"; 27 | 28 | public static final String MQTT_CONNECTIONTIMEOUT = "mqtt.connectionTimeout"; 29 | public static final String MQTT_CONNECTIONTIMEOUT_DOC = "Sets the connection timeout, default is 30"; 30 | 31 | public static final String KAFKA_TOPIC = "topics"; 32 | public static final String KAFKA_TOPIC_DOC = "List of kafka topics to consume from"; 33 | 34 | public static final String MQTT_QOS = "mqtt.qos"; 35 | public static final String MQTT_QOS_DOC = "Quality of service MQTT messaging, default is 1 (at least once)"; 36 | 37 | public static final String MQTT_USERNAME = "mqtt.userName"; 38 | public static final String MQTT_USERNAME_DOC = "Sets the username for the MQTT connection timeout, default is \"\""; 39 | 40 | public static final String MQTT_PASSWORD = "mqtt.password"; 41 | public static final String MQTT_PASSWORD_DOC = "Sets the password for the MQTT connection timeout, default is \"\""; 42 | 43 | public MQTTSinkConnectorConfig(Map originals) { 44 | super(configDef(), originals); 45 | } 46 | 47 | public static ConfigDef configDef() { 48 | return new ConfigDef() 49 | .define(BROKER, 50 | ConfigDef.Type.STRING, 51 | ConfigDef.Importance.HIGH, 52 | BROKER_DOC) 53 | .define(CLIENTID, 54 | ConfigDef.Type.STRING, 55 | ConfigDef.Importance.HIGH, 56 | CLIENTID_DOC) 57 | .define(MQTT_TOPIC, 58 | ConfigDef.Type.STRING, 59 | ConfigDef.Importance.HIGH, 60 | MQTT_TOPIC_DOC) 61 | .define(KAFKA_TOPIC, 62 | ConfigDef.Type.LIST, 63 | ConfigDef.Importance.HIGH, 64 | KAFKA_TOPIC_DOC) 65 | .define(MQTT_QOS, 66 | ConfigDef.Type.INT, 67 | 1, 68 | ConfigDef.Range.between(1,3), 69 | ConfigDef.Importance.MEDIUM, 70 | MQTT_QOS_DOC) 71 | .define(MQTT_ARC, 72 | ConfigDef.Type.BOOLEAN, 73 | true, 74 | ConfigDef.Importance.MEDIUM, 75 | MQTT_ARC_DOC) 76 | .define(MQTT_KEEPALIVEINTERVAL, 77 | ConfigDef.Type.INT, 78 | 60, 79 | ConfigDef.Importance.LOW, 80 | MQTT_KEEPALIVEINTERVAL_DOC) 81 | .define(MQTT_CLEANSESSION, 82 | ConfigDef.Type.BOOLEAN, 83 | true, 84 | ConfigDef.Importance.LOW, 85 | MQTT_CLEANSESSION_DOC) 86 | .define(MQTT_CONNECTIONTIMEOUT, 87 | ConfigDef.Type.INT, 88 | 30, 89 | ConfigDef.Importance.LOW, 90 | MQTT_CONNECTIONTIMEOUT_DOC) 91 | .define(MQTT_USERNAME, 92 | ConfigDef.Type.STRING, 93 | "", 94 | ConfigDef.Importance.LOW, 95 | MQTT_USERNAME_DOC) 96 | .define(MQTT_PASSWORD, 97 | ConfigDef.Type.PASSWORD, 98 | "", 99 | ConfigDef.Importance.LOW, 100 | MQTT_PASSWORD_DOC); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/be/jovacon/kafka/connect/config/MQTTSourceConnectorConfig.java: -------------------------------------------------------------------------------- 1 | package be.jovacon.kafka.connect.config; 2 | 3 | import org.apache.kafka.common.config.AbstractConfig; 4 | import org.apache.kafka.common.config.ConfigDef; 5 | 6 | import java.util.Map; 7 | 8 | public class MQTTSourceConnectorConfig extends AbstractConfig { 9 | 10 | public static final String BROKER = "mqtt.broker"; 11 | public static final String BROKER_DOC = "Host and port of the MQTT broker, eg: tcp://192.168.1.1:1883"; 12 | 13 | public static final String CLIENTID = "mqtt.clientID"; 14 | public static final String CLIENTID_DOC = "clientID"; 15 | 16 | public static final String MQTT_TOPIC = "mqtt.topic"; 17 | public static final String MQTT_TOPIC_DOC = "List of topic names to subscribe to"; 18 | 19 | public static final String MQTT_QOS = "mqtt.qos"; 20 | public static final String MQTT_QOS_DOC = "Quality of service MQTT messaging, default is 1 (at least once)"; 21 | 22 | public static final String MQTT_ARC = "mqtt.automaticReconnect"; 23 | public static final String MQTT_ARC_DOC = "set Automatic reconnect, default true"; 24 | 25 | public static final String MQTT_KEEPALIVEINTERVAL = "mqtt.keepAliveInterval"; 26 | public static final String MQTT_KEEPALIVEINTERVAL_DOC = "set the keepalive interval, default is 60 seconds"; 27 | 28 | public static final String MQTT_CLEANSESSION = "mqtt.cleanSession"; 29 | public static final String MQTT_CLEANSESSION_DOC = "Sets whether the client and server should remember state across restarts and reconnects, default is true"; 30 | 31 | public static final String MQTT_CONNECTIONTIMEOUT = "mqtt.connectionTimeout"; 32 | public static final String MQTT_CONNECTIONTIMEOUT_DOC = "Sets the connection timeout, default is 30"; 33 | 34 | public static final String MQTT_USERNAME = "mqtt.userName"; 35 | public static final String MQTT_USERNAME_DOC = "Sets the username for the MQTT connection timeout, default is \"\""; 36 | 37 | public static final String MQTT_PASSWORD = "mqtt.password"; 38 | public static final String MQTT_PASSWORD_DOC = "Sets the password for the MQTT connection timeout, default is \"\""; 39 | 40 | public static final String KAFKA_TOPIC = "kafka.topic"; 41 | public static final String KAFKA_TOPIC_DOC = "List of kafka topics to publish to"; 42 | 43 | public MQTTSourceConnectorConfig(Map originals) { 44 | super(configDef(), originals); 45 | } 46 | 47 | public static ConfigDef configDef() { 48 | return new ConfigDef() 49 | .define(BROKER, 50 | ConfigDef.Type.STRING, 51 | ConfigDef.Importance.HIGH, 52 | BROKER_DOC) 53 | .define(CLIENTID, 54 | ConfigDef.Type.STRING, 55 | ConfigDef.Importance.HIGH, 56 | CLIENTID_DOC) 57 | .define(MQTT_TOPIC, 58 | ConfigDef.Type.STRING, 59 | ConfigDef.Importance.HIGH, 60 | MQTT_TOPIC_DOC) 61 | .define(KAFKA_TOPIC, 62 | ConfigDef.Type.STRING, 63 | ConfigDef.Importance.HIGH, 64 | KAFKA_TOPIC_DOC) 65 | .define(MQTT_QOS, 66 | ConfigDef.Type.INT, 67 | 1, 68 | ConfigDef.Range.between(1,3), 69 | ConfigDef.Importance.MEDIUM, 70 | MQTT_QOS_DOC) 71 | .define(MQTT_ARC, 72 | ConfigDef.Type.BOOLEAN, 73 | true, 74 | ConfigDef.Importance.MEDIUM, 75 | MQTT_ARC_DOC) 76 | .define(MQTT_KEEPALIVEINTERVAL, 77 | ConfigDef.Type.INT, 78 | 60, 79 | ConfigDef.Importance.LOW, 80 | MQTT_KEEPALIVEINTERVAL_DOC) 81 | .define(MQTT_CLEANSESSION, 82 | ConfigDef.Type.BOOLEAN, 83 | true, 84 | ConfigDef.Importance.LOW, 85 | MQTT_CLEANSESSION_DOC) 86 | .define(MQTT_CONNECTIONTIMEOUT, 87 | ConfigDef.Type.INT, 88 | 30, 89 | ConfigDef.Importance.LOW, 90 | MQTT_CONNECTIONTIMEOUT_DOC) 91 | .define(MQTT_USERNAME, 92 | ConfigDef.Type.STRING, 93 | "", 94 | ConfigDef.Importance.LOW, 95 | MQTT_USERNAME_DOC) 96 | .define(MQTT_PASSWORD, 97 | ConfigDef.Type.PASSWORD, 98 | "", 99 | ConfigDef.Importance.LOW, 100 | MQTT_PASSWORD_DOC) 101 | ; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/be/jovacon/kafka/connect/MQTTSourceTask.java: -------------------------------------------------------------------------------- 1 | package be.jovacon.kafka.connect; 2 | 3 | import be.jovacon.kafka.connect.config.MQTTSourceConnectorConfig; 4 | import com.github.jcustenborder.kafka.connect.utils.data.SourceRecordDeque; 5 | import com.github.jcustenborder.kafka.connect.utils.data.SourceRecordDequeBuilder; 6 | import org.apache.kafka.connect.errors.ConnectException; 7 | import org.apache.kafka.connect.source.SourceRecord; 8 | import org.apache.kafka.connect.source.SourceTask; 9 | import org.eclipse.paho.client.mqttv3.*; 10 | import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | /** 18 | * Actual implementation of the Kafka Connect MQTT Source Task 19 | */ 20 | public class MQTTSourceTask extends SourceTask implements IMqttMessageListener { 21 | 22 | private Logger log = LoggerFactory.getLogger(MQTTSourceConnector.class); 23 | private MQTTSourceConnectorConfig config; 24 | private MQTTSourceConverter mqttSourceConverter; 25 | private SourceRecordDeque sourceRecordDeque; 26 | 27 | private IMqttClient mqttClient; 28 | 29 | public void start(Map props) { 30 | config = new MQTTSourceConnectorConfig(props); 31 | mqttSourceConverter = new MQTTSourceConverter(config); 32 | this.sourceRecordDeque = SourceRecordDequeBuilder.of().batchSize(4096).emptyWaitMs(100).maximumCapacityTimeoutMs(60000).maximumCapacity(50000).build(); 33 | try { 34 | mqttClient = new MqttClient(config.getString(MQTTSourceConnectorConfig.BROKER), config.getString(MQTTSourceConnectorConfig.CLIENTID), new MemoryPersistence()); 35 | 36 | log.info("Connecting to MQTT Broker " + config.getString(MQTTSourceConnectorConfig.BROKER)); 37 | connect(mqttClient); 38 | log.info("Connected to MQTT Broker"); 39 | 40 | String topicSubscription = this.config.getString(MQTTSourceConnectorConfig.MQTT_TOPIC); 41 | int qosLevel = this.config.getInt(MQTTSourceConnectorConfig.MQTT_QOS); 42 | 43 | log.info("Subscribing to " + topicSubscription + " with QOS " + qosLevel); 44 | mqttClient.subscribe(topicSubscription, qosLevel, (topic, message) -> { 45 | log.debug("Message arrived in connector from topic " + topic); 46 | SourceRecord record = mqttSourceConverter.convert(topic, message); 47 | log.debug("Converted record: " + record); 48 | sourceRecordDeque.add(record); 49 | }); 50 | log.info("Subscribed to " + topicSubscription + " with QOS " + qosLevel); 51 | } 52 | catch (MqttException e) { 53 | throw new ConnectException(e); 54 | } 55 | } 56 | 57 | private void connect(IMqttClient mqttClient) throws MqttException{ 58 | MqttConnectOptions connOpts = new MqttConnectOptions(); 59 | connOpts.setCleanSession(config.getBoolean(MQTTSourceConnectorConfig.MQTT_CLEANSESSION)); 60 | connOpts.setKeepAliveInterval(config.getInt(MQTTSourceConnectorConfig.MQTT_KEEPALIVEINTERVAL)); 61 | connOpts.setConnectionTimeout(config.getInt(MQTTSourceConnectorConfig.MQTT_CONNECTIONTIMEOUT)); 62 | connOpts.setAutomaticReconnect(config.getBoolean(MQTTSourceConnectorConfig.MQTT_ARC)); 63 | 64 | if (!config.getString(MQTTSourceConnectorConfig.MQTT_USERNAME).equals("") && !config.getPassword(MQTTSourceConnectorConfig.MQTT_PASSWORD).equals("")) { 65 | connOpts.setUserName(config.getString(MQTTSourceConnectorConfig.MQTT_USERNAME)); 66 | connOpts.setPassword(config.getPassword(MQTTSourceConnectorConfig.MQTT_PASSWORD).value().toCharArray()); 67 | } 68 | 69 | log.info("MQTT Connection properties: " + connOpts); 70 | 71 | mqttClient.connect(connOpts); 72 | } 73 | 74 | /** 75 | * method is called periodically by the Connect framework 76 | * 77 | * @return 78 | * @throws InterruptedException 79 | */ 80 | public List poll() throws InterruptedException { 81 | List records = sourceRecordDeque.getBatch(); 82 | log.trace("Records returning to poll(): " + records); 83 | return records; 84 | } 85 | 86 | public void stop() { 87 | if (mqttClient.isConnected()) { 88 | try { 89 | log.debug("Disconnecting from MQTT Broker " + config.getString(MQTTSourceConnectorConfig.BROKER)); 90 | mqttClient.disconnect(); 91 | } catch (MqttException mqttException) { 92 | log.error("Exception thrown while disconnecting client.", mqttException); 93 | } 94 | } 95 | } 96 | 97 | public String version() { 98 | return Version.getVersion(); 99 | } 100 | 101 | 102 | /** 103 | * Callback method when a MQTT message arrives at the Topic 104 | */ 105 | @Override 106 | public void messageArrived(String topic, MqttMessage mqttMessage) throws Exception { 107 | log.debug("Message arrived in connector from topic " + topic); 108 | SourceRecord record = mqttSourceConverter.convert(topic, mqttMessage); 109 | log.debug("Converted record: " + record); 110 | sourceRecordDeque.add(record); 111 | } 112 | } 113 | --------------------------------------------------------------------------------