├── .github └── workflows │ └── maven.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── docker-compose ├── fullstack │ ├── docker-compose.yml │ └── env.list └── standalone │ └── docker-compose.yml ├── docker └── setConfiguration.sh ├── pom.xml ├── simple_kafka_mqtt_connector.iml └── src └── main ├── java └── de │ └── fhg │ └── ipa │ └── null70 │ └── simple_kafka_mqtt_connector │ ├── Main.java │ └── SimpleKafkaMQTTConnector.java └── resources ├── application.properties └── log4j2.xml /.github/workflows/maven.yml: -------------------------------------------------------------------------------- 1 | name: Java CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Set up JDK 1.8 13 | uses: actions/setup-java@v1 14 | with: 15 | java-version: 1.8 16 | - name: Build with Maven 17 | run: mvn -B package --file pom.xml 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | target 3 | !target/*jar-with-dependencies.jar 4 | /target 5 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8u312-jdk-slim 2 | 3 | ENV SERVER_INSTALL_FOLDER=/app/kafka-mqtt-connector/ 4 | ENV JAR_FILE_NAME=simple_kafka_mqtt_connector-0.1.0-jar-with-dependencies.jar 5 | 6 | # Application propertis 7 | ENV KAFKA_HOST=localhosts 8 | ENV KAFKA_PORT=9092 9 | ENV KAFKA_CLIENT_ID=testing-kafka-producer-1 10 | 11 | ENV MQTT_HOST=localhost 12 | ENV MQTT_PORT=1883 13 | ENV MQTT_CLIENT_ID=mqtt-client-1 14 | ENV MQTT_QOS=2 15 | 16 | ENV TOPIC_MAPPING=robotgroup001/robot001>>>test;robotgroup001/robot002>>>test02;robotgroup001/robot003>>>test03 17 | 18 | RUN mkdir -p "${SERVER_INSTALL_FOLDER}log" 19 | 20 | #SERVER: 21 | ADD src/main/resources/application.properties ${SERVER_INSTALL_FOLDER} 22 | ADD target/${JAR_FILE_NAME} ${SERVER_INSTALL_FOLDER} 23 | 24 | ADD docker/setConfiguration.sh ${SERVER_INSTALL_FOLDER} 25 | 26 | WORKDIR ${SERVER_INSTALL_FOLDER} 27 | 28 | 29 | CMD bash ${SERVER_INSTALL_FOLDER}setConfiguration.sh "${SERVER_INSTALL_FOLDER}application.properties" \ 30 | "$KAFKA_HOST" "$KAFKA_PORT" "$KAFKA_CLIENT_ID" \ 31 | "$MQTT_HOST" "$MQTT_PORT" "$MQTT_CLIENT_ID" "$MQTT_QOS" \ 32 | "$TOPIC_MAPPING" \ 33 | && java -jar -Xmx1024m -Xms512m ${JAR_FILE_NAME} 34 | 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Arthur Grigorjan, Fraunhofer Institute for Manufacturing Engineering and Automation (IPA) 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A simple MQTT to Apache Kafka Connector 2 | [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fipa-digitools%2Fsimple-kafka-mqtt-connector.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fipa-digitools%2Fsimple-kafka-mqtt-connector?ref=badge_shield) 3 | 4 | 5 | 6 | 7 | ## Usage 8 | 9 | #### Prerequisites 10 | * Java 8 or higher 11 | 12 | 13 | #### Configuration 14 | 15 | * Edit the application.properties-file 16 | 17 | * kafka.host = IP THROUGH WHICH THE KAFKA BROKER WILL BE REACHED [HOSTNAME|IP] 18 | * kafka.port = [INTEGER] 19 | * kafka.client.id = [STRING] 20 | 21 | * mqtt.host = IP THROUGH WHICH THE MQTT BROKER WILL BE REACHED [HOSTNAME|IP] 22 | * mqtt.port = [INTEGER] 23 | * mqtt.client.id = [STRING] 24 | * mqtt.qos = Quality of service for MQTT - [INTEGER] Allowed[0,1,2] 25 | 26 | * topic.mapping = How are topics routed from MQTT to Kafka - (Separators >>> and ;) Example mqttTopicA>>>kafkaTopicA;mqttTopicB>>>kafkaTopicB;mqttTopicC>>>kafkaTopicC 27 | 28 | #### How to build 29 | 30 | * Run: `mvn clean install` 31 | 32 | 33 | #### How to run 34 | 35 | * Place the jar with dependencies and your edited application.properties-file in the same directory 36 | 37 | * Open a bash or CMD in the same directory as the .jar 38 | 39 | * Run: `java -jar simple_kafka_mqtt_connector-0.0.1-SNAPSHOT-jar-with-dependencies.jar` 40 | 41 | 42 | ## Usage with Docker 43 | 44 | * Docker Hub https://hub.docker.com/r/arthurgrigo/simple-kafka-mqtt-connector 45 | 46 | * Run (edit enviroment variables to your needs!) : `docker run -d -t -i -e KAFKA_HOST='localhost' -e KAFKA_PORT=9092 -e KAFKA_CLIENT_ID='testing-kafka-producer-1' -e MQTT_HOST='localhost' -e MQTT_PORT=1883 -e MQTT_CLIENT_ID='mqtt-client-1' -e MQTT_QOS=2 -e TOPIC_MAPPING='robotgroup001/robot001>>>test;robotgroup001/robot002>>>test02;robotgroup001/robot003>>>test03' --name simple-kafka-mqtt-connector arthurgrigo/simple-kafka-mqtt-connector:latest` 47 | 48 | 49 | ## Usage with Docker-Compose 50 | 51 | * See [docker-compose-examples](docker-compose) 52 | 53 | #### Standalone 54 | 55 | * Stand-alone container 56 | 57 | * [docker-compose.yml](docker-compose/standalone/docker-compose.yml) 58 | 59 | * Run: `docker-compose up -d` 60 | 61 | 62 | #### Full Stack 63 | 64 | * Full Stack (mqtt-broker, zookeeper, kafka-broker, simple-kafka-mqtt-connector) 65 | 66 | * [docker-compose.yml](docker-compose/fullstack/docker-compose.yml) 67 | 68 | * [env.list](docker-compose/fullstack/env.list) 69 | 70 | * Place docker-compose.yml and env.list in the same directory 71 | 72 | * Edit env.list to your needs! 73 | 74 | * Run: `docker-compose up -d` 75 | 76 | 77 | ## License 78 | See [LICENSE](LICENSE) file for License 79 | 80 | [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fipa-digitools%2Fsimple-kafka-mqtt-connector.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fipa-digitools%2Fsimple-kafka-mqtt-connector?ref=badge_large) -------------------------------------------------------------------------------- /docker-compose/fullstack/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | simple-kafka-mqtt-connector: 6 | image: arthurgrigo/simple-kafka-mqtt-connector:0.1.0 7 | restart: always 8 | hostname: simple-kafka-mqtt-connector 9 | container_name: simple-kafka-mqtt-connector 10 | env_file: 11 | - env.list 12 | networks: 13 | - kafka_mqtt_network 14 | 15 | mosquitto: 16 | image: eclipse-mosquitto:2.0.14 17 | restart: always 18 | hostname: mosquitto 19 | container_name: mosquitto 20 | ports: 21 | - "1883:1883" 22 | networks: 23 | - kafka_mqtt_network 24 | 25 | zookeeper: 26 | image: wurstmeister/zookeeper 27 | restart: always 28 | hostname: zookeeper 29 | container_name: zookeeper 30 | ports: 31 | - "2181:2181" 32 | env_file: 33 | - env.list 34 | volumes: 35 | - ./zookeeper/data:/data 36 | - ./zookeeper/datalog:/datalog 37 | networks: 38 | - kafka_mqtt_network 39 | 40 | kafka: 41 | image: wurstmeister/kafka:2.13-2.8.1 42 | restart: always 43 | hostname: kafka 44 | container_name: kafka 45 | ports: 46 | - "9092:9092" 47 | env_file: 48 | - env.list 49 | volumes: 50 | - /var/run/docker.sock:/var/run/docker.sock 51 | networks: 52 | - kafka_mqtt_network 53 | 54 | networks: 55 | kafka_mqtt_network: 56 | driver: bridge 57 | ipam: 58 | config: 59 | - subnet: 192.168.115.0/24 60 | gateway: 192.168.115.1 61 | -------------------------------------------------------------------------------- /docker-compose/fullstack/env.list: -------------------------------------------------------------------------------- 1 | ######## 2 | # 3 | # simple-kafka-mqtt-connector 4 | # 5 | ######## 6 | 7 | # !EDITME! IP THROUGH WHICH THE KAFKA BROKER WILL BE REACHED 8 | KAFKA_HOST= kafka 9 | 10 | KAFKA_PORT= 9092 11 | KAFKA_CLIENT_ID= simple-kafka-mqtt-connector-1 12 | 13 | # !EDITME! IP THROUGH WHICH THE MQTT BROKER WILL BE REACHED 14 | MQTT_HOST= mosquitto 15 | MQTT_PORT= 1883 16 | MQTT_CLIENT_ID= simple-kafka-mqtt-connector-1 17 | MQTT_QOS= 2 18 | 19 | #!EDITME! mqttTopicA>>>kafkaTopicA;mqttTopicB>>>kafkaTopicB;mqttTopicC>>>kafkaTopicC 20 | TOPIC_MAPPING= robotgroup001/robot001>>>test;robotgroup001/robot002>>>test02;robotgroup001/robot003>>>test03 21 | 22 | ######## 23 | # 24 | # zookeeper 25 | # 26 | ######## 27 | ZOO_MY_ID= 1 28 | ZOO_PORT= 2181 29 | ZOO_SERVERS= server.1=zookeeper:2888:3888 30 | 31 | 32 | ######## 33 | # 34 | # kafka 35 | # 36 | ######## 37 | 38 | # !EDITME! How will kafka producers reach this broker? 39 | KAFKA_ADVERTISED_HOST_NAME= kafka 40 | 41 | KAFKA_AUTO_CREATE_TOPICS_ENABLE= true 42 | # KAFKA_CREATE_TOPICS= "test:1:1" 43 | KAFKA_ZOOKEEPER_CONNECT= zookeeper:2181 -------------------------------------------------------------------------------- /docker-compose/standalone/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | simple-kafka-mqtt-connector: 6 | image: arthurgrigo/simple-kafka-mqtt-connector:0.1.0 7 | restart: always 8 | hostname: simple-kafka-mqtt-connector 9 | container_name: simple-kafka-mqtt-connector 10 | environment: 11 | KAFKA_HOST: 172.21.5.239 #EDITME IP THROUGH WHICH THE KAFKA BROKER WILL BE REACHED 12 | KAFKA_PORT: 9092 13 | KAFKA_CLIENT_ID: simple-kafka-mqtt-connector-1 14 | MQTT_HOST: 172.21.5.239 #EDITME IP THROUGH WHICH THE MQTT BROKER WILL BE REACHED 15 | MQTT_PORT: 1883 16 | MQTT_CLIENT_ID: simple-kafka-mqtt-connector-1 17 | MQTT_QOS: 2 18 | TOPIC_MAPPING: robotgroup001/robot001>>>test;robotgroup001/robot002>>>test02;robotgroup001/robot003>>>test03 #EDITME mqttTopicA>>>kafkaTopicA;mqttTopicB>>>kafkaTopicB;mqttTopicC>>>kafkaTopicC 19 | -------------------------------------------------------------------------------- /docker/setConfiguration.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Read arguments 4 | application_properties_file=${1} 5 | 6 | KAFKA_HOST=${2} 7 | KAFKA_PORT=${3} 8 | KAFKA_CLIENT_ID=${4} 9 | 10 | MQTT_HOST=${5} 11 | MQTT_PORT=${6} 12 | MQTT_CLIENT_ID=${7} 13 | MQTT_QOS=${8} 14 | 15 | TOPIC_MAPPING=${9} 16 | 17 | 18 | echo "Configuration:" 19 | echo "Application properties file: $application_properties_file" 20 | echo "KAFKA_HOST: $KAFKA_HOST" 21 | echo "KAFKA_PORT: $KAFKA_PORT" 22 | echo "KAFKA_CLIENT_ID: $KAFKA_CLIENT_ID" 23 | echo "MQTT_HOST: $MQTT_HOST" 24 | echo "MQTT_PORT: $MQTT_PORT" 25 | echo "MQTT_CLIENT_ID: $MQTT_CLIENT_ID" 26 | echo "MQTT_QOS: $MQTT_QOS" 27 | echo "TOPIC_MAPPING: $TOPIC_MAPPING" 28 | 29 | # overwrite properties 30 | sed -i "/kafka.host/c\kafka.host = $KAFKA_HOST" ${application_properties_file} 31 | sed -i "/kafka.port/c\kafka.port = $KAFKA_PORT" ${application_properties_file} 32 | sed -i "/kafka.client.id/c\kafka.client.id = $KAFKA_CLIENT_ID" ${application_properties_file} 33 | 34 | sed -i "/mqtt.host/c\mqtt.host = $MQTT_HOST" ${application_properties_file} 35 | sed -i "/mqtt.port/c\mqtt.port = $MQTT_PORT" ${application_properties_file} 36 | sed -i "/mqtt.client.id/c\mqtt.client.id = $MQTT_CLIENT_ID" ${application_properties_file} 37 | sed -i "/mqtt.qos/c\mqtt.qos = $MQTT_QOS" ${application_properties_file} 38 | 39 | sed -i "/topic.mapping/c\topic.mapping = $TOPIC_MAPPING" ${application_properties_file} 40 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | de.fhg.ipa.null70 8 | simple_kafka_mqtt_connector 9 | 0.1.0 10 | jar 11 | 12 | 13 | This application receives messages from a mqtt broker and sends the messages to a kafka cluster. 14 | Topic mapping is configurable. 15 | Keep the mqtt-topic-semantics in mind. 16 | Make sure the kafka topics are created or enable auto.create.topics.enable in your kafka cluster. 17 | 18 | 19 | 20 | 21 | agrg 22 | Arthur Grigorjan 23 | arthur.grigorjan@ipa.fraunhofer.de 24 | 25 | 26 | dkt 27 | Dinh Khoi Tran 28 | dinh.khoi.tran@ipa.fraunhofer.de 29 | 30 | 31 | 32 | 33 | 34 | 1.8 35 | 1.8 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | maven-assembly-plugin 44 | 45 | 46 | 47 | de.fhg.ipa.null70.simple_kafka_mqtt_connector.Main 48 | 49 | 50 | 51 | jar-with-dependencies 52 | 53 | 54 | 55 | 56 | make-assembly 57 | install 58 | 59 | single 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | commons-configuration 74 | commons-configuration 75 | 1.10 76 | 77 | 78 | 79 | 80 | org.eclipse.paho 81 | org.eclipse.paho.client.mqttv3 82 | 1.2.5 83 | 84 | 85 | 86 | 87 | org.apache.kafka 88 | kafka-clients 89 | 2.8.1 90 | 91 | 92 | 93 | org.apache.kafka 94 | kafka-streams 95 | 2.8.1 96 | 97 | 98 | 99 | org.apache.logging.log4j 100 | log4j-slf4j-impl 101 | 2.17.0 102 | 103 | 104 | 105 | 106 | org.junit.jupiter 107 | junit-jupiter-api 108 | 5.3.1 109 | test 110 | 111 | 112 | 113 | org.junit.jupiter 114 | junit-jupiter-engine 115 | 5.3.1 116 | test 117 | 118 | 119 | 120 | org.junit.jupiter 121 | junit-jupiter-params 122 | 5.3.1 123 | test 124 | 125 | 126 | 127 | 128 | -------------------------------------------------------------------------------- /simple_kafka_mqtt_connector.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/main/java/de/fhg/ipa/null70/simple_kafka_mqtt_connector/Main.java: -------------------------------------------------------------------------------- 1 | package de.fhg.ipa.null70.simple_kafka_mqtt_connector; 2 | 3 | import org.apache.commons.configuration.CompositeConfiguration; 4 | import org.apache.commons.configuration.ConfigurationException; 5 | import org.apache.commons.configuration.PropertiesConfiguration; 6 | import org.apache.commons.configuration.SystemConfiguration; 7 | import org.apache.logging.log4j.LogManager; 8 | import org.apache.logging.log4j.Logger; 9 | 10 | public class Main { 11 | 12 | private static final Logger logger = LogManager.getLogger(Main.class); 13 | 14 | public static void main(String[] args) { 15 | logger.info("######## STARTING THE SIMPLE-KAFKA-MQTT-CONNECTOR ########"); 16 | SimpleKafkaMQTTConnector simpleKafkaMQTTConnector = new SimpleKafkaMQTTConnector(); 17 | 18 | 19 | // Read application.properties 20 | CompositeConfiguration config = new CompositeConfiguration(); 21 | config.addConfiguration(new SystemConfiguration()); 22 | 23 | config.addConfiguration(new PropertiesConfiguration()); 24 | try { 25 | config.addConfiguration(new PropertiesConfiguration("application.properties")); 26 | } catch (ConfigurationException e) { 27 | e.printStackTrace(); 28 | } 29 | 30 | // Properties configuration 31 | String kafkaHost = config.getString("kafka.host"); 32 | String kafkaPort = config.getString("kafka.port"); 33 | String kafkaClientId = config.getString("kafka.client.id"); 34 | 35 | String mqttHost = config.getString("mqtt.host"); 36 | String mqttPort = config.getString("mqtt.port"); 37 | String mqttClientId = config.getString("mqtt.client.id"); 38 | Integer mqttQos = Integer.parseInt(config.getString("mqtt.qos").trim()); 39 | 40 | String topicMapping = config.getString("topic.mapping"); 41 | 42 | logger.info("-------- APPLICATION PROPERTIES --------"); 43 | logger.info("kafkaHost = " + kafkaHost); 44 | logger.info("kafkaPort = " + kafkaPort); 45 | logger.info("mqttHost = " + mqttHost); 46 | logger.info("mqttPort = " + mqttPort); 47 | logger.info("topicMapping = " + topicMapping); 48 | logger.info("----------------------------------------"); 49 | logger.info(""); 50 | 51 | simpleKafkaMQTTConnector.run(kafkaHost, kafkaPort, kafkaClientId, mqttHost, mqttPort, mqttClientId, mqttQos, topicMapping); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/de/fhg/ipa/null70/simple_kafka_mqtt_connector/SimpleKafkaMQTTConnector.java: -------------------------------------------------------------------------------- 1 | package de.fhg.ipa.null70.simple_kafka_mqtt_connector; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | import java.util.Properties; 8 | 9 | import org.apache.kafka.clients.producer.KafkaProducer; 10 | import org.apache.kafka.clients.producer.ProducerConfig; 11 | import org.apache.kafka.clients.producer.ProducerRecord; 12 | import org.apache.kafka.common.KafkaException; 13 | import org.apache.kafka.common.serialization.IntegerSerializer; 14 | import org.apache.kafka.common.serialization.StringSerializer; 15 | import org.apache.logging.log4j.LogManager; 16 | import org.apache.logging.log4j.Logger; 17 | import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; 18 | import org.eclipse.paho.client.mqttv3.MqttCallback; 19 | import org.eclipse.paho.client.mqttv3.MqttClient; 20 | import org.eclipse.paho.client.mqttv3.MqttConnectOptions; 21 | import org.eclipse.paho.client.mqttv3.MqttException; 22 | import org.eclipse.paho.client.mqttv3.MqttMessage; 23 | import org.eclipse.paho.client.mqttv3.MqttTopic; 24 | 25 | public class SimpleKafkaMQTTConnector { 26 | private static final Logger logger = LogManager.getLogger(SimpleKafkaMQTTConnector.class); 27 | 28 | // Key = mqtt-topic input , Value = kafka-topics for output 29 | private static HashMap> mqttKafkaTopicMap = new HashMap(); 30 | 31 | 32 | public void run(String kafkaHost, String kafkaPort, String kafkaClientId, String mqttHost, String mqttPort, String mqttClientId, Integer mqttQos, String topicMapping) { 33 | // Initialize topic routing map 34 | initTopicsRoutingMap(topicMapping); 35 | 36 | // Init and start kafka producer 37 | KafkaProducer kafkaProducer = initKafkaProducer(kafkaHost, kafkaPort, kafkaClientId); 38 | 39 | // Setup and start the mqtt client 40 | initMqttClient(mqttHost, mqttPort, mqttClientId, mqttQos, kafkaProducer); 41 | } 42 | 43 | private KafkaProducer initKafkaProducer(String kafkaHost, String kafkaPort, String kafkaClientId) { 44 | logger.trace("Creating Kafka Producer..."); 45 | Properties props = new Properties(); 46 | props.put(ProducerConfig.CLIENT_ID_CONFIG, kafkaClientId); 47 | props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaHost + ":" + kafkaPort); 48 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName()); 49 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); 50 | KafkaProducer kafkaProducer = new KafkaProducer<>(props); 51 | logger.trace("Kafka producer ready to produce..."); 52 | return kafkaProducer; 53 | } 54 | 55 | private HashMap> checkForKey(String mqttTopic) 56 | { 57 | HashMap> keysForWhichTheTopicApplies = new HashMap>(); 58 | try{ 59 | for (Map.Entry mapElement : mqttKafkaTopicMap.entrySet()) 60 | { 61 | String key = (String)mapElement.getKey(); 62 | if(MqttTopic.isMatched(key, mqttTopic)) 63 | { 64 | keysForWhichTheTopicApplies.put((String)mapElement.getKey(), ( ArrayList)mapElement.getValue()); 65 | } 66 | } 67 | }catch(Exception ex) 68 | { 69 | ex.printStackTrace(); 70 | } 71 | return keysForWhichTheTopicApplies; 72 | } 73 | 74 | private void initMqttClient(String mqttHost, String mqttPort, String mqttClientId, Integer mqttQos, KafkaProducer kafkaProducer) { 75 | 76 | /*** 77 | * MQTT Client 78 | * **/ 79 | MqttClient client = null; 80 | try { 81 | client = new MqttClient( 82 | "tcp://" + mqttHost + ":" + mqttPort, mqttClientId); 83 | } catch (MqttException e) { 84 | e.printStackTrace(); 85 | } 86 | 87 | MqttConnectOptions options = new MqttConnectOptions(); 88 | // use a persistent session.. 89 | options.setCleanSession(false); 90 | 91 | options.setWill( 92 | "will/topic", //Topic 93 | "Disconnected!".getBytes(), //Nachrichteninhalt 94 | mqttQos, //QoS 95 | false); //Retained message? 96 | 97 | try { 98 | client.connect(options); 99 | } catch (MqttException e) { 100 | e.printStackTrace(); 101 | } 102 | 103 | try { 104 | // Subscribe all configured topics via mqtt 105 | for (Map.Entry mapElement : mqttKafkaTopicMap.entrySet()) { 106 | String key = (String)mapElement.getKey(); 107 | client.subscribe(key); 108 | } 109 | 110 | } catch (MqttException e) { 111 | e.printStackTrace(); 112 | } 113 | 114 | 115 | client.setCallback(new MqttCallback() { 116 | 117 | @Override 118 | public void connectionLost(Throwable throwable) { } 119 | 120 | @Override 121 | public void messageArrived(String mqttTopic, MqttMessage mqttMessage) throws Exception { 122 | // Checks through which mqtt-topic this message was sent and sends it to the pre-configured corresponding kafka topics.. 123 | 124 | String message = new String(mqttMessage.getPayload()); 125 | logger.info(mqttTopic + " - " + message); 126 | 127 | try { 128 | HashMap> erg = checkForKey(mqttTopic); 129 | if(erg.size()>=1) { 130 | for (Map.Entry mapElement : erg.entrySet()) { 131 | ((ArrayList)mapElement.getValue()).forEach(kafkaTopic -> { 132 | kafkaProducer.send(new ProducerRecord<>(kafkaTopic, message)); 133 | }); 134 | logger.trace("send Message to kafka - " + message); 135 | } 136 | } 137 | } catch (KafkaException e) { 138 | logger.error("Exception occurred – Check log for more details.\n" + e.getMessage()); 139 | logger.warn("There seems to be an issue with the kafka connection. Currently no messages are forwarded to the kafka cluster!!!!"); 140 | // System.exit(-1); 141 | } 142 | 143 | } 144 | 145 | @Override 146 | public void deliveryComplete(IMqttDeliveryToken t) { } 147 | }); 148 | } 149 | 150 | public static void initTopicsRoutingMap(String topicMappingString){ 151 | logger.info("Setting up topic mapping (MQTT >>> Kafka) ..."); 152 | Arrays.asList(topicMappingString.split(";")).forEach(s -> { 153 | String[] pair = s.split(">>>"); 154 | String mqttTopic = pair[0]; 155 | String kafkaTopic = pair[1]; 156 | if( !mqttKafkaTopicMap.containsKey(mqttTopic) ){ 157 | mqttKafkaTopicMap.put(mqttTopic, new ArrayList()); 158 | } 159 | mqttKafkaTopicMap.get(mqttTopic).add(kafkaTopic); 160 | logger.info(mqttTopic + " >>> " + kafkaTopic); 161 | } 162 | ); 163 | } 164 | 165 | } 166 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | kafka.host=192.168.56.102 2 | #kafka.host=localhost 3 | 4 | kafka.port=9092 5 | kafka.client.id=testing-kafka-producer-1 6 | 7 | mqtt.host=192.168.56.102 8 | #mqtt.host=localhost 9 | mqtt.port=1883 10 | mqtt.client.id=mqtt-client-1 11 | mqtt.qos=2 12 | 13 | topic.mapping=robotgroup001/robot001>>>test;robotgroup001/robot002>>>test02;robotgroup001/robot003>>>test03 14 | 15 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | --------------------------------------------------------------------------------