├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── build.gradle ├── config ├── checkstyle │ └── checkstyle.xml └── mqtt.properties ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle └── src ├── main ├── java │ └── com │ │ └── evokly │ │ └── kafka │ │ └── connect │ │ └── mqtt │ │ ├── MqttMessageProcessor.java │ │ ├── MqttSourceConnector.java │ │ ├── MqttSourceConnectorConfig.java │ │ ├── MqttSourceConstant.java │ │ ├── MqttSourceInterceptMessage.java │ │ ├── MqttSourceTask.java │ │ ├── sample │ │ └── DumbProcessor.java │ │ ├── ssl │ │ └── SslUtils.java │ │ └── util │ │ ├── Utils.java │ │ └── Version.java └── resources │ └── kafka-connect-mqtt-version.properties └── test ├── java └── com │ └── evokly │ └── kafka │ └── connect │ └── mqtt │ ├── MqttSourceConnectorTest.java │ ├── MqttSourceTaskTest.java │ └── util │ └── VersionTest.java └── resources └── kafka-connect-mqtt-version.properties /.gitignore: -------------------------------------------------------------------------------- 1 | *.pem 2 | 3 | # Created by https://www.gitignore.io/api/intellij,java,gradle,osx 4 | 5 | ### Intellij ### 6 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 7 | 8 | *.iml 9 | 10 | ## Directory-based project format: 11 | .idea/ 12 | # if you remove the above rule, at least ignore the following: 13 | 14 | # User-specific stuff: 15 | # .idea/workspace.xml 16 | # .idea/tasks.xml 17 | # .idea/dictionaries 18 | # .idea/shelf 19 | 20 | # Sensitive or high-churn files: 21 | # .idea/dataSources.ids 22 | # .idea/dataSources.xml 23 | # .idea/sqlDataSources.xml 24 | # .idea/dynamic.xml 25 | # .idea/uiDesigner.xml 26 | 27 | # Gradle: 28 | # .idea/gradle.xml 29 | # .idea/libraries 30 | 31 | # Mongo Explorer plugin: 32 | # .idea/mongoSettings.xml 33 | 34 | ## File-based project format: 35 | *.ipr 36 | *.iws 37 | 38 | ## Plugin-specific files: 39 | 40 | # IntelliJ 41 | /out/ 42 | 43 | # mpeltonen/sbt-idea plugin 44 | .idea_modules/ 45 | 46 | # JIRA plugin 47 | atlassian-ide-plugin.xml 48 | 49 | # Crashlytics plugin (for Android Studio and IntelliJ) 50 | com_crashlytics_export_strings.xml 51 | crashlytics.properties 52 | crashlytics-build.properties 53 | fabric.properties 54 | 55 | 56 | ### Java ### 57 | *.class 58 | 59 | # Mobile Tools for Java (J2ME) 60 | .mtj.tmp/ 61 | 62 | # Package Files # 63 | *.jar 64 | *.war 65 | *.ear 66 | 67 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 68 | hs_err_pid* 69 | 70 | 71 | ### Gradle ### 72 | .gradle 73 | build/ 74 | 75 | # Ignore Gradle GUI config 76 | gradle-app.setting 77 | 78 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 79 | !gradle-wrapper.jar 80 | 81 | # Cache of project 82 | .gradletasknamecache 83 | 84 | 85 | ### OSX ### 86 | .DS_Store 87 | .AppleDouble 88 | .LSOverride 89 | 90 | # Icon must end with two \r 91 | Icon 92 | 93 | # Thumbnails 94 | ._* 95 | 96 | # Files that might appear in the root of a volume 97 | .DocumentRevisions-V100 98 | .fseventsd 99 | .Spotlight-V100 100 | .TemporaryItems 101 | .Trashes 102 | .VolumeIcon.icns 103 | 104 | # Directories potentially created on remote AFP share 105 | .AppleDB 106 | .AppleDesktop 107 | Network Trash Folder 108 | Temporary Items 109 | .apdisk 110 | 111 | /msg.bin 112 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | notifications: 3 | slack: 4 | secure: C3MDCot+GPaGJ15IQwJ6qD16wzYpXY2ddYyTsEW5CjOVxP6sPN6KGZ7MlzlRmcRwD+a+XYskqVkmKgZFN8JMemS6Y5JeCzeFXRY2FkB7zVdvDmn1AOxblnJP5TUY6WbVMf4SjDzRxs2tpdbT2dO2OgujlGw1GK13iL7cgLOYbuqiPU+IQHZFRMIeoxk79GQ1nIbMkux9LHW0g5HPivWuYiQ8DYtxHBO3PzEefPJVJD4JDpPJaayzB5ZzGdD4qeoQh6cfIKymcJ/7qoXdl5Rzl5RkwWdsaMqME3C88xa949YqJ9EhuYYGhCZeA9qikplXsQxFbIWEdeJXYPWLN8LP1dPm+FjWdmObQpIBquJy1i4rhOpAKdUF7Wny8Yoe/2qaXFBKO9DuNc1nWwcv1TOE2RM4qtDMGBugSs9zCA6lcnELMLLLj8YGtS+2irgGEDOTNfk/ZSKuvgCT6Y2RHWUILtJw9oDeiFvFD+uPiPyptOtuufLBFtxRb0A2Jju1r4D+IyC7NPBb4Y55yQG2tM/5qodZOKBdhtiQGHhPkVhFryzSGh55mdBuM85Vf//UVJ7ASWFLmg1LprQ015DeQS8YnNZmc3Syl5h5iYly5gk9VjIuc3L6I0GF1dXcfSe906i/gYVDlAAfmLSOY02DrFBIAKZY2If0s2D2cy0UAgsfPQI= 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Evokly S.A. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mqtt to Apache Kafka Connect [![Build Status](https://travis-ci.org/evokly/kafka-connect-mqtt.svg?branch=master)](https://travis-ci.org/evokly/kafka-connect-mqtt) [ ![Download](https://api.bintray.com/packages/evokly/maven/kafka-connect-mqtt/images/download.svg) ](https://bintray.com/evokly/maven/kafka-connect-mqtt/_latestVersion) 2 | 3 | ## Prerequisites 4 | * [Apache Kafka](https://kafka.apache.org/) (0.10.x version) is publish-subscribe messaging rethought as a distributed commit log. 5 | 6 | ## Usage 7 | **For development:** 8 | 9 | * run check (checkstyle, findbugs, test): 10 | `./gradlew clean check` 11 | 12 | * run project: 13 | `connect-standalone.sh /usr/local/etc/kafka/connect-standalone.properties config/mqtt.properties` 14 | * libs needs to be added to CLASSPATH: 15 | * `kafka-connect-mqtt-{project.version}.jar` 16 | * `org.eclipse.paho.client.mqttv3-1.0.2.jar` 17 | * if used with ssl there are more.. (`./gradlew copyRuntimeLibs` copies all runtime libs to `./build/output/lib`) 18 | 19 | **For production:** 20 | 21 | * build project: `./gradlew clean jar` - output `./build/libs` 22 | 23 | * generate API documentation: `./gradlew javadoc` - output `./build/docs/javadoc` 24 | 25 | ## License 26 | See [LICENSE](LICENSE) file for License -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'checkstyle' 4 | id 'findbugs' 5 | id 'pmd' 6 | id 'jacoco' 7 | id 'net.researchgate.release' version '2.4.0' 8 | id "com.jfrog.bintray" version '1.7' 9 | id 'maven' 10 | id 'maven-publish' 11 | } 12 | 13 | group 'com.evokly' 14 | 15 | publishing { 16 | publications { 17 | MyPublication(MavenPublication) { 18 | from components.java 19 | groupId 'com.evokly' 20 | artifactId 'kafka-connect-mqtt' 21 | version '1.0' 22 | } 23 | } 24 | } 25 | 26 | 27 | ext { 28 | kafkaVersion = '0.10.0.0' 29 | } 30 | 31 | 32 | processResources { 33 | expand project.properties 34 | } 35 | 36 | repositories { 37 | mavenCentral() 38 | 39 | maven { url "https://repo.eclipse.org/content/repositories/paho-releases/" } 40 | } 41 | 42 | dependencies { 43 | testCompile group: 'junit', name: 'junit', version: '4.12' 44 | compile "org.apache.kafka:connect-api:$kafkaVersion" 45 | compile 'org.eclipse.paho:org.eclipse.paho.client.mqttv3:1.0.2' 46 | compile 'org.bouncycastle:bcprov-jdk15on:1.54' 47 | compile 'org.bouncycastle:bcpkix-jdk15on:1.54' 48 | compile 'org.bouncycastle:bcpg-jdk15on:1.54' 49 | compile 'commons-io:commons-io:2.4' 50 | compile 'org.slf4j:slf4j-api:1.7.14' 51 | testCompile 'org.slf4j:slf4j-simple:1.7.14' 52 | } 53 | 54 | checkstyle { 55 | repositories { 56 | mavenCentral() 57 | } 58 | configurations { 59 | checkstyle 60 | } 61 | dependencies { 62 | checkstyle 'com.puppycrawl.tools:checkstyle:6.12.1' 63 | } 64 | } 65 | 66 | tasks.withType(FindBugs) { 67 | reports { 68 | xml.enabled = true 69 | html.enabled = false 70 | } 71 | } 72 | 73 | task copyRuntimeLibs(type: Copy) { 74 | into "$buildDir/output/lib" 75 | from configurations.runtime 76 | } 77 | 78 | 79 | bintray { 80 | user = System.getenv('BINTRAY_USER') 81 | key = System.getenv('BINTRAY_KEY') 82 | pkg { 83 | repo = 'maven' 84 | name = 'kafka-connect-mqtt' 85 | userOrg = 'evokly' 86 | licenses = ['MIT'] 87 | vcsUrl = 'https://github.com/evokly/kafka-connect-mqtt.git' 88 | version { 89 | name = '1.0' 90 | vcsTag = '1.0' 91 | } 92 | publications = ['MyPublication'] 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /config/checkstyle/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 76 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 101 | 102 | 103 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 123 | 124 | 125 | 126 | 128 | 129 | 130 | 131 | 132 | 133 | 135 | 136 | 137 | 138 | 140 | 141 | 142 | 143 | 145 | 146 | 147 | 148 | 150 | 152 | 154 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | -------------------------------------------------------------------------------- /config/mqtt.properties: -------------------------------------------------------------------------------- 1 | ## 2 | # Basic 3 | ## 4 | name=mqtt 5 | connector.class=com.evokly.kafka.connect.mqtt.MqttSourceConnector 6 | tasks.max=1 7 | 8 | #Settings 9 | # Where to put processed messages - default to `mqtt` 10 | #kafka.topic=mqtt 11 | # What client id to use - defaults to `null` which means random client_id 12 | #mqtt.client_id=null 13 | # Use clean session in connection? - default `true` 14 | #mqtt.clean_session=true 15 | # What mqtt connection timeout to use - defaults to `30` seconds 16 | #mqtt.connection_timeout=30 17 | # What mqtt connection keep alive to use - defaults to `60` seconds 18 | #mqtt.keep_alive_interval=60 19 | # Mqtt broker address to use - defaults to `tcp://localhost:1883` 20 | # if using TLS then certs can be used 21 | #mqtt.server_uris=tcp://localhost:1883 22 | # Mqtt topic to listen to - defaults to `#` (wildcard - all) 23 | #mqtt.topic=# 24 | # Mqtt QoS to use - defaults to 1 25 | #mqtt.qos=1 26 | # CA cert to use to connect to mqtt broker, can be used when connecting to TLS secured brokers - default `null` 27 | #mqtt.ssl.ca_cert=null 28 | # Client cert to use to connect to mqtt broker, can be used when connecting to TLS secured brokers - default `null` 29 | #mqtt.ssl.cert=null 30 | # Client key to use to connect to mqtt broker, can be used when connecting to TLS secured brokers - default `null` 31 | #mqtt.ssl.key=null 32 | # Username to use to connect to mqtt broker - default `null` 33 | #mqtt.user=null 34 | # Username to use to connect to mqtt broker - default `null` 35 | #mqtt.password=null 36 | # Message processor class to use to process mqtt messages before puting them on kafka - defaults to samle `DumbProcessor` 37 | #message_processor_class=com.evokly.kafka.connect.mqtt.sample.DumbProcessor 38 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | version=1.1-SNAPSHOT -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evokly/kafka-connect-mqtt/05bf8454c08b0067ac613b61ea5f25e46926fc06/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Fri Feb 05 04:48:03 CET 2016 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-2.14-all.zip 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # Attempt to set APP_HOME 46 | # Resolve links: $0 may be a link 47 | PRG="$0" 48 | # Need this for relative symlinks. 49 | while [ -h "$PRG" ] ; do 50 | ls=`ls -ld "$PRG"` 51 | link=`expr "$ls" : '.*-> \(.*\)$'` 52 | if expr "$link" : '/.*' > /dev/null; then 53 | PRG="$link" 54 | else 55 | PRG=`dirname "$PRG"`"/$link" 56 | fi 57 | done 58 | SAVED="`pwd`" 59 | cd "`dirname \"$PRG\"`/" >/dev/null 60 | APP_HOME="`pwd -P`" 61 | cd "$SAVED" >/dev/null 62 | 63 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 64 | 65 | # Determine the Java command to use to start the JVM. 66 | if [ -n "$JAVA_HOME" ] ; then 67 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 68 | # IBM's JDK on AIX uses strange locations for the executables 69 | JAVACMD="$JAVA_HOME/jre/sh/java" 70 | else 71 | JAVACMD="$JAVA_HOME/bin/java" 72 | fi 73 | if [ ! -x "$JAVACMD" ] ; then 74 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 75 | 76 | Please set the JAVA_HOME variable in your environment to match the 77 | location of your Java installation." 78 | fi 79 | else 80 | JAVACMD="java" 81 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 82 | 83 | Please set the JAVA_HOME variable in your environment to match the 84 | location of your Java installation." 85 | fi 86 | 87 | # Increase the maximum file descriptors if we can. 88 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 89 | MAX_FD_LIMIT=`ulimit -H -n` 90 | if [ $? -eq 0 ] ; then 91 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 92 | MAX_FD="$MAX_FD_LIMIT" 93 | fi 94 | ulimit -n $MAX_FD 95 | if [ $? -ne 0 ] ; then 96 | warn "Could not set maximum file descriptor limit: $MAX_FD" 97 | fi 98 | else 99 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 100 | fi 101 | fi 102 | 103 | # For Darwin, add options to specify how the application appears in the dock 104 | if $darwin; then 105 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 106 | fi 107 | 108 | # For Cygwin, switch paths to Windows format before running java 109 | if $cygwin ; then 110 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 111 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 112 | JAVACMD=`cygpath --unix "$JAVACMD"` 113 | 114 | # We build the pattern for arguments to be converted via cygpath 115 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 116 | SEP="" 117 | for dir in $ROOTDIRSRAW ; do 118 | ROOTDIRS="$ROOTDIRS$SEP$dir" 119 | SEP="|" 120 | done 121 | OURCYGPATTERN="(^($ROOTDIRS))" 122 | # Add a user-defined pattern to the cygpath arguments 123 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 124 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 125 | fi 126 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 127 | i=0 128 | for arg in "$@" ; do 129 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 130 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 131 | 132 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 133 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 134 | else 135 | eval `echo args$i`="\"$arg\"" 136 | fi 137 | i=$((i+1)) 138 | done 139 | case $i in 140 | (0) set -- ;; 141 | (1) set -- "$args0" ;; 142 | (2) set -- "$args0" "$args1" ;; 143 | (3) set -- "$args0" "$args1" "$args2" ;; 144 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 145 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 146 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 147 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 148 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 149 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 150 | esac 151 | fi 152 | 153 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 154 | function splitJvmOpts() { 155 | JVM_OPTS=("$@") 156 | } 157 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 158 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 159 | 160 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 161 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kafka-connect-mqtt' 2 | 3 | -------------------------------------------------------------------------------- /src/main/java/com/evokly/kafka/connect/mqtt/MqttMessageProcessor.java: -------------------------------------------------------------------------------- 1 | package com.evokly.kafka.connect.mqtt; 2 | 3 | import org.apache.kafka.connect.source.SourceRecord; 4 | import org.eclipse.paho.client.mqttv3.MqttMessage; 5 | 6 | /** 7 | * Copyright 2016 Evokly S.A. 8 | * 9 | *

See LICENSE file for License 10 | **/ 11 | public interface MqttMessageProcessor { 12 | 13 | MqttMessageProcessor process(String topic, MqttMessage message); 14 | 15 | SourceRecord[] getRecords(String kafkaTopic); 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/evokly/kafka/connect/mqtt/MqttSourceConnector.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Evokly S.A. 3 | * See LICENSE file for License 4 | **/ 5 | 6 | package com.evokly.kafka.connect.mqtt; 7 | 8 | import com.evokly.kafka.connect.mqtt.util.Version; 9 | 10 | import org.apache.kafka.common.config.ConfigDef; 11 | import org.apache.kafka.connect.connector.Task; 12 | import org.apache.kafka.connect.source.SourceConnector; 13 | 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | import java.util.ArrayList; 18 | import java.util.HashMap; 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | /** 23 | * MqttSourceConnector is a Kafka Connect SourceConnector implementation that generates tasks to 24 | * ingest mqtt messages. 25 | **/ 26 | public class MqttSourceConnector extends SourceConnector { 27 | private static final Logger log = LoggerFactory.getLogger(MqttSourceConnector.class); 28 | 29 | MqttSourceConnectorConfig mConfig; 30 | private Map mConfigProperties; 31 | 32 | /** 33 | * Get the version of this connector. 34 | * 35 | * @return the version, formatted as a String 36 | */ 37 | @Override 38 | public String version() { 39 | return Version.getVersion(); 40 | } 41 | 42 | /** 43 | * Start this connector. This method will only be called on a clean Connector, i.e. it has 44 | * either just been instantiated and initialized or {@link #stop()} has been invoked. 45 | * 46 | * @param props configuration settings 47 | */ 48 | @Override 49 | public void start(Map props) { 50 | log.info("Start a MqttSourceConnector"); 51 | mConfigProperties = props; 52 | mConfig = new MqttSourceConnectorConfig(props); 53 | } 54 | 55 | /** 56 | * SourceTask implementation for this connector. 57 | * 58 | * @return SourceTask class instance 59 | */ 60 | @Override 61 | public Class taskClass() { 62 | return MqttSourceTask.class; 63 | } 64 | 65 | /** 66 | * Returns a set of configurations for Tasks based on the current configuration, 67 | * producing at most count configurations. 68 | * 69 | * @param maxTasks maximum number of configurations to generate 70 | * 71 | * @return configurations for Tasks 72 | */ 73 | @Override 74 | public List> taskConfigs(int maxTasks) { 75 | List> taskConfigs = new ArrayList<>(1); 76 | Map taskProps = new HashMap<>(mConfigProperties); 77 | taskConfigs.add(taskProps); 78 | return taskConfigs; 79 | } 80 | 81 | /** 82 | * Stop this connector. 83 | */ 84 | @Override 85 | public void stop() { 86 | log.info("Stop the MqttSourceConnector"); 87 | } 88 | 89 | @Override 90 | public ConfigDef config() { 91 | return MqttSourceConnectorConfig.config; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/com/evokly/kafka/connect/mqtt/MqttSourceConnectorConfig.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Evokly S.A. 3 | * See LICENSE file for License 4 | **/ 5 | 6 | package com.evokly.kafka.connect.mqtt; 7 | 8 | import com.evokly.kafka.connect.mqtt.sample.DumbProcessor; 9 | import org.apache.kafka.common.config.AbstractConfig; 10 | import org.apache.kafka.common.config.ConfigDef; 11 | import org.apache.kafka.common.config.ConfigDef.Importance; 12 | import org.apache.kafka.common.config.ConfigDef.Type; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.util.LinkedList; 17 | import java.util.List; 18 | import java.util.Map; 19 | 20 | /** 21 | * MqttSourceConnectorConfig is responsible for correct configuration management. 22 | */ 23 | public class MqttSourceConnectorConfig extends AbstractConfig { 24 | private static final Logger log = LoggerFactory.getLogger(MqttSourceConnector.class); 25 | private static final ConfigDef.Recommender MODE_SSL_RECOMMENDER = new SslRecommender(); 26 | 27 | /** 28 | * Create default mConfig. 29 | * @return default mConfig 30 | */ 31 | public static ConfigDef baseConfigDef() { 32 | return new ConfigDef() 33 | .define(MqttSourceConstant.KAFKA_TOPIC, Type.STRING, "mqtt", Importance.LOW, 34 | "Kafka topic to put received data \n Depends on message processor") 35 | .define(MqttSourceConstant.MQTT_CLIENT_ID, Type.STRING, null, Importance.MEDIUM, 36 | "mqtt client id to use don't set to use random") 37 | .define(MqttSourceConstant.MQTT_CLEAN_SESSION, Type.BOOLEAN, true, Importance.HIGH, 38 | "use clean session in connection?") 39 | .define(MqttSourceConstant.MQTT_CONNECTION_TIMEOUT, Type.INT, 30, Importance.LOW, 40 | "connection timeout to use") 41 | .define(MqttSourceConstant.MQTT_KEEP_ALIVE_INTERVAL, Type.INT, 60, Importance.LOW, 42 | "keepalive interval to use") 43 | .define(MqttSourceConstant.MQTT_SERVER_URIS, Type.STRING, 44 | "tcp://localhost:1883", Importance.HIGH, 45 | "mqtt server to connect to") 46 | .define(MqttSourceConstant.MQTT_TOPIC, Type.STRING, "#", Importance.HIGH, 47 | "mqtt server to connect to") 48 | .define(MqttSourceConstant.MQTT_QUALITY_OF_SERVICE, Type.INT, 1, Importance.LOW, 49 | "mqtt qos to use") 50 | .define(MqttSourceConstant.MQTT_SSL_CA_CERT, Type.STRING, null, Importance.LOW, 51 | "CA cert file to use if using ssl", 52 | "SSL", 1, ConfigDef.Width.LONG, "CA cert", MODE_SSL_RECOMMENDER) 53 | .define(MqttSourceConstant.MQTT_SSL_CERT, Type.STRING, null, Importance.LOW, 54 | "cert file to use if using ssl", 55 | "SSL", 2, ConfigDef.Width.LONG, "Cert", MODE_SSL_RECOMMENDER) 56 | .define(MqttSourceConstant.MQTT_SSL_PRIV_KEY, Type.STRING, null, Importance.LOW, 57 | "cert priv key to use if using ssl", 58 | "SSL", 3, ConfigDef.Width.LONG, "Key", MODE_SSL_RECOMMENDER) 59 | .define(MqttSourceConstant.MQTT_USERNAME, Type.STRING, null, Importance.MEDIUM, 60 | "username to authenticate to mqtt broker") 61 | .define(MqttSourceConstant.MQTT_PASSWORD, Type.STRING, null, Importance.MEDIUM, 62 | "password to authenticate to mqtt broker") 63 | .define(MqttSourceConstant.MESSAGE_PROCESSOR, Type.CLASS, 64 | DumbProcessor.class, Importance.HIGH, 65 | "message processor to use"); 66 | } 67 | 68 | static ConfigDef config = baseConfigDef(); 69 | 70 | /** 71 | * Transform process properties. 72 | * 73 | * @param properties associative array with properties to be process 74 | */ 75 | public MqttSourceConnectorConfig(Map properties) { 76 | super(config, properties); 77 | log.info("Initialize transform process properties"); 78 | } 79 | 80 | private static class SslRecommender implements ConfigDef.Recommender { 81 | 82 | @Override 83 | public List validValues(String name, Map parsedConfig) { 84 | return new LinkedList<>(); 85 | } 86 | 87 | @Override 88 | public boolean visible(String name, Map parsedConfig) { 89 | String mode = (String) parsedConfig.get(MqttSourceConstant.MQTT_SERVER_URIS); 90 | return mode.startsWith("ssl://"); 91 | } 92 | } 93 | 94 | public static void main(String[] args) { 95 | System.out.println(config.toRst()); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/com/evokly/kafka/connect/mqtt/MqttSourceConstant.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Evokly S.A. 3 | * See LICENSE file for License 4 | **/ 5 | 6 | package com.evokly.kafka.connect.mqtt; 7 | 8 | public class MqttSourceConstant { 9 | 10 | public static final String KAFKA_TOPIC = "kafka.topic"; 11 | 12 | public static final String MQTT_CLIENT_ID = "mqtt.client_id"; 13 | public static final String MQTT_CLEAN_SESSION = "mqtt.clean_session"; 14 | public static final String MQTT_CONNECTION_TIMEOUT = "mqtt.connection_timeout"; 15 | public static final String MQTT_KEEP_ALIVE_INTERVAL = "mqtt.keep_alive_interval"; 16 | public static final String MQTT_SERVER_URIS = "mqtt.server_uris"; 17 | public static final String MQTT_TOPIC = "mqtt.topic"; 18 | public static final String MQTT_QUALITY_OF_SERVICE = "mqtt.qos"; 19 | public static final String MQTT_SSL_CA_CERT = "mqtt.ssl.ca_cert"; 20 | public static final String MQTT_SSL_CERT = "mqtt.ssl.cert"; 21 | public static final String MQTT_SSL_PRIV_KEY = "mqtt.ssl.key"; 22 | public static final String MQTT_USERNAME = "mqtt.user"; 23 | public static final String MQTT_PASSWORD = "mqtt.password"; 24 | 25 | public static final String MESSAGE_PROCESSOR = "message_processor_class"; 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/evokly/kafka/connect/mqtt/MqttSourceInterceptMessage.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Evokly S.A. 3 | * See LICENSE file for License 4 | **/ 5 | 6 | package com.evokly.kafka.connect.mqtt; 7 | 8 | import org.eclipse.paho.client.mqttv3.MqttMessage; 9 | 10 | /** 11 | * MqttSourceInterceptMessage is a container for mqtt message. 12 | */ 13 | public class MqttSourceInterceptMessage { 14 | private String mTopic; 15 | private MqttMessage mMessage; 16 | 17 | public MqttSourceInterceptMessage(String topic, MqttMessage message) { 18 | this.mTopic = topic; 19 | this.mMessage = message; 20 | } 21 | 22 | public String getTopic() { 23 | return mTopic; 24 | } 25 | 26 | public byte[] getMessage() { 27 | return mMessage.getPayload(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/evokly/kafka/connect/mqtt/MqttSourceTask.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Evokly S.A. 3 | * See LICENSE file for License 4 | **/ 5 | 6 | package com.evokly.kafka.connect.mqtt; 7 | 8 | import com.evokly.kafka.connect.mqtt.ssl.SslUtils; 9 | import com.evokly.kafka.connect.mqtt.util.Version; 10 | 11 | import org.apache.kafka.connect.source.SourceRecord; 12 | import org.apache.kafka.connect.source.SourceTask; 13 | 14 | import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken; 15 | import org.eclipse.paho.client.mqttv3.MqttCallback; 16 | import org.eclipse.paho.client.mqttv3.MqttClient; 17 | import org.eclipse.paho.client.mqttv3.MqttConnectOptions; 18 | import org.eclipse.paho.client.mqttv3.MqttException; 19 | import org.eclipse.paho.client.mqttv3.MqttMessage; 20 | import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence; 21 | 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.util.ArrayList; 26 | import java.util.Collections; 27 | import java.util.List; 28 | import java.util.Map; 29 | import java.util.concurrent.BlockingQueue; 30 | import java.util.concurrent.LinkedBlockingQueue; 31 | 32 | /** 33 | * MqttSourceTask is a Kafka Connect SourceTask implementation that reads 34 | * from MQTT and generates Kafka Connect records. 35 | */ 36 | public class MqttSourceTask extends SourceTask implements MqttCallback { 37 | private static final Logger log = LoggerFactory.getLogger(MqttSourceConnector.class); 38 | 39 | MqttClient mClient; 40 | String mKafkaTopic; 41 | String mMqttClientId; 42 | BlockingQueue mQueue = new LinkedBlockingQueue<>(); 43 | MqttSourceConnectorConfig mConfig; 44 | 45 | /** 46 | * Get the version of this task. Usually this should be the same as the corresponding 47 | * {@link MqttSourceConnector} class's version. 48 | * 49 | * @return the version, formatted as a String 50 | */ 51 | @Override 52 | public String version() { 53 | return Version.getVersion(); 54 | } 55 | 56 | /** 57 | * Start the task. 58 | * 59 | * @param props initial configuration 60 | */ 61 | @Override 62 | public void start(Map props) { 63 | log.info("Start a MqttSourceTask"); 64 | 65 | mConfig = new MqttSourceConnectorConfig(props); 66 | 67 | 68 | mMqttClientId = mConfig.getString(MqttSourceConstant.MQTT_CLIENT_ID) != null 69 | ? mConfig.getString(MqttSourceConstant.MQTT_CLIENT_ID) 70 | : MqttClient.generateClientId(); 71 | 72 | // Setup Kafka 73 | mKafkaTopic = mConfig.getString(MqttSourceConstant.KAFKA_TOPIC); 74 | 75 | 76 | // Setup MQTT Connect Options 77 | MqttConnectOptions connectOptions = new MqttConnectOptions(); 78 | 79 | String sslCa = mConfig.getString(MqttSourceConstant.MQTT_SSL_CA_CERT); 80 | String sslCert = mConfig.getString(MqttSourceConstant.MQTT_SSL_CERT); 81 | String sslPrivateKey = mConfig.getString(MqttSourceConstant.MQTT_SSL_PRIV_KEY); 82 | 83 | if (sslCa != null 84 | && sslCert != null 85 | && sslPrivateKey != null) { 86 | try { 87 | connectOptions.setSocketFactory( 88 | SslUtils.getSslSocketFactory(sslCa, sslCert, sslPrivateKey, "") 89 | ); 90 | } catch (Exception e) { 91 | log.info("[{}] error creating socketFactory", mMqttClientId); 92 | e.printStackTrace(); 93 | return; 94 | } 95 | } 96 | 97 | if (mConfig.getBoolean(MqttSourceConstant.MQTT_CLEAN_SESSION)) { 98 | connectOptions.setCleanSession( 99 | mConfig.getBoolean(MqttSourceConstant.MQTT_CLEAN_SESSION)); 100 | } 101 | connectOptions.setConnectionTimeout( 102 | mConfig.getInt(MqttSourceConstant.MQTT_CONNECTION_TIMEOUT)); 103 | connectOptions.setKeepAliveInterval( 104 | mConfig.getInt(MqttSourceConstant.MQTT_KEEP_ALIVE_INTERVAL)); 105 | connectOptions.setServerURIs( 106 | mConfig.getString(MqttSourceConstant.MQTT_SERVER_URIS).split(",")); 107 | 108 | if (mConfig.getString(MqttSourceConstant.MQTT_USERNAME) != null) { 109 | connectOptions.setUserName( 110 | mConfig.getString(MqttSourceConstant.MQTT_USERNAME)); 111 | } 112 | 113 | if (mConfig.getString(MqttSourceConstant.MQTT_PASSWORD) != null) { 114 | connectOptions.setPassword( 115 | mConfig.getString(MqttSourceConstant.MQTT_PASSWORD).toCharArray()); 116 | } 117 | 118 | // Connect to Broker 119 | try { 120 | // Address of the server to connect to, specified as a URI, is overridden using 121 | // MqttConnectOptions#setServerURIs(String[]) bellow. 122 | mClient = new MqttClient("tcp://127.0.0.1:1883", mMqttClientId, 123 | new MemoryPersistence()); 124 | mClient.setCallback(this); 125 | mClient.connect(connectOptions); 126 | 127 | log.info("[{}] Connected to Broker", mMqttClientId); 128 | } catch (MqttException e) { 129 | log.error("[{}] Connection to Broker failed!", mMqttClientId, e); 130 | } 131 | 132 | // Setup topic 133 | try { 134 | String topic = mConfig.getString(MqttSourceConstant.MQTT_TOPIC); 135 | Integer qos = mConfig.getInt(MqttSourceConstant.MQTT_QUALITY_OF_SERVICE); 136 | 137 | mClient.subscribe(topic, qos); 138 | 139 | log.info("[{}] Subscribe to '{}' with QoS '{}'", mMqttClientId, topic, 140 | qos.toString()); 141 | } catch (MqttException e) { 142 | log.error("[{}] Subscribe failed! ", mMqttClientId, e); 143 | } 144 | } 145 | 146 | /** 147 | * Stop this task. 148 | */ 149 | @Override 150 | public void stop() { 151 | log.info("Stoping the MqttSourceTask"); 152 | 153 | try { 154 | mClient.disconnect(); 155 | 156 | log.info("[{}] Disconnected from Broker.", mMqttClientId); 157 | } catch (MqttException e) { 158 | log.error("[{}] Disconnecting from Broker failed!", mMqttClientId, e); 159 | } 160 | } 161 | 162 | /** 163 | * Poll this SourceTask for new records. This method should block if no data is currently 164 | * available. 165 | * 166 | * @return a list of source records 167 | * 168 | * @throws InterruptedException thread is waiting, sleeping, or otherwise occupied, 169 | * and the thread is interrupted, either before or during the 170 | * activity 171 | */ 172 | @Override 173 | public List poll() throws InterruptedException { 174 | List records = new ArrayList<>(); 175 | MqttMessageProcessor message = mQueue.take(); 176 | log.debug("[{}] Polling new data from queue for '{}' topic.", 177 | mMqttClientId, mKafkaTopic); 178 | 179 | Collections.addAll(records, message.getRecords(mKafkaTopic)); 180 | 181 | return records; 182 | } 183 | 184 | /** 185 | * This method is called when the connection to the server is lost. 186 | * 187 | * @param cause the reason behind the loss of connection. 188 | */ 189 | @Override 190 | public void connectionLost(Throwable cause) { 191 | log.error("MQTT connection lost!", cause); 192 | } 193 | 194 | /** 195 | * Called when delivery for a message has been completed, and all acknowledgments have been 196 | * received. 197 | * 198 | * @param token the delivery token associated with the message. 199 | */ 200 | @Override 201 | public void deliveryComplete(IMqttDeliveryToken token) { 202 | // Nothing to implement. 203 | } 204 | 205 | /** 206 | * This method is called when a message arrives from the server. 207 | * 208 | * @param topic name of the topic on the message was published to 209 | * @param message the actual message. 210 | * 211 | * @throws Exception if a terminal error has occurred, and the client should be 212 | * shut down. 213 | */ 214 | @Override 215 | public void messageArrived(String topic, MqttMessage message) throws Exception { 216 | log.debug("[{}] New message on '{}' arrived.", mMqttClientId, topic); 217 | 218 | this.mQueue.add( 219 | mConfig.getConfiguredInstance(MqttSourceConstant.MESSAGE_PROCESSOR, 220 | MqttMessageProcessor.class) 221 | .process(topic, message) 222 | ); 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /src/main/java/com/evokly/kafka/connect/mqtt/sample/DumbProcessor.java: -------------------------------------------------------------------------------- 1 | package com.evokly.kafka.connect.mqtt.sample; 2 | 3 | import com.evokly.kafka.connect.mqtt.MqttMessageProcessor; 4 | import org.apache.kafka.connect.data.Schema; 5 | import org.apache.kafka.connect.source.SourceRecord; 6 | import org.eclipse.paho.client.mqttv3.MqttMessage; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | 11 | /** 12 | * Copyright 2016 Evokly S.A. 13 | * 14 | *

See LICENSE file for License 15 | **/ 16 | public class DumbProcessor implements MqttMessageProcessor { 17 | private static final Logger log = LoggerFactory.getLogger(DumbProcessor.class); 18 | private MqttMessage mMessage; 19 | private Object mTopic; 20 | 21 | @Override 22 | public MqttMessageProcessor process(String topic, MqttMessage message) { 23 | log.debug("processing data for topic: {}; with message {}", topic, message); 24 | this.mTopic = topic; 25 | this.mMessage = message; 26 | return this; 27 | } 28 | 29 | @Override 30 | public SourceRecord[] getRecords(String kafkaTopic) { 31 | return new SourceRecord[]{new SourceRecord(null, null, kafkaTopic, null, 32 | Schema.STRING_SCHEMA, mTopic, 33 | Schema.BYTES_SCHEMA, mMessage.getPayload())}; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/evokly/kafka/connect/mqtt/ssl/SslUtils.java: -------------------------------------------------------------------------------- 1 | package com.evokly.kafka.connect.mqtt.ssl; 2 | 3 | import org.bouncycastle.cert.X509CertificateHolder; 4 | import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; 5 | import org.bouncycastle.jce.provider.BouncyCastleProvider; 6 | import org.bouncycastle.openssl.PEMDecryptorProvider; 7 | import org.bouncycastle.openssl.PEMEncryptedKeyPair; 8 | import org.bouncycastle.openssl.PEMKeyPair; 9 | import org.bouncycastle.openssl.PEMParser; 10 | import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; 11 | import org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder; 12 | 13 | import java.io.FileInputStream; 14 | import java.io.FileReader; 15 | import java.io.IOException; 16 | import java.io.InputStreamReader; 17 | import java.nio.charset.Charset; 18 | import java.security.KeyManagementException; 19 | import java.security.KeyPair; 20 | import java.security.KeyStore; 21 | import java.security.KeyStoreException; 22 | import java.security.NoSuchAlgorithmException; 23 | import java.security.Security; 24 | import java.security.UnrecoverableKeyException; 25 | import java.security.cert.CertificateException; 26 | import javax.net.ssl.KeyManagerFactory; 27 | import javax.net.ssl.SSLContext; 28 | import javax.net.ssl.SSLSocketFactory; 29 | import javax.net.ssl.TrustManagerFactory; 30 | 31 | /** 32 | * Created by booncol on 07.04.2016. 33 | * 34 | */ 35 | public class SslUtils { 36 | 37 | static { 38 | Security.insertProviderAt(new BouncyCastleProvider(), 1); 39 | } 40 | 41 | /** 42 | * Create SSLSocketFactory. 43 | * 44 | * @param caCrt CA certificate filepath 45 | * @param crt Client certificate filepath 46 | * @param key Client key filepath 47 | * @param password Password 48 | * 49 | * @return SSLSocketFactory 50 | */ 51 | public static SSLSocketFactory getSslSocketFactory(String caCrt, 52 | String crt, 53 | String key, 54 | String password) 55 | throws IOException, KeyStoreException, NoSuchAlgorithmException, CertificateException, 56 | UnrecoverableKeyException, KeyManagementException { 57 | 58 | char[] passwdChars = password != null && password.length() > 0 59 | ? password.toCharArray() : "".toCharArray(); 60 | 61 | // load client private key 62 | PEMParser parser = new PEMParser( 63 | new InputStreamReader(new FileInputStream(key), Charset.forName("UTF-8")) 64 | ); 65 | 66 | Object obj = parser.readObject(); 67 | KeyPair keyPair; 68 | JcaPEMKeyConverter converter = new JcaPEMKeyConverter().setProvider("BC"); 69 | 70 | if (obj instanceof PEMEncryptedKeyPair) { 71 | PEMDecryptorProvider decProv = new JcePEMDecryptorProviderBuilder().build(passwdChars); 72 | converter = new JcaPEMKeyConverter().setProvider("BC"); 73 | keyPair = converter.getKeyPair(((PEMEncryptedKeyPair) obj).decryptKeyPair(decProv)); 74 | } else { 75 | keyPair = converter.getKeyPair((PEMKeyPair) obj); 76 | } 77 | 78 | parser.close(); 79 | JcaX509CertificateConverter certConverter = new JcaX509CertificateConverter(); 80 | certConverter.setProvider("BC"); 81 | 82 | // load CA certificate 83 | parser = new PEMParser( 84 | new InputStreamReader(new FileInputStream(caCrt), Charset.forName("UTF-8")) 85 | ); 86 | X509CertificateHolder caCert = (X509CertificateHolder) parser.readObject(); 87 | parser.close(); 88 | 89 | // CA certificate is used to authenticate server 90 | KeyStore caKs = KeyStore.getInstance(KeyStore.getDefaultType()); 91 | caKs.load(null, null); 92 | caKs.setCertificateEntry("ca-certificate", certConverter.getCertificate(caCert)); 93 | 94 | TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory 95 | .getDefaultAlgorithm()); 96 | 97 | tmf.init(caKs); 98 | 99 | // load client certificate 100 | parser = new PEMParser( 101 | new InputStreamReader(new FileInputStream(crt), Charset.forName("UTF-8")) 102 | ); 103 | 104 | X509CertificateHolder cert = (X509CertificateHolder) parser.readObject(); 105 | parser.close(); 106 | 107 | // Client key and certificates are sent to server so it can authenticate 108 | // us 109 | KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); 110 | ks.load(null, null); 111 | ks.setCertificateEntry("certificate", certConverter.getCertificate(cert)); 112 | ks.setKeyEntry("private-key", keyPair.getPrivate(), passwdChars, 113 | new java.security.cert.Certificate[] { certConverter.getCertificate(cert) }); 114 | KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory 115 | .getDefaultAlgorithm()); 116 | kmf.init(ks, passwdChars); 117 | 118 | // Finally, create SSL socket factory 119 | SSLContext context = SSLContext.getInstance("TLSv1.2"); 120 | context.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null); 121 | 122 | return context.getSocketFactory(); 123 | } 124 | 125 | } -------------------------------------------------------------------------------- /src/main/java/com/evokly/kafka/connect/mqtt/util/Utils.java: -------------------------------------------------------------------------------- 1 | // CHECKSTYLE:OFF 2 | package com.evokly.kafka.connect.mqtt.util; 3 | 4 | /** 5 | * Copyright 2016 Evokly S.A. 6 | * 7 | *

See LICENSE file for License 8 | **/ 9 | public class Utils { 10 | public static T getConfiguredInstance(String key, Class t) { 11 | Class c = null; 12 | try { 13 | c = Class.forName(key); 14 | } catch (ClassNotFoundException e) { 15 | e.printStackTrace(); 16 | } 17 | if (c == null) 18 | return null; 19 | Object o = Utils.newInstance(c); 20 | if (!t.isInstance(o)) 21 | throw new RuntimeException(c.getName() + " is not an instance of " + t.getName()); 22 | return t.cast(o); 23 | } 24 | 25 | /** 26 | * Instantiate the class 27 | */ 28 | public static T newInstance(Class c) { 29 | try { 30 | return c.newInstance(); 31 | } catch (IllegalAccessException e) { 32 | throw new RuntimeException("Could not instantiate class " + c.getName(), e); 33 | } catch (InstantiationException e) { 34 | throw new RuntimeException("Could not instantiate class " + c.getName() + " Does it have a public no-argument constructor?", e); 35 | } catch (NullPointerException e) { 36 | throw new RuntimeException("Requested class was null", e); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/evokly/kafka/connect/mqtt/util/Version.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Evokly S.A. 3 | * 4 | *

See LICENSE file for License

5 | **/ 6 | 7 | package com.evokly.kafka.connect.mqtt.util; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.IOException; 13 | import java.io.InputStream; 14 | import java.util.Properties; 15 | 16 | /** 17 | * Copyright 2016 Evokly S.A. 18 | * 19 | *

See LICENSE file for License 20 | **/ 21 | public class Version { 22 | private static final Logger log = LoggerFactory.getLogger(Version.class); 23 | private static String version = "unknown"; 24 | 25 | static { 26 | InputStream in = null; 27 | try { 28 | Properties props = new Properties(); 29 | in = Version.class.getResourceAsStream( 30 | "/kafka-connect-mqtt-version.properties"); 31 | props.load(in); 32 | version = props.getProperty("version", version).trim(); 33 | in.close(); 34 | } catch (Exception e) { 35 | log.warn("Error while loading version:", e); 36 | } finally { 37 | if (in != null) { 38 | try { 39 | in.close(); 40 | } catch (IOException e) { 41 | log.warn("WTF!", e); 42 | } 43 | } 44 | } 45 | } 46 | 47 | public static String getVersion() { 48 | return version; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/resources/kafka-connect-mqtt-version.properties: -------------------------------------------------------------------------------- 1 | ## 2 | # Copyright 2016 Evokly S.A. 3 | # See LICENSE file for License 4 | ## 5 | 6 | version=${project.version} -------------------------------------------------------------------------------- /src/test/java/com/evokly/kafka/connect/mqtt/MqttSourceConnectorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Evokly S.A. 3 | * See LICENSE file for License 4 | **/ 5 | 6 | package com.evokly.kafka.connect.mqtt; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | 10 | import org.junit.Before; 11 | import org.junit.Test; 12 | 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | public class MqttSourceConnectorTest { 18 | private MqttSourceConnector mConnector; 19 | Map mSourceProperties; 20 | 21 | /** 22 | * Several tests need similar objects created before they can run. 23 | */ 24 | @Before 25 | public void beforeEach() { 26 | mConnector = new MqttSourceConnector(); 27 | 28 | mSourceProperties = new HashMap<>(); 29 | 30 | mSourceProperties.put(MqttSourceConstant.KAFKA_TOPIC, "kafka_topic"); 31 | 32 | mSourceProperties.put(MqttSourceConstant.MQTT_CLEAN_SESSION, "true"); 33 | mSourceProperties.put(MqttSourceConstant.MQTT_CLIENT_ID, "TesetClientId"); 34 | mSourceProperties.put(MqttSourceConstant.MQTT_CONNECTION_TIMEOUT, "15"); 35 | mSourceProperties.put(MqttSourceConstant.MQTT_KEEP_ALIVE_INTERVAL, "30"); 36 | mSourceProperties.put(MqttSourceConstant.MQTT_QUALITY_OF_SERVICE, "2"); 37 | mSourceProperties.put(MqttSourceConstant.MQTT_SERVER_URIS, "tcp://127.0.0.1:1883"); 38 | mSourceProperties.put(MqttSourceConstant.MQTT_TOPIC, "mqtt_topic"); 39 | } 40 | 41 | @Test 42 | public void testTaskClass() { 43 | assertEquals(MqttSourceTask.class, mConnector.taskClass()); 44 | } 45 | 46 | @Test 47 | public void testSourceTasks() { 48 | mConnector.start(mSourceProperties); 49 | List> taskConfigs = mConnector.taskConfigs(1); 50 | 51 | assertEquals(taskConfigs.size(), 1); 52 | 53 | assertEquals(taskConfigs.get(0).get(MqttSourceConstant.KAFKA_TOPIC), "kafka_topic"); 54 | assertEquals(taskConfigs.get(0).get(MqttSourceConstant.MQTT_CLEAN_SESSION), "true"); 55 | assertEquals(taskConfigs.get(0).get(MqttSourceConstant.MQTT_CLIENT_ID), "TesetClientId"); 56 | assertEquals(taskConfigs.get(0).get(MqttSourceConstant.MQTT_CONNECTION_TIMEOUT), "15"); 57 | assertEquals(taskConfigs.get(0).get(MqttSourceConstant.MQTT_KEEP_ALIVE_INTERVAL), "30"); 58 | assertEquals(taskConfigs.get(0).get(MqttSourceConstant.MQTT_QUALITY_OF_SERVICE), "2"); 59 | assertEquals(taskConfigs.get(0).get(MqttSourceConstant.MQTT_SERVER_URIS), 60 | "tcp://127.0.0.1:1883"); 61 | assertEquals(taskConfigs.get(0).get(MqttSourceConstant.MQTT_TOPIC), "mqtt_topic"); 62 | 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/test/java/com/evokly/kafka/connect/mqtt/MqttSourceTaskTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Evokly S.A. 3 | * See LICENSE file for License 4 | **/ 5 | 6 | package com.evokly.kafka.connect.mqtt; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | 10 | import org.apache.kafka.connect.source.SourceRecord; 11 | import org.eclipse.paho.client.mqttv3.MqttMessage; 12 | import org.junit.Before; 13 | import org.junit.Test; 14 | 15 | import java.nio.charset.StandardCharsets; 16 | import java.util.HashMap; 17 | import java.util.List; 18 | import java.util.Map; 19 | 20 | public class MqttSourceTaskTest { 21 | private MqttSourceTask mTask; 22 | private Map mEmptyConfig = new HashMap(); 23 | 24 | /** 25 | * Several tests need similar objects created before they can run. 26 | */ 27 | @Before 28 | public void beforeEach() { 29 | mTask = new MqttSourceTask(); 30 | mTask.start(mEmptyConfig); 31 | } 32 | 33 | @Test 34 | public void testPoll() throws Exception { 35 | // empty queue 36 | assertEquals(mTask.mQueue.size(), 0); 37 | 38 | // add dummy message to queue 39 | MqttMessage mqttMessage = new MqttMessage(); 40 | mqttMessage.setPayload("test_message".getBytes(StandardCharsets.UTF_8)); 41 | mTask.messageArrived("test_topic", mqttMessage); 42 | 43 | // check message to be process 44 | assertEquals(mTask.mQueue.size(), 1); 45 | 46 | // generate and validate SourceRecord 47 | List sourceRecords = mTask.poll(); 48 | 49 | assertEquals(sourceRecords.size(), 1); 50 | assertEquals(sourceRecords.get(0).key(), "test_topic"); 51 | assertEquals(new String((byte[]) sourceRecords.get(0).value(), "UTF-8"), "test_message"); 52 | 53 | // empty queue 54 | assertEquals(mTask.mQueue.size(), 0); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/test/java/com/evokly/kafka/connect/mqtt/util/VersionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Evokly S.A. 3 | * See LICENSE file for License 4 | **/ 5 | 6 | package com.evokly.kafka.connect.mqtt.util; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | 10 | import org.junit.Test; 11 | 12 | public class VersionTest { 13 | @Test 14 | public void testGetVersion() { 15 | assertEquals("test_project_version", Version.getVersion()); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/test/resources/kafka-connect-mqtt-version.properties: -------------------------------------------------------------------------------- 1 | ## 2 | # Copyright 2016 Evokly S.A. 3 | # See LICENSE file for License 4 | ## 5 | 6 | version=test_project_version --------------------------------------------------------------------------------