├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── build.sbt ├── catalog-info.yaml ├── get_sbt.sh ├── project ├── build.properties └── plugins.sbt ├── sbt.bat ├── scripts └── startup │ ├── application.conf │ ├── start_mqtt_mongo_with_cmd_args.sh │ └── start_mqtt_mongo_with_conf_file.sh └── src ├── main ├── resources │ ├── logback.xml │ └── reference.conf └── scala │ └── com │ └── izmailoff │ └── mm │ ├── config │ ├── GlobalAppConfig.scala │ └── SerializationFormat.scala │ ├── mongo │ └── MongoDbProvider.scala │ ├── mqtt │ ├── MqttConsumer.scala │ └── MqttIntermediary.scala │ ├── service │ ├── Boot.scala │ └── MqttMongoService.scala │ └── util │ ├── DbSerialization.scala │ ├── HoconMap.scala │ └── StringUtils.scala └── test_disabled ├── resources └── logback-test.xml └── scala └── com └── izmailoff └── mm ├── config └── GlobalAppConfigSpec.scala ├── service ├── ServiceSpec.scala ├── TestHelpers.scala ├── TestMongoDbProviderImpl.scala ├── TestMqttIntermediaryActor.scala └── TestMqttMongoServiceImpl.scala └── util ├── HoconMapSpec.scala └── StringUtilsSpec.scala /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | 4 | # sbt specific 5 | .cache/ 6 | .history/ 7 | .lib/ 8 | dist/* 9 | target/ 10 | lib_managed/ 11 | src_managed/ 12 | project/boot/ 13 | project/plugins/project/ 14 | # ignore sbt launcher since it can be updated with get_sbt.sh 15 | sbt 16 | 17 | # Scala-IDE specific 18 | .scala_dependencies 19 | .worksheet 20 | 21 | # IntelliJ 22 | .idea/ 23 | 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | scala: 3 | - 2.13.8 4 | jdk: 5 | - oraclejdk8 6 | - oraclejdk11 7 | 8 | # FIXME: tests are disabled due to Fongo incompatibility with Scala Mongo driver. 9 | script: "sbt assembly" 10 | #script: "sbt clean coverage test" 11 | #after_success: "sbt coveralls" 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Aleksey Izmailov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Build Status 2 | 3 | [![Join the chat at https://gitter.im/izmailoff/mqtt-mongo](https://badges.gitter.im/izmailoff/mqtt-mongo.svg)](https://gitter.im/izmailoff/mqtt-mongo?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 4 | [![Build Status](https://travis-ci.org/izmailoff/mqtt-mongo.png?branch=master)](https://travis-ci.org/izmailoff/mqtt-mongo) 5 | 6 | # Test Coverage 7 | [![Coverage Status](https://coveralls.io/repos/izmailoff/mqtt-mongo/badge.svg?branch=master)](https://coveralls.io/r/izmailoff/mqtt-mongo?branch=master) 8 | 9 | Tests have been disabled due to incompatibility of Fongo with latest Scala Mongo driver. This needs to be resolved one 10 | way or the other. For now build without unit tests. 11 | 12 | # MQTT-Mongo 13 | This is a generic service that subscribes to MQTT broker and saves messages to MongoDB. 14 | 15 | Say you have an MQTT broker that receives messages and you want those messages to be saved 16 | in MongoDB. This service should be an ideal fit for such purpose. You can save a message in either 17 | `String`, `JSON` or `Binary` format to one or more collections. 18 | 19 | This service follows [Reactive Application Design](http://www.reactivemanifesto.org), i.e. responsive, 20 | resilient, elastic, message driven. What that means that it should work fast and be able to recover from errors. 21 | 22 | # Docker 23 | You can pull a prebuilt image from [Dockerhub](https://hub.docker.com/repository/docker/izmailoff/mqtt-mongo). 24 | 25 | # Quick Start 26 | Install MQTT broker and MongoDB. Default settings for Mongo and Mosquitto will work without any additional configuration. 27 | 28 | Ubuntu example: 29 | 30 | sudo apt install mosquitto mongodb-org-server 31 | 32 | Start services, then pull and run docker. 33 | 34 | docker pull izmailoff/mqtt-mongo 35 | docker run --network=host izmailoff/mqtt-mongo 36 | 37 | Send some test messages: 38 | 39 | mosquitto_pub -t "test" -m "Should work" 40 | mosquitto_pub -t "test" -m '{ "someField" : "Some value", "int" : 123 }' 41 | 42 | They should be saved in Mongo, which you can find with: 43 | 44 | echo 'db.messages.find({});' | mongo --quiet mqtt 45 | 46 | Default Topic: `test` 47 | 48 | Default Mongo DB: `mqtt` 49 | 50 | Default Mongo Collection: `messages` 51 | 52 | Default Message Storage Format: `JSON` 53 | 54 | # Installation 55 | This section describes steps required to build, configure and run the application. 56 | TODO: add download options here 57 | 58 | ## Fastest way to get started 59 | The instructions below apply to any OS, but command examples below are given for Redhat 60 | based Linux distros like Fedora, CentOS, etc. 61 | 62 | Install [mosquitto](http://mosquitto.org/download/) MQTT broker and client binaries: 63 | 64 | sudo yum install mosquitto 65 | 66 | Install [MongoDB](http://docs.mongodb.org/manual/tutorial/install-mongodb-on-red-hat-centos-or-fedora-linux/). 67 | Essentially run: 68 | 69 | sudo yum install mongodb-org 70 | 71 | Keep default configuration (port, interface) for Mosquitto and MongoDB. 72 | Start mosquitto and mongod services if not started already: 73 | 74 | sudo systemctl start mongod 75 | sudo systemctl start mosquitto 76 | 77 | Download or build a MQTT-Mongo JAR (see below). 78 | 79 | Run MQTT-Mongo: 80 | 81 | java -jar mqtt-mongo-assembly-0.1.jar 82 | 83 | Or from project root dir: 84 | 85 | java -jar target/scala-2.13/mqtt-mongo-assembly-0.1.jar 86 | 87 | By now everything should be working and you can start testing the service. 88 | 89 | Publish a message to the 'test' topic: 90 | 91 | mosquitto_pub -t "test" -m "Should work" 92 | mosquitto_pub -t "test" -m '{ "someField" : "Some value", "int" : 123 }' 93 | 94 | Check mongo `mqtt` database, collection `messages` for 2 messages published above: 95 | 96 | echo 'db.messages.find({});' | mongo --quiet mqtt 97 | 98 | You should be able to find the messages. 99 | 100 | ## Building MQTT-Mongo 101 | 102 | ### Install 103 | Install these applications on your dev machine in order to be able to build the src code: 104 | 105 | * Java Development Kit (JDK) >= 1.7 106 | * Optionally install SBT, or use one provided with the project (see sbt.* shell scripts) 107 | 108 | ### Run SBT to generate JAR 109 | SBT is a build tool that downloads source code dependencies, compiles code, runs tests, 110 | generates scaladocs, and produces executables. 111 | 112 | Start up SBT from Unix/Windows shell: 113 | 114 | > sbt 115 | 116 | or if it's not on a PATH: 117 | 118 | > ./sbt 119 | 120 | In SBT shell type (note semicolons): 121 | 122 | ;clean; assembly 123 | 124 | You can also run it as a single command from OS shell: 125 | 126 | > sbt clean assembly 127 | 128 | This will run all tests and generate a single jar file named similar to: `mqtt-mongo-assembly-0.1.jar`. 129 | 130 | To generate docker image: 131 | 132 | > sbt clean assembly docker 133 | 134 | Docker will be tagged with a name like this: `izmailoff/mqtt-mongo:0.2`. 135 | 136 | Here is a full list of commands in case you want to generate projects and documentation, etc: 137 | 138 | > sbt clean compile test doc assembly docker 139 | 140 | Look at the output to find where build artefacts go (docs, jars, etc). 141 | 142 | ## System Requirements 143 | To run compiled JAR file you should have installed: 144 | 145 | * Java Runtime Environment (JRE) >= 1.8 146 | * MongoDB 147 | * mosquitto or any other MQTT broker 148 | 149 | ## Running MQTT-Mongo 150 | Run from your OS shell: 151 | 152 | > java -jar mqtt-mongo-assembly-0.1.jar 153 | 154 | This will run the application. In particular it will register with MQTT broker 155 | on the topics you've configured and will save all received messages to MongoDB. 156 | 157 | Please use run scripts in production environment. They take care of runtime settings 158 | and environment, so that you don't get it wrong. Example scripts are in `scripts` directory of a 159 | project's root directory. 160 | 161 | ### Advanced Configuration 162 | The service uses [HOCON](https://github.com/typesafehub/config) configuration library. See it's documentation for 163 | detailed information on how you can perform advanced configuration if necessary. Otherwise see brief description 164 | below. 165 | The packaged jar contains `reference.conf` file that has default settings for all configuration parameters. Thus 166 | you can run the service as is if you just want to try it out. For example: 167 | 168 | java -jar mqtt-mongo-assembly-0.1.jar 169 | 170 | In this case it will try to connect to localhost on port 27017 for MongoDB and port 1883 for MQTT broker (mosquitto). 171 | It will subscribe to `test` topic and save all messages to database `mqtt`, collection `messages`. 172 | 173 | In most cases you would want to override topics, location of MQTT broker and MongoDB in your config. Preferred way to 174 | configure the application is via additional configuration file (`application.conf`). Settings in `application.conf` 175 | will override default settings in `reference.conf`. Additionally you can provide settings as command line arguments 176 | to JVM with a `-D` flag. Here are a few examples: 177 | 178 | 1) Create a config file and provide it to the application. Contents of `application.conf`: 179 | 180 | application.mqttMongo.topicsToCollectionsMappings = [ 181 | { "one" : "collectionOne" }, { "two" : "collectionTwo" } 182 | ] 183 | 184 | Run the application with this config: 185 | 186 | java -Dconfig.file=application.conf -jar mqtt-mongo-assembly-0.1.jar 187 | 188 | 2) Provide settings via cmd args: 189 | 190 | java -Dapplication.mongo.host=127.0.0.1 -Dapplication.mongo.dbName=adhocdb -jar mqtt-mongo-assembly-0.1.jar 191 | 192 | This application is based on Akka concurrency framework. If you want to fine tune or debug the application 193 | take a look at Akka [docs](http://doc.akka.io/docs/akka/snapshot/general/configuration.html). Akka can be configured 194 | the same way via conf file or cmd args as described above. 195 | 196 | TODO: configuration examples from environment variables and in Docker. 197 | 198 | # Questions and Answers 199 | Q: Can I create a capped collection before I start using mqtt-mongo so that old messages get deleted automatically? 200 | 201 | A: Absolutely, mqtt-mongo performs inserts only. A new collection is created upon first insert if it does not exist - 202 | standard MongoDB behaviour. 203 | 204 | 205 | 206 | Q: What happens if MongoDB can't keep up with insert rate? 207 | 208 | A: ... 209 | 210 | 211 | 212 | Q: What happens if connection to MongoDB gets lost? 213 | 214 | A: ... 215 | 216 | 217 | Q: What happens if connection to MQTT broker gets lost? 218 | 219 | A: ... 220 | 221 | 222 | 223 | Q: What happens if mqtt-broker encounters any other failures/exceptions? 224 | 225 | A: ... 226 | 227 | 228 | 229 | Q: Can I make it insert into multiple databases? 230 | 231 | A: mqtt-mongo supports inserts into multiple collections. I thought support of 232 | multiple databases would complicate configuration and implementation. 233 | You can simply start another instance of mqtt-mongo with the same config 234 | except for the database connection settings. 235 | 236 | 237 | 238 | Q: What are other known alternatives? 239 | 240 | A: 241 | 242 | 243 | 244 | Q: Do you have any performance test data? 245 | 246 | A: Coming soon ... 247 | 248 | 249 | 250 | Q: How do I scale this service? 251 | 252 | A: There are multiple options. 253 | 254 | * Single instance: 255 | First of all check the config - you can customize how many threads to use and memory settings. 256 | Check that JVM is started with appropriate cmd args - enough heap memory, etc. 257 | 258 | * Multiple independent instances: 259 | Each mqtt-mongo instance can run on a single server, subscribe to a single topic and save messages to a single collection. 260 | This would be quite an unusual setup in case you have enourmous amount of messages being injested. 261 | 262 | * Mongo scaling: 263 | Use sharded collections to distribute writes. Use fire and forget write concern level. 264 | 265 | --If you exhaust all these possibilities or you have a single topic only and can't scale in other ways 266 | --this service can be implemented as a cluster that runs on multiple machines. 267 | 268 | 269 | 270 | Q: What if I want to apply some processing steps before the message is saved to Mongo? 271 | 272 | A: Currently preprocessing/filtering of messages is non-existent or customizable. You can 273 | fork this project and implement that logic yourself in Scala or Java. If you are interested 274 | in plugin support for this purpose let me know and I can easily add it. 275 | 276 | # Contributors and Feedback 277 | Feel free to open a pull request if you like to fix or improve something as long as it doesn't add features out of 278 | scope of this project. 279 | 280 | If you want to share feedback or have questions please try Gitter. 281 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | 2 | name := "mqtt-mongo" 3 | 4 | version := "1.1.0" 5 | 6 | scalaVersion := "3.1.3" 7 | 8 | organization := "izmailoff" 9 | 10 | scalacOptions ++= Seq( 11 | "-encoding", "utf8", 12 | "-deprecation", 13 | "-feature", 14 | "-language:postfixOps", 15 | "-language:implicitConversions", 16 | // "-Xtarget:8", 17 | "-unchecked", 18 | "-indent", 19 | ) 20 | 21 | 22 | enablePlugins(DockerPlugin) 23 | 24 | //addDependencyTreePlugin 25 | 26 | libraryDependencies ++= { 27 | val akkaVersion = "2.6.19" 28 | Seq( 29 | // "com.typesafe.akka" %% "akka-actor" % akkaVersion, 30 | "org.mongodb.scala" %% "mongo-scala-driver" % "4.6.1" cross CrossVersion.for3Use2_13, 31 | "com.sandinh" %% "paho-akka" % "1.6.1", 32 | // "com.typesafe.akka" %% "akka-slf4j" % akkaVersion, 33 | "ch.qos.logback" % "logback-classic" % "1.2.11", 34 | // "org.specs2" %% "specs2-core" % "4.16.1" % "test", // 5.0.2 35 | // "com.github.fakemongo" % "fongo" % "2.1.1" % "test", 36 | // "com.typesafe.akka" %% "akka-testkit" % akkaVersion % "test", 37 | // "org.scalatest" %% "scalatest" % "3.2.12" % "test" 38 | ) 39 | } 40 | 41 | 42 | docker / imageNames := Seq(ImageName(s"${organization.value}/${name.value}:${version.value}")) 43 | 44 | docker / dockerfile := { 45 | // The assembly task generates a fat JAR file 46 | val artifact: File = assembly.value 47 | val artifactTargetPath = s"/app/${artifact.name}" 48 | 49 | new Dockerfile { 50 | from("openjdk:8-jre") 51 | add(artifact, artifactTargetPath) 52 | entryPoint("java", "-jar", artifactTargetPath) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /catalog-info.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: backstage.io/v1alpha1 2 | kind: Component 3 | metadata: 4 | name: mqtt-mongo 5 | annotations: 6 | github.com/project-slug: izmailoff/mqtt-mongo 7 | spec: 8 | type: other 9 | lifecycle: unknown 10 | owner: guests 11 | -------------------------------------------------------------------------------- /get_sbt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Downloads the latest version of SBT runner script which in turn downloads 4 | # SBT launcher JAR and provides lots of convenience methods. 5 | 6 | curl -s https://raw.githubusercontent.com/paulp/sbt-extras/master/sbt > sbt && chmod 0755 sbt 7 | 8 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.6.2 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") // 2.0.0-RC1 available 2 | 3 | //addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.0") 4 | 5 | //addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.3.2") 6 | 7 | addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.1") 8 | 9 | addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.8.3") 10 | 11 | addSbtPlugin("org.scala-sbt" % "sbt-dependency-tree" % "1.7.0-RC2") 12 | -------------------------------------------------------------------------------- /sbt.bat: -------------------------------------------------------------------------------- 1 | rem Download this JAR to this dir: https://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.7/sbt-launch.jar?_ga=1.47612455.1882964693.1406200696 2 | 3 | set SCRIPT_DIR=%~dp0 4 | java -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=256m -Xmx1024M -Xss2M -jar "%SCRIPT_DIR%\sbt-launch.jar" %* 5 | -------------------------------------------------------------------------------- /scripts/startup/application.conf: -------------------------------------------------------------------------------- 1 | 2 | application.mqttMongo.topicsToCollectionsMappings = [ 3 | { "one" : "collectionOne" }, { "two" : "collectionTwo" } 4 | ] 5 | 6 | -------------------------------------------------------------------------------- /scripts/startup/start_mqtt_mongo_with_cmd_args.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run the service overriding settings via JDK cmd args. 4 | java -Dapplication.mongo.host=127.0.0.1 -Dapplication.mongo.dbName=adhocdb -jar /home/alex/repos/mqtt-mongo/target/scala-2.11/mqtt-mongo-assembly-0.1.jar 5 | -------------------------------------------------------------------------------- /scripts/startup/start_mqtt_mongo_with_conf_file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run the service with alternative config file 4 | java -Dconfig.file=application.conf -jar ../../target/scala-2.11/mqtt-mongo-assembly-0.1.jar 5 | 6 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | 2 | # All application settings except for default dependent library settings like Akka. 3 | application { 4 | 5 | # MQTT broker connection settings. 6 | mqttBroker { 7 | 8 | protocol = "tcp" 9 | 10 | hostname = "localhost" 11 | 12 | port = 1883 13 | 14 | url = ${application.mqttBroker.protocol}"://"${application.mqttBroker.hostname}":"${application.mqttBroker.port} 15 | 16 | userName = "" 17 | 18 | password = "" 19 | 20 | stashTimeToLive = 1 minute 21 | stashCapacity = 8000 22 | reconnectDelayMin = 10 ms 23 | reconnectDelayMax = 10 seconds 24 | } 25 | 26 | # MongoDB connectivity settings 27 | mongo { 28 | # Host on which you can find Mongo server, if sharding/replication is used multiple hosts can be defined here later. 29 | host = "localhost" 30 | 31 | # Mongo port to connect (default 27017) 32 | port = 27017 33 | 34 | # Database name to use. No schema configuration is required. 35 | dbName = "mqtt" 36 | 37 | # mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]] 38 | uri = "mongodb://"${application.mongo.host}":"${application.mongo.port} 39 | } 40 | 41 | # Defines how to save MQTT messages to MongoDB 42 | mqttMongo { 43 | 44 | # Each topic can be saved to one or more collections. The service will subscribe to all listed topics and 45 | # save them to corresponding collections. 46 | # Example: 47 | # [ { "topic1" : "collection_name1" }, { "topic2" : "collection_name2;collection_name3" } ] 48 | # use ';' to separate multiple collections for one topic 49 | topicsToCollectionsMappings = [{ 50 | "test": "messages" 51 | }] 52 | 53 | # A conversion format to be used when saving MQTT messages to Mongo DB. 54 | # Default format is "JSON". 55 | # Available options: 56 | # * JSON - try to convert message payload to JSON and save it as it is. If conversion 57 | # fails save it in "payload" field as String. 58 | # * BINARY - save message payload in "payload" field as it is (bytes). 59 | # * STRING - save message payload as String in "payload". 60 | serializationFormat = "JSON" 61 | 62 | # All documents will be saved under this root field, i.e.: {"data": "some data"} 63 | # You can change it to something shorter. 64 | payloadField = "data" 65 | 66 | } 67 | 68 | } 69 | 70 | # AKKA configuration: 71 | akka { 72 | # for debugging Akka config 73 | log-config-on-start = off 74 | 75 | loggers = ["akka.event.slf4j.Slf4jLogger"] 76 | logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" 77 | 78 | # Options: ERROR, WARNING, INFO, DEBUG 79 | loglevel = INFO 80 | 81 | actor { 82 | debug { 83 | receive = on 84 | autoreceive = on 85 | lifecycle = on 86 | } 87 | } 88 | 89 | log-dead-letters = off 90 | log-dead-letters-during-shutdown = off 91 | } 92 | -------------------------------------------------------------------------------- /src/main/scala/com/izmailoff/mm/config/GlobalAppConfig.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.config 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import com.izmailoff.mm.util.HoconMap 6 | import com.typesafe.config.ConfigFactory 7 | 8 | import scala.concurrent.duration.{FiniteDuration, _} 9 | import com.typesafe.config.Config 10 | 11 | object GlobalAppConfig: 12 | 13 | val config: Config = ConfigFactory.load() 14 | 15 | object Application: 16 | 17 | object MqttBroker: 18 | private lazy val brokerConf = config.getConfig("application.mqttBroker") 19 | lazy val url: String = brokerConf.getString("url") 20 | lazy val userName: String = brokerConf.getString("userName") 21 | lazy val password: String = brokerConf.getString("password") 22 | lazy val stashTimeToLive: FiniteDuration = 23 | brokerConf.getDuration("stashTimeToLive", TimeUnit.SECONDS).seconds 24 | lazy val stashCapacity: Int = brokerConf.getInt("stashCapacity") 25 | lazy val reconnectDelayMin: FiniteDuration = 26 | brokerConf.getDuration("reconnectDelayMin", TimeUnit.SECONDS).seconds 27 | lazy val reconnectDelayMax: FiniteDuration = 28 | brokerConf.getDuration("reconnectDelayMax", TimeUnit.SECONDS).seconds 29 | 30 | object Mongo: 31 | private lazy val mongoConf = config.getConfig("application.mongo") 32 | lazy val host: String = mongoConf.getString("host") 33 | lazy val port: Int = mongoConf.getInt("port") 34 | lazy val dbName: String = mongoConf.getString("dbName") 35 | lazy val uri: String = mongoConf.getString("uri") 36 | 37 | object MqttMongo: 38 | private lazy val mqttMongoConf = config.getConfig("application.mqttMongo") 39 | lazy val topicsToCollectionsMappings: Map[String, Set[String]] = 40 | HoconMap.getMap(identity(_), getElems, 41 | mqttMongoConf, "topicsToCollectionsMappings").withDefaultValue(Set.empty) 42 | val getElems: String => Set[String] = 43 | _.split(";").toList.map(_.trim).filter(!_.isEmpty).toSet 44 | lazy val serializationFormat: SerializationFormat.Value = SerializationFormat.withName(mqttMongoConf.getString("serializationFormat")) 45 | lazy val payloadField: String = mqttMongoConf.getString("payloadField") 46 | 47 | 48 | -------------------------------------------------------------------------------- /src/main/scala/com/izmailoff/mm/config/SerializationFormat.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.config 2 | 3 | object SerializationFormat extends Enumeration: 4 | 5 | type SerializationFormat = Value 6 | 7 | val JSON, BINARY, STRING = Value 8 | 9 | -------------------------------------------------------------------------------- /src/main/scala/com/izmailoff/mm/mongo/MongoDbProvider.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.mongo 2 | 3 | import com.izmailoff.mm.config.GlobalAppConfig.Application.Mongo 4 | import org.mongodb.scala.{MongoClient, MongoDatabase} 5 | 6 | trait MongoDbProvider: 7 | 8 | val db: MongoDatabase 9 | 10 | trait MongoDbProviderImpl 11 | extends MongoDbProvider: 12 | 13 | private val client = MongoClient(Mongo.uri) 14 | val db: MongoDatabase = client.getDatabase(Mongo.dbName) 15 | -------------------------------------------------------------------------------- /src/main/scala/com/izmailoff/mm/mqtt/MqttConsumer.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.mqtt 2 | 3 | import akka.actor._ 4 | import com.izmailoff.mm.config.GlobalAppConfig.Application.MqttMongo 5 | import com.izmailoff.mm.mongo.MongoDbProvider 6 | import com.izmailoff.mm.util.DbSerialization._ 7 | import com.sandinh.paho.akka.{Message, Subscribe, SubscribeAck} 8 | import org.mongodb.scala.{Document, MongoDatabase, Observer} 9 | 10 | class MqttConsumer(pubSubIntermediary: ActorRef, db: MongoDatabase) 11 | extends Actor 12 | with ActorLogging: 13 | 14 | val topicsCollectionsMappings = MqttMongo.topicsToCollectionsMappings 15 | 16 | private def resultHandler(doc: Document) = new Observer[Any] { 17 | override def onNext(result: Any): Unit = log.debug(f"Inserted $doc into db successfully.") 18 | override def onError(e: Throwable): Unit = log.error(e, f"Failed to insert $doc into db.") 19 | override def onComplete(): Unit = log.debug(f"Insert $doc finished.") 20 | } 21 | 22 | subscribe() 23 | 24 | def subscribe(): Unit = 25 | topicsCollectionsMappings foreach { 26 | case (t, c) => pubSubIntermediary ! Subscribe(t, self) 27 | } 28 | 29 | def receive: PartialFunction[Any,Unit] = 30 | case msg: Message => 31 | val (doc, collections) = getPersistenceRecipe(msg) 32 | saveDb(doc, collections) 33 | log.debug(s"topic: [${msg.topic}], payload: [$doc] is saved to collections: [$collections].") 34 | case SubscribeAck(Subscribe(topic, `self`, _), None) => 35 | log.info(s"Subscription to topic [$topic] acknowledged.") 36 | case SubscribeAck(Subscribe(topic, `self`, _), Some(err)) => 37 | log.error(err, s"Subscription to topic [$topic] failed.") 38 | // TODO: implement error recovery or terminate the service 39 | 40 | def getPersistenceRecipe(msg: Message): (Document, Set[String]) = 41 | import msg._ 42 | val collections = topicsCollectionsMappings(topic) 43 | val doc = serialize(payload) 44 | (doc, collections) 45 | 46 | def saveDb(doc: Document, collections: Set[String]): Unit = 47 | collections foreach { collName => 48 | val res = db.getCollection[Document](collName).insertOne(doc) 49 | res.subscribe(resultHandler(doc)) 50 | } 51 | 52 | trait MqttConsumerComponent 53 | extends MongoDbProvider: 54 | def system: ActorSystem 55 | 56 | def startMqttConsumer(pubSubIntermediary: ActorRef): ActorRef = 57 | system.actorOf(Props(new MqttConsumer(pubSubIntermediary, db))) 58 | -------------------------------------------------------------------------------- /src/main/scala/com/izmailoff/mm/mqtt/MqttIntermediary.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.mqtt 2 | 3 | import akka.actor.{ActorRef, ActorSystem, Props} 4 | import com.izmailoff.mm.config.GlobalAppConfig.Application.MqttBroker 5 | import com.sandinh.paho.akka.{ConnOptions, MqttPubSub, PSConfig} 6 | import com.izmailoff.mm.util.StringUtils._ 7 | 8 | trait MqttIntermediary 9 | extends MqttIntermediaryComponent: 10 | 11 | def system: ActorSystem 12 | 13 | def startMqttIntermediary(): ActorRef = 14 | system.actorOf(Props(classOf[MqttPubSub], PSConfig( 15 | brokerUrl = MqttBroker.url, 16 | conOpt = ConnOptions(username = emptyToNull(MqttBroker.userName), password = emptyToNull(MqttBroker.password)), 17 | stashTimeToLive = MqttBroker.stashTimeToLive, 18 | stashCapacity = MqttBroker.stashCapacity, 19 | reconnectDelayMin = MqttBroker.reconnectDelayMin, 20 | reconnectDelayMax = MqttBroker.reconnectDelayMax 21 | )), name = "MqttIntermediary") 22 | 23 | trait MqttIntermediaryComponent: 24 | 25 | def startMqttIntermediary(): ActorRef 26 | -------------------------------------------------------------------------------- /src/main/scala/com/izmailoff/mm/service/Boot.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.service 2 | 3 | import akka.actor.ActorSystem 4 | import com.izmailoff.mm.mongo.MongoDbProviderImpl 5 | import com.izmailoff.mm.mqtt.MqttIntermediary 6 | import akka.actor.ActorRef 7 | 8 | object Boot 9 | extends App 10 | with MqttMongoService 11 | with MqttIntermediary 12 | with MongoDbProviderImpl: 13 | 14 | val system: ActorSystem = ActorSystem("mqtt-mongo-system") 15 | import system.log 16 | val banner: String = 17 | """ 18 | | __ __ ___ _____ _____ __ __ 19 | || \/ |/ _ \_ _|_ _| | \/ | ___ _ __ __ _ ___ 20 | || |\/| | | | || | | |_____| |\/| |/ _ \| '_ \ / _` |/ _ \ 21 | || | | | |_| || | | |_____| | | | (_) | | | | (_| | (_) | 22 | ||_| |_|\__\_\|_| |_| |_| |_|\___/|_| |_|\__, |\___/ 23 | | |___/ 24 | | 25 | """.stripMargin 26 | log.info(banner) 27 | 28 | val pubSubIntermediary: ActorRef = startMqttIntermediary() 29 | 30 | val messageConsumer: ActorRef = startMqttConsumer(pubSubIntermediary) 31 | 32 | log.info("APPLICATION STARTED!") 33 | -------------------------------------------------------------------------------- /src/main/scala/com/izmailoff/mm/service/MqttMongoService.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.service 2 | 3 | import com.izmailoff.mm.mqtt.{MqttConsumerComponent, MqttIntermediaryComponent} 4 | 5 | trait MqttMongoService 6 | extends MqttIntermediaryComponent 7 | with MqttConsumerComponent { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/com/izmailoff/mm/util/DbSerialization.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.util 2 | 3 | import com.izmailoff.mm.config.GlobalAppConfig.Application.MqttMongo 4 | import com.izmailoff.mm.config.SerializationFormat._ 5 | import org.mongodb.scala.bson._ 6 | 7 | import scala.util.Try 8 | 9 | object DbSerialization: 10 | 11 | val PAYLOAD_FIELD = MqttMongo.payloadField 12 | 13 | def serialize(payload: Array[Byte]): Document = 14 | MqttMongo.serializationFormat match 15 | case JSON => serializeToJson(payload) 16 | case BINARY => serializeToBinary(payload) 17 | case STRING => serializeToString(payload) 18 | 19 | def serializeToJson(payload: Array[Byte]): Document = 20 | parseSafe(new String(payload)) 21 | 22 | def serializeToBinary(payload: Array[Byte]): Document = 23 | Document(PAYLOAD_FIELD -> payload) 24 | 25 | def serializeToString(payload: Array[Byte]): Document = 26 | Document(PAYLOAD_FIELD -> new String(payload)) 27 | 28 | def parseSafe(msg: String): Document = 29 | Try { 30 | Document(msg) 31 | } recover { 32 | case _ => Document(f"""{"$PAYLOAD_FIELD": $msg}""") 33 | } getOrElse { 34 | Document(PAYLOAD_FIELD -> msg) 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/com/izmailoff/mm/util/HoconMap.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.util 2 | 3 | import com.typesafe.config.Config 4 | 5 | import scala.jdk.CollectionConverters.CollectionHasAsScala 6 | 7 | 8 | 9 | /** 10 | * Reads a map of values from HOCON config file (.conf) since such functionality is not provided by default. 11 | */ 12 | object HoconMap: 13 | 14 | /** 15 | * Reads a map of values from conf 16 | * @param keyF a function for converting to Key type 17 | * @param valF a function for converting to Value type 18 | * @param conf a config at any path level, see @path 19 | * @param path a non-empty path aka a key name which contains the map of values 20 | * @tparam Key 21 | * @tparam Value 22 | * @return returns a map that contains all values from the config or throws an exception if configuration 23 | * was not in a correct format. 24 | */ 25 | def getMap[Key, Value](keyF: String => Key, valF: String => Value, conf: Config, path: String): Map[Key, Value] = 26 | val list = conf.getObjectList(path).asScala 27 | (for { 28 | item <- list 29 | entry <- item.entrySet().asScala 30 | key = keyF(entry.getKey) 31 | value = valF(entry.getValue.unwrapped().toString) 32 | } yield (key, value)).toMap 33 | 34 | -------------------------------------------------------------------------------- /src/main/scala/com/izmailoff/mm/util/StringUtils.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.util 2 | 3 | object StringUtils: 4 | 5 | def emptyToNull(str: String): String = 6 | if (str == null || str.trim.isEmpty) null 7 | else str 8 | 9 | -------------------------------------------------------------------------------- /src/test_disabled/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/test_disabled/scala/com/izmailoff/mm/config/GlobalAppConfigSpec.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.config 2 | 3 | import com.izmailoff.mm.config.GlobalAppConfig.Application._ 4 | import org.specs2.matcher.DataTables 5 | import org.specs2.mutable.Specification 6 | import SerializationFormat._ 7 | 8 | class GlobalAppConfigSpec 9 | extends Specification 10 | with DataTables { 11 | 12 | "Global config" should { 13 | "resolve all settings without any exceptions" in { 14 | // TODO: complete this test 15 | 16 | "config value" || "expected value" | 17 | Mongo.dbName !! "mqtt" | 18 | //MqttBroker.reconnectDelay !! FiniteDuration(10, TimeUnit.SECONDS) | 19 | Mongo.host !! "localhost" | 20 | Mongo.port !! 27017 | 21 | MqttBroker.url !! "tcp://localhost:1883" | 22 | MqttBroker.userName !! "" | 23 | MqttBroker.password !! "" | 24 | MqttMongo.serializationFormat !! JSON |> { 25 | (actual, expected) => 26 | actual must be equalTo expected 27 | } 28 | } 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/test_disabled/scala/com/izmailoff/mm/service/ServiceSpec.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.service 2 | 3 | import akka.actor.ActorSystem 4 | import akka.testkit.{DefaultTimeout, ImplicitSender, TestKit, TestProbe} 5 | import com.izmailoff.mm.config.GlobalAppConfig 6 | import com.sandinh.paho.akka.{Message, Subscribe, SubscribeAck} 7 | import org.scalatest.matchers.should.Matchers 8 | import org.scalatest.BeforeAndAfterAll 9 | import org.scalatest.wordspec.AnyWordSpecLike 10 | 11 | import scala.concurrent.duration._ 12 | import scala.jdk.CollectionConverters._ 13 | 14 | 15 | class ServiceSpec 16 | extends TestKit(ActorSystem("test-mqtt-mongo-system", GlobalAppConfig.config)) 17 | with DefaultTimeout 18 | with ImplicitSender 19 | with AnyWordSpecLike 20 | with Matchers 21 | with BeforeAndAfterAll 22 | with TestMqttMongoServiceImpl 23 | with TestHelpers { 24 | 25 | override def afterAll() { 26 | shutdown() 27 | } 28 | 29 | "Subscription between MQTT Broker and Consumer" should { 30 | "get established when consumer is started" in { 31 | val mqttBroker = startMqttIntermediary() 32 | val probe = TestProbe() 33 | val mqttConsumer = startMqttConsumer(probe.ref) 34 | 35 | probe.expectMsg(Subscribe(testTopic, mqttConsumer)) 36 | probe.forward(mqttBroker, Subscribe(testTopic, probe.ref)) 37 | probe.expectMsg(SubscribeAck(Subscribe(testTopic, probe.ref), None)) 38 | probe.forward(mqttConsumer, SubscribeAck(Subscribe(testTopic, mqttConsumer), None)) 39 | probe.expectNoMsg() 40 | } 41 | } 42 | 43 | "Sending a message to MQTT Broker" should { 44 | "forward it to MQTT Consumer and get saved in DB in proper JSON format" in { 45 | val collection = getCollectionName(testTopic).head 46 | db.getCollection(collection).count() should be(0) 47 | val mqttBroker = startMqttIntermediary() 48 | val mqttConsumer = startMqttConsumer(mqttBroker) 49 | expectNoMsg(1.second) 50 | 51 | mqttBroker ! new Message(testTopic, "test content".getBytes) 52 | mqttBroker ! new Message(testTopic, """{ "field1" : "str val", "field2" : 123 }""".getBytes) 53 | expectNoMsg(1.second) 54 | 55 | db.getCollection(collection).count() should be(2) 56 | val allDocsDb = db.getCollection(collection).find().iterator.asScala 57 | allDocsDb.exists { d => 58 | val fields: Map[Any, Any] = d.toMap.asScala.toMap 59 | fields.size == 2 && 60 | fields("payload") == "test content" 61 | } should be(true) 62 | allDocsDb.exists { d => 63 | val fields: Map[Any, Any] = d.toMap.asScala.toMap 64 | fields.size == 3 && 65 | fields("field1") == "str val" && 66 | fields("field2") == 123 67 | } should be(true) 68 | } 69 | } 70 | 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/test_disabled/scala/com/izmailoff/mm/service/TestHelpers.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.service 2 | 3 | import com.izmailoff.mm.config.GlobalAppConfig 4 | 5 | trait TestHelpers { 6 | 7 | val testTopic = "test" 8 | 9 | def getCollectionName(topic: String = testTopic) = 10 | GlobalAppConfig.Application.MqttMongo.topicsToCollectionsMappings(topic) 11 | 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/test_disabled/scala/com/izmailoff/mm/service/TestMongoDbProviderImpl.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.service 2 | 3 | import com.github.fakemongo.Fongo 4 | //import com.izmailoff.mm.config.GlobalAppConfig.Application.Mongo 5 | import com.izmailoff.mm.mongo.MongoDbProvider 6 | 7 | trait TestMongoDbProviderImpl 8 | extends MongoDbProvider { 9 | 10 | private val mongoClient = new Fongo("in-memory-db") 11 | 12 | // com.mongodb.MongoClient 13 | val db = mongoClient.getMongo() 14 | // val db = mongoClient.getDB(Mongo.dbName) //.getDatabase() 15 | } 16 | -------------------------------------------------------------------------------- /src/test_disabled/scala/com/izmailoff/mm/service/TestMqttIntermediaryActor.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.service 2 | 3 | import akka.actor.{Actor, ActorLogging, ActorRef} 4 | import com.sandinh.paho.akka.{Message, Subscribe, SubscribeAck} 5 | 6 | class TestMqttIntermediaryActor extends Actor with ActorLogging { 7 | var topicSubscribers: Map[String, Set[ActorRef]] = Map() 8 | 9 | def receive = { 10 | case subRequest@Subscribe(topic, subscriber, _) => 11 | updateSubscribers(topic, subscriber) 12 | log.debug(s"Current subscribers: $topicSubscribers") 13 | subscriber ! SubscribeAck(subRequest, None) 14 | case msg: Message => 15 | log.debug(s"Forwarding message: [$msg] to subscribers.") 16 | forwardToSubscribers(msg) 17 | } 18 | 19 | def forwardToSubscribers(msg: Message): Unit = 20 | for { 21 | subscribers <- topicSubscribers.get(msg.topic) 22 | subscriber <- subscribers 23 | } { 24 | subscriber ! msg 25 | } 26 | 27 | def updateSubscribers(topic: String, subscriber: ActorRef): Unit = { 28 | val existingSubscribers = topicSubscribers.getOrElse(topic, Set()) 29 | val updatedSubscribers = existingSubscribers + subscriber 30 | topicSubscribers = topicSubscribers + (topic -> updatedSubscribers) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/test_disabled/scala/com/izmailoff/mm/service/TestMqttMongoServiceImpl.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.service 2 | 3 | import akka.actor.{ActorRef, ActorSystem, Props} 4 | 5 | trait TestMqttMongoServiceImpl 6 | extends MqttMongoService 7 | with TestMqttIntermediary 8 | with TestMongoDbProviderImpl {} 9 | 10 | trait TestMqttIntermediary { 11 | def system: ActorSystem 12 | 13 | def startMqttIntermediary(): ActorRef = 14 | system.actorOf(Props[TestMqttIntermediaryActor]) 15 | } -------------------------------------------------------------------------------- /src/test_disabled/scala/com/izmailoff/mm/util/HoconMapSpec.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.util 2 | 3 | import com.typesafe.config.{ConfigException, ConfigFactory} 4 | import org.specs2.mutable.Specification 5 | 6 | class HoconMapSpec extends Specification { 7 | 8 | "HOCON config" should { 9 | val rawConf = 10 | """ 11 | |application { 12 | | someSettings { 13 | | mappings = [ 14 | | { "one" : 2000 }, 15 | | { "two" : 3000 }, 16 | | { "three" : 5000 } 17 | | ] 18 | | } 19 | |} 20 | """.stripMargin 21 | val conf = ConfigFactory.parseString(rawConf) 22 | 23 | "parse map like entries given a root conf and full path and create a proper Scala map from them" in { 24 | val expected = Map("one" -> 2000L, 25 | "two" -> 3000L, 26 | "three" -> 5000L) 27 | HoconMap.getMap[String, Long](identity(_), _.toLong, conf, "application.someSettings.mappings") === expected 28 | } 29 | 30 | "throw config exception for incomplete/wrong path because full path is required" in { 31 | HoconMap.getMap[String, Long](identity(_), _.toLong, conf, "mappings") must throwA[ConfigException] 32 | } 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/test_disabled/scala/com/izmailoff/mm/util/StringUtilsSpec.scala: -------------------------------------------------------------------------------- 1 | package com.izmailoff.mm.util 2 | 3 | import org.specs2.matcher.DataTables 4 | import org.specs2.mutable.Specification 5 | import com.izmailoff.mm.util.StringUtils._ 6 | 7 | class StringUtilsSpec 8 | extends Specification 9 | with DataTables { 10 | 11 | "String utils java compat" should { 12 | "convert empty strings to null" in { 13 | 14 | "input" || "expected value" | 15 | (null: String) !! null | 16 | " " !! null | 17 | " " !! null | 18 | " \t \r\n \n " !! null | 19 | "abc" !! "abc" |> { 20 | (input, expected) => 21 | emptyToNull(input) must be equalTo expected 22 | } 23 | } 24 | } 25 | } 26 | --------------------------------------------------------------------------------