├── README.md ├── connect-distributed.properties ├── consumer ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── demo │ │ └── azure │ │ ├── Consumer.java │ │ └── ConsumerApp.java │ └── resources │ └── application.yaml ├── gen-timeseries-data.sh ├── grafana_dashboards ├── avg-pressure.json ├── device-pressure.json ├── devide-temprature.json └── temperature-aggregated.json ├── images └── architecture_2.png └── mqtt-source-config.json /README.md: -------------------------------------------------------------------------------- 1 | # Time Series data with Redis and Apache Kafka 2 | 3 | A practical example of how to use [RedisTimeSeries](https://redisearch.io/) with Apache Kafka for analyzing time series data. 4 | 5 | The blog post is coming soon. Meanwhile, here is a talk from RedisConf 2021 which covers this topic as well 6 | 7 | [![RedisConf 2021](https://img.youtube.com/vi/8et5wd60YuY/0.jpg)](https://youtu.be/8et5wd60YuY) 8 | 9 | ### High level architecture 10 | 11 | ![](images/architecture_2.png) 12 | 13 | Individual services: 14 | 15 | - Source (local) components 16 | - MQTT broker (mosquitto): MQTT is a de-facto protocol for IoT use cases. The scenario we will be using is a combination of IoT and Time Series - more on this later. 17 | - Kafka Connect: The MQTT source connector is used to data from MQTT broker to a Kafka cluster. 18 | 19 | - Azure services 20 | - [Azure Cache for Redis Enterprise Tiers](https://docs.microsoft.com/azure/azure-cache-for-redis/quickstart-create-redis-enterprise?WT.mc_id=data-17927-abhishgu): The Enterprise tiers are based on Redis Enterprise, a commercial variant of Redis from Redis Labs. In addition to RedisTimeSeries, Enterprise tier also supports RediSearch and RedisBloom. 21 | - [Confluent Cloud on Azure](https://docs.microsoft.com/azure/partner-solutions/apache-kafka-confluent-cloud/overview?WT.mc_id=data-17927-abhishgu): A fully-managed offering that provides Apache Kafka as a service, thanks to an integrated provisioning layer from Azure to Confluent Cloud. 22 | - [Azure Spring Cloud](https://docs.microsoft.com/azure/spring-cloud/?WT.mc_id=data-17927-abhishgu): Deploying Spring Boot microservices to Azure is easier, thanks to Azure Spring Cloud, which does all the heavy lifting so developers can focus on their code. 23 | 24 | ## Scenario 25 | 26 | Imagine there are many locations, each of them has multiple devices and you're tasked with the responsibility to monitor device metrics - we will consider temperature and pressure. These metrics will be stored in `RedisTimeSeries` (of course!) and use the following naming convention for keys - `::`. For e.g. temperature for device `1` in location `5` will be represented as `temp:5:1`. Each time series data point will also have the following Labels (key value pairs) - metric, location, device. This is to allow for flexible querying as you will see in the upcoming sections. 27 | Here are a couple of examples to give you an idea of how you would add data points using the TS.ADD command: 28 | 29 | ```bash 30 | # temperature for device 2 in location 3 along with labels 31 | TS.ADD temp:3:2 * 20 LABELS metric temp location 3 device 2 32 | 33 | # pressure for device 2 in location 3 34 | TS.ADD pressure:3:2 * 60 LABELS metric pressure location 3 device 2 35 | ``` 36 | 37 | ## End-to-end flow: 38 | 39 | - A script produces simulated device data that is sent to the local MQTT broker. 40 | - This data is picked up by the MQTT Kafka Connect source connector and sent to a topic in the Confluent Cloud Kafka cluster running in Azure. 41 | - It is further processed by the Spring Boot application hosted in Azure Spring Cloud which then persists it to the Azure Cache for Redis instance. -------------------------------------------------------------------------------- /connect-distributed.properties: -------------------------------------------------------------------------------- 1 | bootstrap.servers= 2 | key.converter=org.apache.kafka.connect.storage.StringConverter 3 | value.converter=org.apache.kafka.connect.json.JsonConverter 4 | #value.converter=org.apache.kafka.connect.storage.StringConverter 5 | value.converter.schemas.enable=false 6 | 7 | ssl.endpoint.identification.algorithm=https 8 | security.protocol=SASL_SSL 9 | sasl.mechanism=PLAIN 10 | sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="" password=""; 11 | request.timeout.ms=20000 12 | retry.backoff.ms=500 13 | 14 | producer.bootstrap.servers= 15 | producer.ssl.endpoint.identification.algorithm=https 16 | producer.security.protocol=SASL_SSL 17 | producer.sasl.mechanism=PLAIN 18 | producer.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="" password=""; 19 | producer.request.timeout.ms=20000 20 | producer.retry.backoff.ms=500 21 | 22 | consumer.bootstrap.servers= 23 | consumer.ssl.endpoint.identification.algorithm=https 24 | consumer.security.protocol=SASL_SSL 25 | consumer.sasl.mechanism=PLAIN 26 | consumer.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="" password=""; 27 | consumer.request.timeout.ms=20000 28 | consumer.retry.backoff.ms=500 29 | 30 | offset.flush.interval.ms=10000 31 | offset.storage.file.filename=/tmp/connect.offsets 32 | group.id=connect-cluster 33 | offset.storage.topic=connect-offsets 34 | offset.storage.replication.factor=3 35 | offset.storage.partitions=3 36 | config.storage.topic=connect-configs 37 | config.storage.replication.factor=3 38 | status.storage.topic=connect-status 39 | status.storage.replication.factor=3 40 | #license topic - https://docs.confluent.io/home/connect/license.html#centralized-license-in-the-kconnect-long-worker 41 | confluent.topic.bootstrap.servers= 42 | confluent.topic.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="" password=""; 43 | confluent.topic.security.protocol=SASL_SSL 44 | confluent.topic.sasl.mechanism=PLAIN 45 | #confluent.license= 46 | 47 | plugin.path= -------------------------------------------------------------------------------- /consumer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | org.springframework.boot 6 | spring-boot-starter-parent 7 | 2.4.2 8 | 9 | 10 | com.demo.azure 11 | device-data-processor 12 | 0.0.1-SNAPSHOT 13 | device-data-processor 14 | 15 | 11 16 | 17 | 18 | 19 | org.springframework.boot 20 | spring-boot-starter-web 21 | 22 | 23 | org.springframework.kafka 24 | spring-kafka 25 | 26 | 27 | com.redislabs 28 | jredistimeseries 29 | 1.4.0 30 | 31 | 32 | 33 | 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-maven-plugin 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /consumer/src/main/java/com/demo/azure/Consumer.java: -------------------------------------------------------------------------------- 1 | package com.demo.azure; 2 | 3 | import java.util.HashMap; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | import javax.annotation.PostConstruct; 8 | import javax.annotation.PreDestroy; 9 | 10 | import com.fasterxml.jackson.core.JsonProcessingException; 11 | import com.fasterxml.jackson.databind.JsonMappingException; 12 | import com.fasterxml.jackson.databind.ObjectMapper; 13 | import com.redislabs.redistimeseries.Aggregation; 14 | import com.redislabs.redistimeseries.RedisTimeSeries; 15 | 16 | import org.apache.commons.pool2.impl.GenericObjectPoolConfig; 17 | import org.apache.kafka.clients.consumer.ConsumerRecord; 18 | import org.springframework.beans.factory.annotation.Value; 19 | import org.springframework.context.annotation.Bean; 20 | import org.springframework.kafka.annotation.KafkaListener; 21 | import org.springframework.kafka.listener.ErrorHandler; 22 | import org.springframework.kafka.listener.LoggingErrorHandler; 23 | import org.springframework.kafka.support.converter.ConversionException; 24 | import org.springframework.stereotype.Service; 25 | 26 | import redis.clients.jedis.Jedis; 27 | import redis.clients.jedis.JedisPool; 28 | 29 | @Service 30 | public class Consumer { 31 | 32 | private RedisTimeSeries rts; 33 | 34 | @Value("${redis.host}") 35 | private String redisHost; 36 | 37 | @Value("${redis.port}") 38 | private Integer port; 39 | 40 | @Value("${redis.password}") 41 | private String redisPassword; 42 | 43 | @PostConstruct 44 | public void init() { 45 | System.out.println("Connecting to Redis"); 46 | 47 | int timeout = 2000; 48 | boolean ssl = true; 49 | 50 | GenericObjectPoolConfig jedisPoolConfig = new GenericObjectPoolConfig<>(); 51 | JedisPool jedisPool = new JedisPool(jedisPoolConfig, redisHost, port, timeout, redisPassword, ssl); 52 | 53 | this.rts = new RedisTimeSeries(jedisPool); 54 | } 55 | 56 | final static String TEMP_METRIC = "temp"; 57 | final static String PRESSURE_METRIC = "pressure"; 58 | 59 | @KafkaListener(topics = "${topic.name}", groupId = "${spring.kafka.consumer.group-id}") 60 | public void consume(ConsumerRecord record) { 61 | 62 | System.out.println("Record - " + record.value()); 63 | 64 | // sample records format: location,device,temp,pressure 65 | // sample reading: 1,2,40,50 66 | String[] data = record.value().split(","); 67 | String location = data[0]; // 1 68 | String device = data[1]; // 2 69 | String temp = data[2]; 70 | String pressure = data[3]; 71 | 72 | long tstamp = record.timestamp(); // ideally, this should come from source system 73 | 74 | Map labels = new HashMap<>(); 75 | labels.put("location", location); 76 | labels.put("device", device); 77 | 78 | // add temperature reading first 79 | labels.put("metric", TEMP_METRIC); 80 | String timeSeriesKey = TEMP_METRIC + ":" + location + ":" + device; 81 | rts.add(timeSeriesKey, tstamp, Double.parseDouble(temp), labels); 82 | 83 | System.out.println("adding " + TEMP_METRIC + " to time series " + timeSeriesKey + " for device " + device 84 | + " with labels " + labels); 85 | 86 | // add pressure reading 87 | labels.put("metric", PRESSURE_METRIC); 88 | timeSeriesKey = PRESSURE_METRIC + ":" + location + ":" + device; 89 | rts.add(timeSeriesKey, tstamp, Double.parseDouble(pressure), labels); 90 | 91 | System.out.println("adding " + PRESSURE_METRIC + " to time series " + timeSeriesKey + " for device " + device 92 | + " with labels " + labels); 93 | } 94 | 95 | @Bean 96 | ErrorHandler errorHandler() { 97 | return new LoggingErrorHandler(); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /consumer/src/main/java/com/demo/azure/ConsumerApp.java: -------------------------------------------------------------------------------- 1 | package com.demo.azure; 2 | 3 | import org.apache.kafka.clients.admin.NewTopic; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | import org.springframework.context.annotation.Bean; 8 | 9 | @SpringBootApplication 10 | public class ConsumerApp { 11 | 12 | @Value("${topic.name}") 13 | private String topicName; 14 | 15 | @Value("${topic.replication-factor}") 16 | private short replicationFactor; 17 | 18 | @Value("${topic.partitions-num}") 19 | private Integer partitions; 20 | 21 | @Bean 22 | NewTopic topic() { 23 | return new NewTopic(topicName, partitions, replicationFactor); 24 | } 25 | 26 | public static void main(String[] args) { 27 | SpringApplication.run(ConsumerApp.class, args); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /consumer/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | redis: 2 | host: 3 | port: 10000 4 | password: 5 | topic: 6 | name: mqtt.device-stats 7 | partitions-num: 6 8 | replication-factor: 3 9 | server: 10 | port: 9080 11 | spring: 12 | kafka: 13 | bootstrap-servers: 14 | - 15 | properties: 16 | ssl.endpoint.identification.algorithm: https 17 | sasl.mechanism: PLAIN 18 | request.timeout.ms: 20000 19 | retry.backoff.ms: 500 20 | sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="" password=""; 21 | security.protocol: SASL_SSL 22 | 23 | consumer: 24 | group-id: device-data-processor 25 | auto-offset-reset: latest 26 | key-deserializer: org.apache.kafka.common.serialization.StringDeserializer 27 | #value-deserializer: org.springframework.kafka.support.serializer.JsonDeserializer 28 | value-deserializer: org.apache.kafka.common.serialization.StringDeserializer 29 | max-poll-interval-ms: 10000 30 | properties: 31 | spring.json.use.type.headers: false 32 | template: 33 | default-topic: 34 | logging: 35 | level: 36 | root: info 37 | -------------------------------------------------------------------------------- /gen-timeseries-data.sh: -------------------------------------------------------------------------------- 1 | mqtt_host=localhost 2 | mqtt_port=1883 3 | mqtt_topic=device-stats 4 | 5 | echo "message format - ,,," 6 | 7 | while true; do 8 | for loc in $(seq 1 5); do #5 locations 9 | for device in $(seq 1 5); do #5 devices per location 10 | #temp=$(jot -r 1 20 50) 11 | temp=$(jot -r 1) #erratic! 12 | pressure=$(jot -r 1 51 100) 13 | 14 | #msg=`echo temp,$temp,device$device,loc$loc`; 15 | msg=`echo $loc,$device,$temp,$pressure`; 16 | $(mosquitto_pub -h ${mqtt_host} -t ${mqtt_topic} -q 2 -m "${msg}"); 17 | echo $msg 18 | #msg=`echo pressure,$pressure,device$device,loc$loc`; 19 | #$(mosquitto_pub -h ${mqtt_host} -p ${mqtt_port} -t ${mqtt_topic} -q 2 -m "${msg}"); 20 | 21 | #sleep 0.5s 22 | done 23 | done 24 | done -------------------------------------------------------------------------------- /grafana_dashboards/avg-pressure.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "id": 1, 19 | "links": [], 20 | "panels": [ 21 | { 22 | "aliasColors": {}, 23 | "bars": false, 24 | "dashLength": 10, 25 | "dashes": false, 26 | "datasource": "azure redis", 27 | "fieldConfig": { 28 | "defaults": { 29 | "custom": {} 30 | }, 31 | "overrides": [] 32 | }, 33 | "fill": 1, 34 | "fillGradient": 0, 35 | "gridPos": { 36 | "h": 9, 37 | "w": 12, 38 | "x": 0, 39 | "y": 0 40 | }, 41 | "hiddenSeries": false, 42 | "id": 2, 43 | "legend": { 44 | "avg": false, 45 | "current": false, 46 | "max": false, 47 | "min": false, 48 | "show": true, 49 | "total": false, 50 | "values": false 51 | }, 52 | "lines": true, 53 | "linewidth": 1, 54 | "nullPointMode": "null", 55 | "options": { 56 | "alertThreshold": true 57 | }, 58 | "percentage": false, 59 | "pluginVersion": "7.3.6", 60 | "pointradius": 2, 61 | "points": false, 62 | "renderer": "flot", 63 | "seriesOverrides": [], 64 | "spaceLength": 10, 65 | "stack": false, 66 | "steppedLine": false, 67 | "targets": [ 68 | { 69 | "aggregation": "avg", 70 | "bucket": 30000, 71 | "command": "ts.mrange", 72 | "fill": true, 73 | "filter": "location=1 device=5 metric=pressure", 74 | "legend": "", 75 | "query": "", 76 | "refId": "A", 77 | "streaming": false, 78 | "type": "timeSeries", 79 | "value": "" 80 | } 81 | ], 82 | "thresholds": [], 83 | "timeFrom": null, 84 | "timeRegions": [], 85 | "timeShift": null, 86 | "title": "Statistics", 87 | "tooltip": { 88 | "shared": true, 89 | "sort": 0, 90 | "value_type": "individual" 91 | }, 92 | "type": "graph", 93 | "xaxis": { 94 | "buckets": null, 95 | "mode": "time", 96 | "name": null, 97 | "show": true, 98 | "values": [] 99 | }, 100 | "yaxes": [ 101 | { 102 | "format": "short", 103 | "label": null, 104 | "logBase": 1, 105 | "max": null, 106 | "min": null, 107 | "show": true 108 | }, 109 | { 110 | "format": "short", 111 | "label": null, 112 | "logBase": 1, 113 | "max": null, 114 | "min": null, 115 | "show": true 116 | } 117 | ], 118 | "yaxis": { 119 | "align": false, 120 | "alignLevel": null 121 | } 122 | } 123 | ], 124 | "refresh": "5s", 125 | "schemaVersion": 26, 126 | "style": "dark", 127 | "tags": [], 128 | "templating": { 129 | "list": [] 130 | }, 131 | "time": { 132 | "from": "now-30m", 133 | "to": "now" 134 | }, 135 | "timepicker": {}, 136 | "timezone": "", 137 | "title": "Average Pressure", 138 | "uid": "XPL6pdQGk", 139 | "version": 22 140 | } -------------------------------------------------------------------------------- /grafana_dashboards/device-pressure.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "id": 2, 19 | "links": [], 20 | "panels": [ 21 | { 22 | "aliasColors": {}, 23 | "bars": false, 24 | "dashLength": 10, 25 | "dashes": false, 26 | "datasource": "azure redis", 27 | "fieldConfig": { 28 | "defaults": { 29 | "custom": {} 30 | }, 31 | "overrides": [] 32 | }, 33 | "fill": 1, 34 | "fillGradient": 0, 35 | "gridPos": { 36 | "h": 9, 37 | "w": 12, 38 | "x": 0, 39 | "y": 0 40 | }, 41 | "hiddenSeries": false, 42 | "id": 2, 43 | "legend": { 44 | "avg": false, 45 | "current": false, 46 | "max": false, 47 | "min": false, 48 | "show": true, 49 | "total": false, 50 | "values": false 51 | }, 52 | "lines": true, 53 | "linewidth": 1, 54 | "nullPointMode": "null", 55 | "options": { 56 | "alertThreshold": true 57 | }, 58 | "percentage": false, 59 | "pluginVersion": "7.3.6", 60 | "pointradius": 2, 61 | "points": false, 62 | "renderer": "flot", 63 | "seriesOverrides": [], 64 | "spaceLength": 10, 65 | "stack": false, 66 | "steppedLine": false, 67 | "targets": [ 68 | { 69 | "command": "ts.get", 70 | "keyName": "pressure:4:2", 71 | "query": "", 72 | "refId": "A", 73 | "streaming": true, 74 | "streamingInterval": 500, 75 | "type": "timeSeries" 76 | } 77 | ], 78 | "thresholds": [], 79 | "timeFrom": null, 80 | "timeRegions": [], 81 | "timeShift": null, 82 | "title": "Pressure", 83 | "tooltip": { 84 | "shared": true, 85 | "sort": 0, 86 | "value_type": "individual" 87 | }, 88 | "type": "graph", 89 | "xaxis": { 90 | "buckets": null, 91 | "mode": "time", 92 | "name": null, 93 | "show": true, 94 | "values": [] 95 | }, 96 | "yaxes": [ 97 | { 98 | "format": "short", 99 | "label": null, 100 | "logBase": 1, 101 | "max": null, 102 | "min": null, 103 | "show": true 104 | }, 105 | { 106 | "format": "short", 107 | "label": null, 108 | "logBase": 1, 109 | "max": null, 110 | "min": null, 111 | "show": true 112 | } 113 | ], 114 | "yaxis": { 115 | "align": false, 116 | "alignLevel": null 117 | } 118 | } 119 | ], 120 | "refresh": "", 121 | "schemaVersion": 26, 122 | "style": "dark", 123 | "tags": [], 124 | "templating": { 125 | "list": [] 126 | }, 127 | "time": { 128 | "from": "now-6h", 129 | "to": "now" 130 | }, 131 | "timepicker": {}, 132 | "timezone": "", 133 | "title": "Pressure (device/location)", 134 | "uid": "gS1pcKwMk", 135 | "version": 7 136 | } -------------------------------------------------------------------------------- /grafana_dashboards/devide-temprature.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "id": 3, 19 | "links": [], 20 | "panels": [ 21 | { 22 | "aliasColors": {}, 23 | "bars": false, 24 | "dashLength": 10, 25 | "dashes": false, 26 | "datasource": "azure redis", 27 | "fieldConfig": { 28 | "defaults": { 29 | "custom": {} 30 | }, 31 | "overrides": [] 32 | }, 33 | "fill": 1, 34 | "fillGradient": 0, 35 | "gridPos": { 36 | "h": 9, 37 | "w": 12, 38 | "x": 0, 39 | "y": 0 40 | }, 41 | "hiddenSeries": false, 42 | "id": 2, 43 | "legend": { 44 | "avg": false, 45 | "current": false, 46 | "max": false, 47 | "min": false, 48 | "show": true, 49 | "total": false, 50 | "values": false 51 | }, 52 | "lines": true, 53 | "linewidth": 1, 54 | "nullPointMode": "null", 55 | "options": { 56 | "alertThreshold": true 57 | }, 58 | "percentage": false, 59 | "pluginVersion": "7.3.6", 60 | "pointradius": 2, 61 | "points": false, 62 | "renderer": "flot", 63 | "seriesOverrides": [], 64 | "spaceLength": 10, 65 | "stack": false, 66 | "steppedLine": false, 67 | "targets": [ 68 | { 69 | "command": "ts.get", 70 | "keyName": "temp:3:4", 71 | "query": "", 72 | "refId": "A", 73 | "streaming": true, 74 | "type": "timeSeries" 75 | } 76 | ], 77 | "thresholds": [], 78 | "timeFrom": null, 79 | "timeRegions": [], 80 | "timeShift": null, 81 | "title": "Temperature", 82 | "tooltip": { 83 | "shared": true, 84 | "sort": 0, 85 | "value_type": "individual" 86 | }, 87 | "type": "graph", 88 | "xaxis": { 89 | "buckets": null, 90 | "mode": "time", 91 | "name": null, 92 | "show": true, 93 | "values": [] 94 | }, 95 | "yaxes": [ 96 | { 97 | "format": "short", 98 | "label": null, 99 | "logBase": 1, 100 | "max": null, 101 | "min": null, 102 | "show": true 103 | }, 104 | { 105 | "format": "short", 106 | "label": null, 107 | "logBase": 1, 108 | "max": null, 109 | "min": null, 110 | "show": true 111 | } 112 | ], 113 | "yaxis": { 114 | "align": false, 115 | "alignLevel": null 116 | } 117 | } 118 | ], 119 | "refresh": "", 120 | "schemaVersion": 26, 121 | "style": "dark", 122 | "tags": [], 123 | "templating": { 124 | "list": [] 125 | }, 126 | "time": { 127 | "from": "now-6h", 128 | "to": "now" 129 | }, 130 | "timepicker": {}, 131 | "timezone": "", 132 | "title": "Temperature monitor (location/device)", 133 | "uid": "eJE9o5wMz", 134 | "version": 3 135 | } -------------------------------------------------------------------------------- /grafana_dashboards/temperature-aggregated.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "id": 4, 19 | "links": [], 20 | "panels": [ 21 | { 22 | "aliasColors": {}, 23 | "bars": false, 24 | "dashLength": 10, 25 | "dashes": false, 26 | "datasource": "azure redis", 27 | "fieldConfig": { 28 | "defaults": { 29 | "custom": {} 30 | }, 31 | "overrides": [] 32 | }, 33 | "fill": 1, 34 | "fillGradient": 0, 35 | "gridPos": { 36 | "h": 9, 37 | "w": 12, 38 | "x": 0, 39 | "y": 0 40 | }, 41 | "hiddenSeries": false, 42 | "id": 2, 43 | "legend": { 44 | "avg": false, 45 | "current": false, 46 | "max": false, 47 | "min": false, 48 | "show": true, 49 | "total": false, 50 | "values": false 51 | }, 52 | "lines": true, 53 | "linewidth": 1, 54 | "nullPointMode": "null", 55 | "options": { 56 | "alertThreshold": true 57 | }, 58 | "percentage": false, 59 | "pluginVersion": "7.3.6", 60 | "pointradius": 2, 61 | "points": false, 62 | "renderer": "flot", 63 | "seriesOverrides": [], 64 | "spaceLength": 10, 65 | "stack": false, 66 | "steppedLine": false, 67 | "targets": [ 68 | { 69 | "aggregation": "max", 70 | "bucket": 15000, 71 | "command": "ts.mrange", 72 | "filter": "metric=temp location=3 device=(1,2,3)", 73 | "query": "", 74 | "refId": "A", 75 | "type": "timeSeries" 76 | } 77 | ], 78 | "thresholds": [], 79 | "timeFrom": null, 80 | "timeRegions": [], 81 | "timeShift": null, 82 | "title": "Statistics", 83 | "tooltip": { 84 | "shared": true, 85 | "sort": 0, 86 | "value_type": "individual" 87 | }, 88 | "type": "graph", 89 | "xaxis": { 90 | "buckets": null, 91 | "mode": "time", 92 | "name": null, 93 | "show": true, 94 | "values": [] 95 | }, 96 | "yaxes": [ 97 | { 98 | "format": "short", 99 | "label": null, 100 | "logBase": 1, 101 | "max": null, 102 | "min": null, 103 | "show": true 104 | }, 105 | { 106 | "format": "short", 107 | "label": null, 108 | "logBase": 1, 109 | "max": null, 110 | "min": null, 111 | "show": true 112 | } 113 | ], 114 | "yaxis": { 115 | "align": false, 116 | "alignLevel": null 117 | } 118 | } 119 | ], 120 | "refresh": "5s", 121 | "schemaVersion": 26, 122 | "style": "dark", 123 | "tags": [], 124 | "templating": { 125 | "list": [] 126 | }, 127 | "time": { 128 | "from": "now-30m", 129 | "to": "now" 130 | }, 131 | "timepicker": {}, 132 | "timezone": "", 133 | "title": "Max temp (Aggregated)", 134 | "uid": "zPYG0cQMk", 135 | "version": 2 136 | } -------------------------------------------------------------------------------- /images/architecture_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abhirockzz/redis-timeseries-kafka/3b54c3564f1761a3e24bdaf83ddbe4f94e5e449b/images/architecture_2.png -------------------------------------------------------------------------------- /mqtt-source-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mqtt-source", 3 | "config": { 4 | "connector.class": "io.confluent.connect.mqtt.MqttSourceConnector", 5 | "tasks.max": "1", 6 | "mqtt.server.uri": "tcp://127.0.0.1:1883", 7 | "mqtt.topics": "device-stats", 8 | "kafka.topic": "mqtt.device-stats", 9 | "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter", 10 | "value.converter.schemas.enable": false, 11 | "key.converter": "org.apache.kafka.connect.json.JsonConverter", 12 | "key.converter.schemas.enable": false, 13 | "transforms": "convertToMap,convertKey,extract", 14 | "transforms.convertToMap.type": "org.apache.kafka.connect.transforms.HoistField$Value", 15 | "transforms.convertToMap.field": "data", 16 | "transforms.convertKey.type": "org.apache.kafka.connect.transforms.ValueToKey", 17 | "transforms.convertKey.fields": "data", 18 | "transforms.extract.type": "org.apache.kafka.connect.transforms.ExtractField$Value", 19 | "transforms.extract.field": "data" 20 | } 21 | } --------------------------------------------------------------------------------