├── .dockerignore ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── architecture.png ├── mqti ├── Makefile ├── VERSION ├── cmd │ └── main.go ├── commands │ ├── forward.go │ ├── root.go │ └── watch.go ├── config.go ├── config.yaml.example ├── influxdb.go ├── log.go ├── mapping.go ├── mqtt.go ├── time.go ├── tls.go ├── version.go └── worker.go └── scripts └── update_version.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | 16 | *.swp 17 | .envrc 18 | bin/ 19 | release/ 20 | vendor/ 21 | tmp/ 22 | reports 23 | mqti/config*.toml 24 | mqti/config*.yaml 25 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | install: 3 | - go get -v golang.org/x/tools/cmd/cover 4 | script: 5 | - make test 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | MAINTAINER Ash McKenzie 3 | 4 | RUN apk --update add bash curl 5 | 6 | RUN mkdir /app 7 | WORKDIR /app 8 | 9 | RUN curl -L https://github.com/ashmckenzie/go-mqti/releases/download/v0.1.1/mqti_linux_v0.1.1 > mqti && chmod 755 mqti 10 | 11 | CMD ["/app/mqti", "forward"] 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Ash McKenzie 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT_NAME = mqti 2 | 3 | .PHONY: test docker_image 4 | 5 | test: 6 | cd $(PROJECT_NAME) ; make test 7 | 8 | docker_image: 9 | docker build -t ashmckenzie/mqti . 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mqti 2 | 3 | [![Build Status](https://travis-ci.org/ashmckenzie/go-mqti.svg?branch=master)](https://travis-ci.org/ashmckenzie/go-mqti) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/ashmckenzie/go-mqti)](https://goreportcard.com/report/github.com/ashmckenzie/go-mqti) 5 | 6 | MQTT subscriber that pumps data into InfluxDB. 7 | 8 | Pronounced 'm-cutey' :) 9 | 10 | ## How it works 11 | 12 | ![Architecture](./architecture.png) 13 | 14 | ## Features 15 | 16 | * MQTT 3.1.1 supported, TLS, username/password 17 | * InfluxDB with TLS, username/password 18 | * Consume MQTT messages and inspect (`watch`) or `forward` with the following abilities: 19 | * Filter messages with AND + OR 20 | * Receive MQTT messages and write into InfluxDB, with the following abilities: 21 | * Add tags based on MQTT fields (when MQTT payload is JSON) 22 | * Geohash support (applicable when consuming MQTT messages from [Owntracks](http://owntracks.org/) 23 | * Includes `docker-compose.yaml` to get a full setup up and running! 24 | 25 | ## Configuration 26 | 27 | Configuration is handled through a `config.yaml` file. The following example reads as: 28 | 29 | * Setup four workers for incoming MQTT messages 30 | * Consume message from MQTT server `tcp://localhost:1883` with the client ID of `mqti` 31 | * When a MQTT message is consumed from the `temperature` topic, send write requests to InfluxDB server `http://localhost:8086`, into the `iot` database as measurement `temperature` 32 | 33 | ```yaml 34 | --- 35 | mqti: 36 | workers: 4 37 | 38 | mqtt: 39 | host: "localhost" 40 | port: "1883" 41 | client_id: "mqti" 42 | 43 | influxdb: 44 | host: "localhost" 45 | port: "8086" 46 | 47 | mappings: 48 | - mqtt: 49 | topic: "temperature" 50 | influxdb: 51 | database: "iot" 52 | measurement: "temperature" 53 | ``` 54 | 55 | ## Install 56 | 57 | `go get github.com/ashmckenzie/go-mqti/mqti` 58 | 59 | or download a release: 60 | 61 | [github.com/ashmckenzie/go-mqti/releases](https://github.com/ashmckenzie/go-mqti/releases) 62 | 63 | ## Usage 64 | 65 | 1. Ensure you have a `config.yaml` setup (see above) 66 | 67 | ### To consume MQTT messages only 68 | 69 | 1. `$GOPATH/bin/mqti watch` 70 | 71 | ### To consume MQTT messages *and* forward to InfluxDB 72 | 73 | 1. `$GOPATH/bin/mqti forward` 74 | 75 | ## Trying out with Docker 76 | 77 | See the [getting started](https://github.com/ashmckenzie/golang-melbourne-july-2017#getting-started) section of a Golang Melbourne presentation for a full demonstration :) 78 | 79 | ## Help 80 | 81 | ```shell 82 | MQTT subscriber that pumps data into InfluxDB 83 | 84 | Usage: 85 | mqti [flags] 86 | mqti [command] 87 | 88 | Available Commands: 89 | forward Forward MQTT messages on to InfluxDB 90 | watch Watch MQTT messages 91 | help Help about any command 92 | 93 | Flags: 94 | --config string config file (default is config.yaml) 95 | --debug enable debugging 96 | -h, --help help for mqti 97 | -v, --version show version 98 | 99 | Use "mqti [command] --help" for more information about a command. 100 | ``` 101 | 102 | ## Building 103 | 104 | 1. `make` 105 | 106 | ### TODO 107 | 108 | * [ ] Tests 109 | * [ ] Document externals 110 | * [ ] Fully document mungers 111 | * [ ] Leverage viper's default values 112 | 113 | ## Thanks 114 | 115 | * [github.com/eclipse/paho.mqtt.golang](https://github.com/eclipse/paho.mqtt.golang) 116 | * [github.com/influxdata/influxdb/client](https://github.com/influxdata/influxdb/client) 117 | * [github.com/spf13/cobra](https://github.com/spf13/cobra) 118 | * [github.com/spf13/viper](https://github.com/spf13/viper) 119 | * [github.com/Sirupsen/logrus](https://github.com/Sirupsen/logrus) 120 | 121 | ## License 122 | 123 | MIT License 124 | 125 | Copyright (c) 2017 Ash McKenzie 126 | 127 | Permission is hereby granted, free of charge, to any person obtaining a copy 128 | of this software and associated documentation files (the "Software"), to deal 129 | in the Software without restriction, including without limitation the rights 130 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 131 | copies of the Software, and to permit persons to whom the Software is 132 | furnished to do so, subject to the following conditions: 133 | 134 | The above copyright notice and this permission notice shall be included in all 135 | copies or substantial portions of the Software. 136 | 137 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 138 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 139 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 140 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 141 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 142 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 143 | SOFTWARE. 144 | -------------------------------------------------------------------------------- /architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashmckenzie/go-mqti/09261bf9fba008e87939271c105b52f21c640baa/architecture.png -------------------------------------------------------------------------------- /mqti/Makefile: -------------------------------------------------------------------------------- 1 | PROJECT_NAME ?= mqti 2 | VERSION=$(shell cat VERSION) 3 | 4 | BUILD_IMAGE_NAME = $(PROJECT_NAME)-build 5 | 6 | APP_DIR ?= $(PWD) 7 | BIN_DIR=$(APP_DIR)/bin 8 | RELEASE_DIR=$(APP_DIR)/release 9 | VENDOR_DIR=$(APP_DIR)/vendor 10 | 11 | BUILD_ARGS=-i -ldflags="" 12 | 13 | DARWIN_RELEASE=${PROJECT_NAME}_darwin_$(VERSION) 14 | LINUX_RELEASE=${PROJECT_NAME}_linux_$(VERSION) 15 | 16 | GITHUB_RELEASE_ARGS=--user ${GITHUB_USERNAME} --repo ${GITHUB_REPO} --tag ${VERSION} 17 | 18 | .PHONY: create_build_image deps update_deps make_dirs test run clean 19 | 20 | build: make_dirs test 21 | go build ${BUILD_ARGS} -o $(BIN_DIR)/$(PROJECT_NAME) ./cmd 22 | 23 | run: test 24 | go run cmd/main.go $(filter-out $@, $(MAKECMDGOALS)) 25 | 26 | release: make_dirs test 27 | # FIXME 28 | CGO_ENABLED=0 GOOS=darwin go build -a -installsuffix cgo ${BUILD_ARGS} -o ${RELEASE_DIR}/${DARWIN_RELEASE} ./cmd 29 | CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo ${BUILD_ARGS} -o ${RELEASE_DIR}/${LINUX_RELEASE} ./cmd 30 | 31 | release_upload: release 32 | git tag $(VERSION) 33 | git push --tags 34 | 35 | github-release release ${GITHUB_RELEASE_ARGS} 36 | github-release upload ${GITHUB_RELEASE_ARGS} --name ${DARWIN_RELEASE} --file ${RELEASE_DIR}/${DARWIN_RELEASE} 37 | github-release upload ${GITHUB_RELEASE_ARGS} --name ${LINUX_RELEASE} --file ${RELEASE_DIR}/${LINUX_RELEASE} 38 | 39 | test: deps 40 | go tool vet *.go cmd/*.go commands/*.go 41 | errcheck 42 | go test -cover 43 | 44 | report: deps 45 | mkdir -p reports 46 | goreporter -p ../mqti -r ../mqti/reports/ -e vendor 47 | 48 | deps: make_dirs 49 | go get github.com/kardianos/govendor 50 | go get github.com/kisielk/errcheck 51 | go get github.com/aktau/github-release 52 | govendor fetch +missing 53 | go generate -v -x ./ ./cmd ./commands 54 | 55 | update_deps: 56 | go get -u github.com/kardianos/govendor 57 | go get -u github.com/aktau/github-release 58 | govendor update +e 59 | 60 | make_dirs: 61 | @mkdir -p ${BIN_DIR} ${RELEASE_DIR} ${VENDOR_DIR} 62 | 63 | clean: 64 | rm -rf $(BIN_DIR) $(RELEASE_DIR) 65 | -------------------------------------------------------------------------------- /mqti/VERSION: -------------------------------------------------------------------------------- 1 | v0.1.2 2 | -------------------------------------------------------------------------------- /mqti/cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/ashmckenzie/go-mqti/mqti" 7 | "github.com/ashmckenzie/go-mqti/mqti/commands" 8 | ) 9 | 10 | func main() { 11 | if err := commands.RootCmd.Execute(); err != nil { 12 | mqti.Log.Fatalf("%s\n", err) 13 | os.Exit(1) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /mqti/commands/forward.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "github.com/ashmckenzie/go-mqti/mqti" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | var forwardCmd = &cobra.Command{ 9 | Use: "forward", 10 | Short: "Forward MQTT messages on to InfluxDB", 11 | Run: func(cmd *cobra.Command, args []string) { 12 | forwardMessages() 13 | }, 14 | } 15 | 16 | func init() { 17 | RootCmd.AddCommand(forwardCmd) 18 | } 19 | 20 | func forwardMessages() { 21 | influxDB, err := mqti.NewInfluxDBConnection() 22 | if err != nil { 23 | mqti.Log.Fatal(nil) 24 | } 25 | 26 | incoming := make(chan *mqti.MQTTMessage) 27 | forward := make(chan *mqti.MQTTMessage) 28 | 29 | go mqti.CreateWorkers(influxDB, forward) 30 | go mqti.MQTTSubscribe(incoming) 31 | 32 | for m := range incoming { 33 | mqti.DebugLogMQTTMessage(m) 34 | forward <- m 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /mqti/commands/root.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/ashmckenzie/go-mqti/mqti" 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | ) 11 | 12 | var configFile string 13 | var debug, showVersion bool 14 | 15 | // RootCmd ... 16 | var RootCmd = &cobra.Command{ 17 | Use: "mqti", 18 | Short: "MQTT subscriber that pumps data into InfluxDB", 19 | Long: `MQTT subscriber that pumps data into InfluxDB`, 20 | SilenceErrors: true, 21 | PersistentPreRunE: func(cmd *cobra.Command, args []string) error { 22 | if err := validateInput(); err != nil { 23 | return err 24 | } 25 | 26 | mqti.EnableDebugging(debug) 27 | 28 | return nil 29 | }, 30 | Run: func(cmd *cobra.Command, args []string) { 31 | if showVersion { 32 | fmt.Println(mqti.Version) 33 | } 34 | cmd.Help() 35 | }, 36 | } 37 | 38 | func validateInput() error { 39 | return nil 40 | } 41 | 42 | // Execute ... 43 | func Execute() error { 44 | err := RootCmd.Execute() 45 | return err 46 | } 47 | 48 | func init() { 49 | cobra.OnInitialize(initConfig) 50 | 51 | RootCmd.Flags().BoolVarP(&showVersion, "version", "v", false, "show version") 52 | RootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "enable debugging") 53 | 54 | RootCmd.PersistentFlags().StringVar(&configFile, "config", "", "config file (default is config.yaml)") 55 | } 56 | 57 | func initConfig() { 58 | if configFile != "" { 59 | viper.SetConfigFile(configFile) 60 | } else { 61 | viper.AddConfigPath(".") 62 | viper.SetConfigName("config") 63 | } 64 | 65 | viper.AutomaticEnv() 66 | 67 | if err := viper.ReadInConfig(); err != nil { 68 | mqti.Log.Fatal("Can't read config:", err) 69 | os.Exit(1) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /mqti/commands/watch.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "github.com/ashmckenzie/go-mqti/mqti" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | var watchCmd = &cobra.Command{ 9 | Use: "watch", 10 | Short: "Watch MQTT messages", 11 | Run: func(cmd *cobra.Command, args []string) { 12 | watchMessages() 13 | }, 14 | } 15 | 16 | func init() { 17 | RootCmd.AddCommand(watchCmd) 18 | } 19 | 20 | func watchMessages() { 21 | incoming := make(chan *mqti.MQTTMessage) 22 | go mqti.MQTTSubscribe(incoming) 23 | 24 | for m := range incoming { 25 | mqti.LogMQTTMessage(m) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /mqti/config.go: -------------------------------------------------------------------------------- 1 | package mqti 2 | 3 | type mQtiConfiguration struct { 4 | Workers int 5 | } 6 | 7 | type mQTTConfiguration struct { 8 | Host string 9 | Port string 10 | ClientID string 11 | } 12 | 13 | type influxDBConfiguration struct { 14 | Host string 15 | Port string 16 | } 17 | -------------------------------------------------------------------------------- /mqti/config.yaml.example: -------------------------------------------------------------------------------- 1 | --- 2 | mqti: 3 | workers: 4 4 | 5 | mqtt: 6 | host: "localhost" 7 | port: "1883" 8 | client_id: "mqti" 9 | 10 | influxdb: 11 | host: "localhost" 12 | port: "8086" 13 | 14 | mappings: 15 | - mqtt: 16 | topic: "temperature" 17 | influxdb: 18 | database: "iot" 19 | measurement: "temperature" 20 | -------------------------------------------------------------------------------- /mqti/influxdb.go: -------------------------------------------------------------------------------- 1 | package mqti 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "time" 7 | 8 | InfluxDBClient "github.com/influxdata/influxdb/client" 9 | "github.com/mmcloughlin/geohash" 10 | "github.com/spf13/viper" 11 | ) 12 | 13 | // InfluxDBConnection ... 14 | type InfluxDBConnection struct { 15 | *InfluxDBClient.Client 16 | } 17 | 18 | func (i InfluxDBConnection) geoHashFieldsDefined(g GeohashMungerConfiguration) bool { 19 | return len(g.LatitudeField) > 0 && len(g.LongitudeField) > 0 && len(g.ResultField) > 0 20 | } 21 | 22 | func (i InfluxDBConnection) applyGeohashMunger(g GeohashMungerConfiguration, fields map[string]interface{}, tags map[string]string) error { 23 | if i.geoHashFieldsDefined(g) { 24 | tags[g.ResultField] = geohash.Encode( 25 | fields[g.LatitudeField].(float64), 26 | fields[g.LongitudeField].(float64)) 27 | } 28 | 29 | return nil 30 | } 31 | 32 | func (i InfluxDBConnection) applyTagsMunger(t TagsMungerConfiguration, fields map[string]interface{}, tags map[string]string) error { 33 | for _, x := range t.From { 34 | for k, v := range x { 35 | if fields[k] != nil { 36 | tags[v] = fields[k].(string) 37 | } 38 | } 39 | } 40 | return nil 41 | } 42 | 43 | func (i InfluxDBConnection) applyMungers(m struct { 44 | Tags TagsMungerConfiguration 45 | Geohash GeohashMungerConfiguration 46 | }, fields map[string]interface{}, tags map[string]string) error { 47 | var err error 48 | 49 | if err = i.applyGeohashMunger(m.Geohash, fields, tags); err != nil { 50 | Log.Warn(err) 51 | } 52 | 53 | if err = i.applyTagsMunger(m.Tags, fields, tags); err != nil { 54 | Log.Warn(err) 55 | } 56 | 57 | return err 58 | } 59 | 60 | // Forward ... 61 | func (i InfluxDBConnection) Forward(m *MQTTMessage) error { 62 | var err error 63 | var fields map[string]interface{} 64 | 65 | config := m.MappingConfiguration.InfluxDB 66 | 67 | tags := config.Tags 68 | if tags == nil { 69 | tags = make(map[string]string) 70 | } 71 | 72 | fields, err = m.PayloadAsJSON() 73 | if err == nil { 74 | mungers := m.MappingConfiguration.InfluxDB.Mungers 75 | if err = i.applyMungers(mungers, fields, tags); err != nil { 76 | Log.Warn(err) 77 | } 78 | } else { 79 | fields = map[string]interface{}{"value": m.PayloadAsString()} 80 | } 81 | 82 | p := InfluxDBClient.Point{ 83 | Measurement: config.Measurement, 84 | Tags: tags, 85 | Fields: fields, 86 | Time: time.Now(), 87 | } 88 | 89 | Log.Info(p) 90 | 91 | _, err = i.Write(InfluxDBClient.BatchPoints{ 92 | Points: []InfluxDBClient.Point{p}, 93 | Database: m.MappingConfiguration.InfluxDB.Database, 94 | }) 95 | 96 | return err 97 | } 98 | 99 | func influxDBConfig() map[string]interface{} { 100 | return viper.GetStringMap("influxdb") 101 | } 102 | 103 | func influxDBURI() *url.URL { 104 | host, _ := url.Parse(fmt.Sprintf("%s://%s:%s", influxDBProtocol(), influxDBConfig()["host"], influxDBConfig()["port"])) 105 | return host 106 | } 107 | 108 | func influxDBProtocol() string { 109 | t := influxDBConfig()["tls"] 110 | if t != nil && t.(bool) { 111 | return "https" 112 | } 113 | return "http" 114 | } 115 | 116 | func influxDBUsername() string { 117 | u := influxDBConfig()["username"] 118 | if u != nil { 119 | return u.(string) 120 | } 121 | return "" 122 | } 123 | 124 | func influxDBPassword() string { 125 | p := influxDBConfig()["password"] 126 | if p != nil { 127 | return p.(string) 128 | } 129 | return "" 130 | } 131 | 132 | // NewInfluxDBConnection ... 133 | func NewInfluxDBConnection() (*InfluxDBConnection, error) { 134 | var err error 135 | var influxDBConn *InfluxDBClient.Client 136 | 137 | opts := InfluxDBClient.Config{URL: *influxDBURI()} 138 | 139 | if influxDBUsername() != "" && influxDBPassword() != "" { 140 | opts.Username = influxDBUsername() 141 | opts.Password = influxDBPassword() 142 | } 143 | 144 | influxDBConn, err = InfluxDBClient.NewClient(opts) 145 | if err != nil { 146 | return nil, err 147 | } 148 | 149 | return &InfluxDBConnection{influxDBConn}, nil 150 | } 151 | -------------------------------------------------------------------------------- /mqti/log.go: -------------------------------------------------------------------------------- 1 | package mqti 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "runtime" 8 | 9 | "github.com/Sirupsen/logrus" 10 | MQTT "github.com/eclipse/paho.mqtt.golang" 11 | ) 12 | 13 | // Log ... 14 | var Log = logrus.New() 15 | 16 | // DiskLog ... 17 | var DiskLog *logrus.Logger 18 | 19 | // DiskLogFile ... 20 | var DiskLogFile *os.File 21 | 22 | // DEBUGDISKFILE ... 23 | const DEBUGDISKFILE string = "/tmp/mqti-debug.log" 24 | 25 | func init() { 26 | setupStderrLogging() 27 | } 28 | 29 | // EnableDebugging ... 30 | func EnableDebugging(yes bool) { 31 | var err error 32 | 33 | if yes { 34 | // Log.Infof("Debugging output will go to %s", DEBUGDISKFILE) 35 | DiskLog = logrus.New() 36 | 37 | MQTT.DEBUG = log.New(os.Stderr, "DEBUG - ", log.LstdFlags) 38 | MQTT.CRITICAL = log.New(os.Stderr, "CRITICAL - ", log.LstdFlags) 39 | MQTT.WARN = log.New(os.Stderr, "WARN - ", log.LstdFlags) 40 | MQTT.ERROR = log.New(os.Stderr, "ERROR - ", log.LstdFlags) 41 | 42 | setLogLevelFor(Log, logrus.DebugLevel) 43 | setLogLevelFor(DiskLog, logrus.DebugLevel) 44 | DiskLog.Formatter = &logrus.JSONFormatter{} 45 | if DiskLogFile, err = os.OpenFile(DEBUGDISKFILE, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600); err != nil { 46 | Log.Panic(err) 47 | } 48 | DiskLog.Out = DiskLogFile 49 | } 50 | } 51 | 52 | func setupStderrLogging() { 53 | Log.Out = os.Stderr 54 | setLogLevelFor(Log, logrus.InfoLevel) 55 | } 56 | 57 | func setLogLevelFor(l *logrus.Logger, level logrus.Level) { 58 | l.Level = level 59 | } 60 | 61 | // DebugLog ... 62 | func DebugLog(line ...interface{}) { 63 | if DiskLog != nil { 64 | logIt(DiskLog, logrus.DebugLevel, line) 65 | } 66 | } 67 | 68 | // LogMQTTMessage ... 69 | func LogMQTTMessage(m *MQTTMessage) { 70 | logMQTTMessage(m, logrus.InfoLevel) 71 | } 72 | 73 | // DebugLogMQTTMessage ... 74 | func DebugLogMQTTMessage(m *MQTTMessage) { 75 | logMQTTMessage(m, logrus.DebugLevel) 76 | } 77 | 78 | func logMQTTMessage(m *MQTTMessage, level logrus.Level) { 79 | payload := string(m.Payload()) 80 | fields := logrus.Fields{ 81 | "topic": m.Topic(), 82 | "mqtt": m.MappingConfiguration.MQTT, 83 | "influxdb": m.MappingConfiguration.InfluxDB, 84 | } 85 | 86 | switch level { 87 | case logrus.InfoLevel: 88 | Log.WithFields(fields).Info(payload) 89 | case logrus.DebugLevel: 90 | Log.WithFields(fields).Debug(payload) 91 | } 92 | } 93 | 94 | func logIt(l *logrus.Logger, level logrus.Level, msg ...interface{}) { 95 | pc, _, _, _ := runtime.Caller(2) 96 | details := runtime.FuncForPC(pc) 97 | fileFunc, lineFunc := details.FileLine(pc) 98 | location := fmt.Sprintf("%s:%d", fileFunc, lineFunc-2) 99 | msgAsString := fmt.Sprintf("%s", msg) 100 | fields := logrus.Fields{"location": location} 101 | 102 | switch level { 103 | case logrus.DebugLevel: 104 | l.WithFields(fields).Debug(msgAsString) 105 | break 106 | case logrus.InfoLevel: 107 | l.WithFields(fields).Info(msgAsString) 108 | break 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /mqti/mapping.go: -------------------------------------------------------------------------------- 1 | package mqti 2 | 3 | import "github.com/spf13/viper" 4 | 5 | type mQTTMappingConfiguration struct { 6 | Topic string 7 | Mungers struct { 8 | Filter FilterMungerConfiguration `mapstructure:"filter"` 9 | } 10 | } 11 | 12 | type influxDBMappingConfiguration struct { 13 | Database string 14 | Measurement string 15 | Tags map[string]string 16 | Mungers struct { 17 | Tags TagsMungerConfiguration 18 | Geohash GeohashMungerConfiguration 19 | } 20 | } 21 | 22 | // FilterMungerConfiguration ... 23 | type FilterMungerConfiguration struct { 24 | JSON FilterJSONMungerConfiguration 25 | } 26 | 27 | // FilterJSONMungerConfiguration ... 28 | type FilterJSONMungerConfiguration struct { 29 | And []map[string]string 30 | Or []map[string]string 31 | } 32 | 33 | // TagsMungerConfiguration ... 34 | type TagsMungerConfiguration struct { 35 | From []map[string]string 36 | } 37 | 38 | // GeohashMungerConfiguration ... 39 | type GeohashMungerConfiguration struct { 40 | LatitudeField string `mapstructure:"lat_field"` 41 | LongitudeField string `mapstructure:"lng_field"` 42 | ResultField string `mapstructure:"result_field"` 43 | } 44 | 45 | // MappingConfiguration ... 46 | type MappingConfiguration struct { 47 | Name string 48 | MQTT mQTTMappingConfiguration 49 | InfluxDB influxDBMappingConfiguration 50 | } 51 | 52 | // Config ... 53 | type Config struct { 54 | MQti mQtiConfiguration 55 | MQTT mQTTConfiguration 56 | InfluxDB influxDBConfiguration 57 | Mappings []MappingConfiguration 58 | } 59 | 60 | // GetConfig ... 61 | func GetConfig() (*Config, error) { 62 | var err error 63 | var c Config 64 | 65 | if err = viper.Unmarshal(&c); err != nil { 66 | return nil, err 67 | } 68 | 69 | return &c, err 70 | } 71 | -------------------------------------------------------------------------------- /mqti/mqtt.go: -------------------------------------------------------------------------------- 1 | package mqti 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | "time" 11 | 12 | MQTT "github.com/eclipse/paho.mqtt.golang" 13 | "github.com/spf13/viper" 14 | ) 15 | 16 | const mQTTDefaultPort string = "1883" 17 | 18 | // MQTTMessage ... 19 | type MQTTMessage struct { 20 | MQTT.Message 21 | MappingConfiguration 22 | } 23 | 24 | // PayloadAsString ... 25 | func (m MQTTMessage) PayloadAsString() string { 26 | return string(m.Payload()) 27 | } 28 | 29 | // PayloadAsJSON ... 30 | func (m MQTTMessage) PayloadAsJSON() (map[string]interface{}, error) { 31 | var fields map[string]interface{} 32 | 33 | err := json.Unmarshal(m.Payload(), &fields) 34 | 35 | return fields, err 36 | } 37 | 38 | func (m MQTTMessage) jSONFilterShouldSkip(j map[string]interface{}, f []map[string]string, invert bool) bool { 39 | skip := false 40 | 41 | for _, x := range f { 42 | skip = invert 43 | for k, v := range x { 44 | if (j[k] == v) == invert { 45 | skip = !invert 46 | } 47 | if !invert && skip { 48 | break 49 | } 50 | } 51 | if !invert && skip { 52 | break 53 | } 54 | } 55 | 56 | return skip 57 | } 58 | 59 | func (m MQTTMessage) shouldSkip() bool { 60 | if m.jSONFiltersDefined() { 61 | payload, err := m.PayloadAsJSON() 62 | 63 | if err == nil { 64 | jsonFilters := m.MQTT.Mungers.Filter.JSON 65 | return m.jSONFilterShouldSkip(payload, jsonFilters.And, false) || m.jSONFilterShouldSkip(payload, jsonFilters.Or, true) 66 | } 67 | 68 | return true 69 | } 70 | 71 | return false 72 | } 73 | 74 | func (m MQTTMessage) jSONFiltersDefined() bool { 75 | return (len(m.MQTT.Mungers.Filter.JSON.And) > 0 || len(m.MQTT.Mungers.Filter.JSON.Or) > 0) 76 | } 77 | 78 | func mQTTConfig() map[string]interface{} { 79 | return viper.GetStringMap("mqtt") 80 | } 81 | 82 | func mQTTBrokerURI() string { 83 | return fmt.Sprintf("%s://%s:%s", mQTTProtocol(), mQTTConfig()["host"], mQTTPort()) 84 | } 85 | 86 | func mQTTPort() string { 87 | var port string 88 | if p := mQTTConfig()["port"]; p != nil { 89 | port = p.(string) 90 | } else { 91 | port = mQTTDefaultPort 92 | } 93 | return port 94 | } 95 | 96 | func mQTTProtocol() string { 97 | if p := mQTTConfig()["protocol"]; p != nil { 98 | return p.(string) 99 | } 100 | if mQTTTLSDefined() { 101 | return "ssl" 102 | } 103 | return "tcp" 104 | } 105 | 106 | func mQTTClientID() string { 107 | return mQTTConfig()["client_id"].(string) 108 | } 109 | 110 | func mQTTUsername() string { 111 | u := mQTTConfig()["username"] 112 | if u != nil { 113 | return u.(string) 114 | } 115 | return "" 116 | } 117 | 118 | func mQTTPassword() string { 119 | p := mQTTConfig()["password"] 120 | if p != nil { 121 | return p.(string) 122 | } 123 | return "" 124 | } 125 | 126 | func mQTTTLSDefined() bool { 127 | return mQTTConfig()["tls_cert"] != nil && mQTTConfig()["tls_private_key"] != nil 128 | } 129 | 130 | func mQTTTLSConfig() tls.Config { 131 | return *NewTLSConfig(mQTTConfig()["tls_cert"].(string), mQTTConfig()["tls_private_key"].(string)) 132 | } 133 | 134 | func mQTTCleanSession() bool { 135 | return mQTTConfig()["clean_session"] != nil && (mQTTConfig()["clean_session"].(bool) == true) 136 | } 137 | 138 | // MQTTSubscribe ... 139 | func MQTTSubscribe(incoming chan *MQTTMessage) { 140 | var outgoing chan *MQTTMessage 141 | outgoing = incoming 142 | 143 | cs := make(chan os.Signal, 1) 144 | signal.Notify(cs, os.Interrupt, syscall.SIGTERM) 145 | go func() { 146 | <-cs 147 | Log.Error("signal received, exiting") 148 | os.Exit(0) 149 | }() 150 | 151 | opts := MQTT.NewClientOptions() 152 | 153 | opts.ClientID = mQTTClientID() 154 | opts.Username = mQTTUsername() 155 | opts.Password = mQTTPassword() 156 | opts.CleanSession = mQTTCleanSession() 157 | opts.TLSConfig = tls.Config{} 158 | 159 | if mQTTTLSDefined() { 160 | opts.TLSConfig = mQTTTLSConfig() 161 | } 162 | 163 | opts.AddBroker(mQTTBrokerURI()) 164 | 165 | opts.OnConnect = func(c MQTT.Client) { 166 | var err error 167 | var config *Config 168 | 169 | config, err = GetConfig() 170 | if err != nil { 171 | Log.Fatal(err) 172 | } 173 | 174 | for _, mapping := range config.Mappings { 175 | m := mapping 176 | var f MQTT.MessageHandler = func(client MQTT.Client, msg MQTT.Message) { 177 | mQTTMessage := &MQTTMessage{msg, m} 178 | 179 | if mQTTMessage.shouldSkip() { 180 | Log.Debugf("No match! %v", mQTTMessage.PayloadAsString()) 181 | } else { 182 | Log.Debugf("Match! %v", mQTTMessage.PayloadAsString()) 183 | outgoing <- mQTTMessage 184 | } 185 | } 186 | 187 | c.Subscribe(mapping.MQTT.Topic, 0, f) 188 | } 189 | } 190 | 191 | opts.OnConnectionLost = func(c MQTT.Client, e error) { 192 | Log.Error(e) 193 | } 194 | 195 | client := MQTT.NewClient(opts) 196 | 197 | if token := client.Connect(); token.Wait() && token.Error() != nil { 198 | panic(token.Error()) 199 | } 200 | 201 | for { 202 | time.Sleep(1 * time.Second) 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /mqti/time.go: -------------------------------------------------------------------------------- 1 | package mqti 2 | 3 | import ( 4 | "strconv" 5 | "time" 6 | ) 7 | 8 | // EndOfTime ... 9 | const EndOfTime string = "9999-12-31T23:59:59" 10 | 11 | // ParseEpoch ... 12 | func ParseEpoch(in string) time.Time { 13 | var err error 14 | var i int64 15 | 16 | if i, err = strconv.ParseInt(in, 10, 64); err != nil { 17 | Log.Panic(err) 18 | } 19 | 20 | return time.Unix(i, 0).UTC() 21 | } 22 | 23 | // ParseTime ... 24 | func ParseTime(in string) time.Time { 25 | var err error 26 | var t time.Time 27 | 28 | if t, err = time.Parse("2006-01-02T15:04:05", in); err != nil { 29 | Log.Panic(err) 30 | } 31 | 32 | return t.UTC() 33 | } 34 | -------------------------------------------------------------------------------- /mqti/tls.go: -------------------------------------------------------------------------------- 1 | package mqti 2 | 3 | import "crypto/tls" 4 | 5 | // NewTLSConfig ... 6 | func NewTLSConfig(certFile, keyFile string) *tls.Config { 7 | var err error 8 | var cert tls.Certificate 9 | 10 | cert, err = tls.LoadX509KeyPair(certFile, keyFile) 11 | if err != nil { 12 | panic(err) 13 | } 14 | 15 | return &tls.Config{ 16 | InsecureSkipVerify: false, 17 | Certificates: []tls.Certificate{cert}, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /mqti/version.go: -------------------------------------------------------------------------------- 1 | package mqti 2 | 3 | //go:generate bash ../scripts/update_version.sh 4 | 5 | // Version ... 6 | var Version = "v0.1.2" 7 | -------------------------------------------------------------------------------- /mqti/worker.go: -------------------------------------------------------------------------------- 1 | package mqti 2 | 3 | // CreateWorkers ... 4 | func CreateWorkers(influxDB *InfluxDBConnection, jobs <-chan *MQTTMessage) { 5 | var err error 6 | var config *Config 7 | 8 | config, err = GetConfig() 9 | if err != nil { 10 | Log.Fatal(err) 11 | } 12 | 13 | for w := 1; w <= config.MQti.Workers; w++ { 14 | createWorker(w, influxDB, jobs) 15 | } 16 | } 17 | 18 | func createWorker(id int, influxDB *InfluxDBConnection, jobs <-chan *MQTTMessage) { 19 | var err error 20 | for j := range jobs { 21 | if err = influxDB.Forward(j); err != nil { 22 | Log.Error(err) 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /scripts/update_version.sh: -------------------------------------------------------------------------------- 1 | # Get the version. 2 | version=`cat VERSION` 3 | 4 | # Write out the package. 5 | cat << EOF > version.go 6 | package mqti 7 | 8 | //go:generate bash ../scripts/update_version.sh 9 | 10 | // Version ... 11 | var Version = "$version" 12 | EOF 13 | --------------------------------------------------------------------------------