├── tests ├── __init__.py ├── test_mqtt.py └── test_valuemap.py ├── version ├── mqtt_kube ├── __init__.py ├── binding │ ├── __init__.py │ ├── association.py │ ├── patcher.py │ ├── watcher.py │ ├── actioner.py │ ├── binding.py │ └── valuemap.py ├── k8s │ ├── text.py │ ├── api.py │ ├── jsonpathadaptor.py │ ├── template.py │ ├── listener.py │ ├── locus.py │ └── workload.py ├── server.py ├── mqttjson.py ├── main.py ├── config.py └── mqtt.py ├── .gitignore ├── CHANGELOG.md ├── python3-requirements.txt ├── setup.py ├── job-example.yaml ├── docker └── Dockerfile.alpine ├── LICENSE ├── config-basic.yaml ├── Makefile └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /version: -------------------------------------------------------------------------------- 1 | 0.0.0 2 | -------------------------------------------------------------------------------- /mqtt_kube/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mqtt_kube/binding/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/__pycache__/ 2 | *.egg-info/ 3 | virtualenv/ 4 | .vscode/ 5 | reports/ 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | MQTT Kube :: Changelog 2 | === 3 | 4 | 5 | # v0.0.0 6 | Alpha release 7 | -------------------------------------------------------------------------------- /python3-requirements.txt: -------------------------------------------------------------------------------- 1 | PyYAML 2 | paho-mqtt 3 | gevent 4 | requests 5 | kubernetes 6 | jsonpath-ng 7 | Mako 8 | -------------------------------------------------------------------------------- /mqtt_kube/binding/association.py: -------------------------------------------------------------------------------- 1 | class Association: 2 | def __init__(self, mqtt, topic, valuemap): 3 | self._mqtt = mqtt 4 | self._topic = topic 5 | self._valuemap = valuemap 6 | -------------------------------------------------------------------------------- /mqtt_kube/binding/patcher.py: -------------------------------------------------------------------------------- 1 | import mqtt_kube.binding.association 2 | 3 | 4 | class Patcher(mqtt_kube.binding.association.Association): 5 | def __init__(self, locus, mqtt, topic, valuemap): 6 | super().__init__(mqtt, topic, valuemap) 7 | self._locus = locus 8 | 9 | def open(self): 10 | self._mqtt.subscribe(self._topic, self._on_payload) 11 | 12 | def _on_payload(self, payload, _timestamp): 13 | match = self._valuemap.lookup(payload) 14 | if match: 15 | self._locus.write(match.value) 16 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup( 5 | name="mqtt-kube", 6 | version=open('version', 'rt', encoding="utf8").read().strip(), 7 | description="Kubernetes API to MQTT connector service", 8 | author="Source Simian", 9 | url="https://github.com/sourcesimian/mqtt-panel", 10 | license="MIT", 11 | packages=['mqtt_kube'], 12 | install_requires=open('python3-requirements.txt', encoding="utf8").readlines(), 13 | entry_points={ 14 | "console_scripts": [ 15 | "mqtt-kube=mqtt_kube.main:cli", 16 | ] 17 | }, 18 | ) 19 | -------------------------------------------------------------------------------- /job-example.yaml: -------------------------------------------------------------------------------- 1 | kind: Job 2 | apiVersion: batch/v1 3 | metadata: 4 | name: ${binding.name} #-${uid} 5 | spec: 6 | activeDeadlineSeconds: 60 7 | ttlSecondsAfterFinished: 300 8 | backoffLimit: 0 9 | template: 10 | spec: 11 | containers: 12 | - name: ${binding.name} 13 | image: alpine 14 | command: 15 | - wget 16 | - -O 17 | - /dev/stdout 18 | - http://www.example.com 19 | resources: 20 | requests: 21 | memory: "32Mi" 22 | cpu: "10m" 23 | limits: 24 | memory: "32Mi" 25 | cpu: "10m" 26 | restartPolicy: Never 27 | -------------------------------------------------------------------------------- /mqtt_kube/k8s/text.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | 4 | from datetime import datetime 5 | 6 | _re_camel_to_snake = re.compile(r'(?Hello from mqtt-mqtt

"] 45 | 46 | start_response('404 Not Found', [('Content-Type', 'text/html')]) 47 | return [b'

Not Found

'] 48 | 49 | 50 | class Handler(WSGIHandler): 51 | def log_request(self): 52 | if '101' not in str(self.status): 53 | logging.debug(self.format_request()) 54 | -------------------------------------------------------------------------------- /mqtt_kube/mqttjson.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import jsonpath_ng 4 | 5 | from mqtt_kube.mqtt import Mqtt 6 | 7 | 8 | class MqttJson(Mqtt): 9 | def __init__(self, config): 10 | Mqtt.__init__(self, config) 11 | 12 | @classmethod 13 | def _json_loads(cls, payload): 14 | try: 15 | if isinstance(payload, str): 16 | return json.loads(payload) 17 | except json.JSONDecodeError: 18 | pass 19 | raise ValueError('Could not extract JSON') 20 | 21 | @classmethod 22 | def _partial_jsonpath_find(cls, jsonpath): 23 | jsonpath = jsonpath_ng.parse(jsonpath) 24 | 25 | def jsonpath_find(payload): 26 | value = jsonpath.find(payload) 27 | if value: 28 | return value[0].value # take first value 29 | raise ValueError('JSONPath not found') 30 | return jsonpath_find 31 | 32 | @classmethod 33 | def _split_topic(cls, topic): 34 | parts = topic.split('|') 35 | topic = parts[0] 36 | modifiers = [] 37 | for part in parts[1:]: 38 | if part.startswith(('$', '.')): 39 | modifiers.append(cls._json_loads) 40 | modifiers.append(cls._partial_jsonpath_find(part)) 41 | else: 42 | modifiers.append(eval(part)) # pylint: disable=W0123 43 | return topic, modifiers 44 | 45 | def subscribe(self, topic, on_payload): 46 | topic, modifiers = self._split_topic(topic) 47 | 48 | def _on_payload(payload, timestamp): 49 | value = payload 50 | for modifier in modifiers: 51 | value = modifier(value) 52 | if value is None: 53 | break 54 | on_payload(self._json_loads(payload), value, timestamp) 55 | 56 | Mqtt.subscribe(self, topic, _on_payload) 57 | -------------------------------------------------------------------------------- /config-basic.yaml: -------------------------------------------------------------------------------- 1 | mqtt: 2 | host: broker.emqx.io 3 | port: 1883 4 | client-id: mqtt-kube-demo 5 | topic-prefix: sourcesimian/mqtt-kube/demo 6 | 7 | http: 8 | host: 0.0.0.0 9 | port: 8080 10 | max-connections: 10 11 | 12 | logging: 13 | level: DEBUG 14 | 15 | bindings: 16 | - namespace: mqtt-kube-demo 17 | resource: Deployment 18 | name: jellyfin 19 | 20 | patch: 21 | - topic: jellyfin/power/set 22 | locus: "$.spec.replicas" 23 | values: 24 | map: 25 | "ON": 1 26 | "OFF": 0 27 | 28 | watch: 29 | - locus: "$.status.availableReplicas" 30 | topic: jellyfin/power/state 31 | retain: true 32 | qos: 1 33 | values: 34 | map: 35 | 1: "ON" 36 | ~: "OFF" 37 | 38 | - namespace: mqtt-kube-demo 39 | resource: Job 40 | name: example-wget 41 | 42 | watch: 43 | - locus: "$.status" 44 | topic: example/status 45 | 46 | - locus: "$.status.failed" 47 | topic: example/status/failed 48 | values: 49 | map: 50 | ~: ~ 51 | "lambda v: v > 0": "FAILED" 52 | 53 | - locus: "$.status.active" 54 | topic: example/status/active 55 | values: 56 | map: 57 | ~: ~ 58 | 1: "ACTIVE" 59 | 60 | - locus: "$.status.succeeded" 61 | topic: example/status/succeeded 62 | values: 63 | map: 64 | ~: ~ 65 | 1: "DONE" 66 | 67 | - locus: "$.status.startTime" 68 | topic: example/status/startTime 69 | 70 | action: 71 | - topic: example/cmd # e.g. topic: sourcesimian/mqtt-kube/demo/example/cmd 72 | launch: !relpath job-example.yaml 73 | values: 74 | jsonpath: "$.action" 75 | map: 76 | "RUN": "launch" # e.g. payload: {"action": "RUN"} 77 | "STOP": "delete" # e.g. payload: {"action": "STOP"} 78 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | REGISTRY= 2 | REPO=sourcesimian/mqtt-kube 3 | TAG=$(shell cat version) 4 | 5 | check: 6 | flake8 ./mqtt_kube --ignore E501 7 | find ./mqtt_kube -name '*.py' \ 8 | | xargs pylint -d invalid-name \ 9 | -d missing-docstring \ 10 | -d too-few-public-methods \ 11 | -d line-too-long \ 12 | -d too-many-arguments \ 13 | -d too-many-instance-attributes \ 14 | -d no-self-use 15 | 16 | test: 17 | pytest ./tests/ -vvv --junitxml=./reports/unittest-results.xml 18 | 19 | docker-armv6: 20 | $(eval REPOTAG := ${REGISTRY}${REPO}:${TAG}-armv6) 21 | docker buildx build \ 22 | --platform linux/arm/v6 \ 23 | --load \ 24 | -t ${REPOTAG} \ 25 | -f docker/Dockerfile.alpine \ 26 | . 27 | 28 | docker-amd64: 29 | $(eval REPOTAG := ${REGISTRY}${REPO}:${TAG}-amd64) 30 | docker buildx build \ 31 | --platform linux/amd64 \ 32 | --load \ 33 | -t ${REPOTAG} \ 34 | -f docker/Dockerfile.alpine \ 35 | . 36 | 37 | push: docker-armv6 docker-amd64 38 | $(eval REPOTAG := ${REGISTRY}${REPO}:${TAG}) 39 | $(eval LATEST := ${REGISTRY}${REPO}:latest) 40 | docker push ${REPOTAG}-amd64 41 | docker push ${REPOTAG}-armv6 42 | 43 | docker manifest rm ${REPOTAG} &>/dev/null || true 44 | docker manifest create \ 45 | ${REPOTAG} \ 46 | --amend ${REPOTAG}-amd64 \ 47 | --amend ${REPOTAG}-armv6 48 | docker manifest push ${REPOTAG} 49 | 50 | docker manifest rm ${LATEST} &>/dev/null || true 51 | docker manifest create \ 52 | ${LATEST} \ 53 | --amend ${REPOTAG}-amd64 \ 54 | --amend ${REPOTAG}-armv6 55 | docker manifest push ${LATEST} 56 | 57 | run-armv6: 58 | docker run -it --rm -p 8080:8080 --volume ${KUBECONFIG}:/kubeconfig --env KUBECONFIG=/kubeconfig ${REGISTRY}${REPO}:${TAG}-armv6 59 | 60 | run-amd64: 61 | docker run -it --rm -p 8080:8080 --volume ${KUBECONFIG}:/kubeconfig --env KUBECONFIG=/kubeconfig ${REGISTRY}${REPO}:${TAG}-amd64 62 | -------------------------------------------------------------------------------- /mqtt_kube/main.py: -------------------------------------------------------------------------------- 1 | import importlib.metadata 2 | import logging 3 | import os 4 | import sys 5 | 6 | import gevent 7 | import gevent.monkey 8 | gevent.monkey.patch_all() 9 | gevent.get_hub().SYSTEM_ERROR = BaseException 10 | 11 | import urllib3 # noqa: E402 pylint: disable=C0413 12 | 13 | import mqtt_kube.binding.binding # noqa: E402 pylint: disable=C0413 14 | import mqtt_kube.config # noqa: E402 pylint: disable=C0413 15 | import mqtt_kube.k8s.api # noqa: E402 pylint: disable=C0413 16 | import mqtt_kube.mqtt # noqa: E402 pylint: disable=C0413 17 | import mqtt_kube.mqttjson # noqa: E402 pylint: disable=C0413 18 | import mqtt_kube.server # noqa: E402 pylint: disable=C0413 19 | 20 | FORMAT = '%(asctime)s.%(msecs)03d %(levelname)s [%(module)s] %(message)s' 21 | logging.basicConfig(format=FORMAT, level=logging.DEBUG, datefmt='%Y-%m-%dT%H:%M:%S') 22 | 23 | logging.getLogger('kubernetes.client.rest').setLevel(logging.INFO) 24 | 25 | urllib3.disable_warnings() 26 | 27 | 28 | def cli(): 29 | meta = dict(importlib.metadata.metadata('mqtt_kube')) 30 | logging.info('Running mqtt-kube v%s', meta['Version']) 31 | 32 | config_file = 'config-dev.yaml' if len(sys.argv) < 2 else sys.argv[1] 33 | config = mqtt_kube.config.Config(config_file) 34 | 35 | logging.getLogger().setLevel(level=config.log_level) 36 | 37 | mqtt = mqtt_kube.mqtt.Mqtt(config.mqtt) 38 | 39 | kubeconfig = os.environ.get('KUBECONFIG', None) 40 | kube_client = mqtt_kube.k8s.api.get_client_api(kubeconfig) 41 | if kube_client is None: 42 | return 1 43 | 44 | binding = mqtt_kube.binding.binding.Binding(kube_client, mqtt, config.bindings) 45 | 46 | server = mqtt_kube.server.Server(**config.http) 47 | 48 | try: 49 | # mqtt.open() 50 | server.open() 51 | mqtt_loop = mqtt.run() 52 | binding.open() 53 | 54 | logging.info('Started') 55 | gevent.joinall((mqtt_loop,)) 56 | except KeyboardInterrupt: 57 | server.close() 58 | mqtt.close() 59 | 60 | return 0 61 | -------------------------------------------------------------------------------- /tests/test_valuemap.py: -------------------------------------------------------------------------------- 1 | from mqtt_kube.binding.valuemap import ValueMap 2 | 3 | class TestTopicMatcher: 4 | def test_miss(self): 5 | v = ValueMap({ 6 | 'map': { 7 | None: 1, 8 | } 9 | }) 10 | assert v.lookup(2) == None 11 | 12 | def test_basic(self): 13 | v = ValueMap({ 14 | 'map': { 15 | None: "BUSY", 16 | 1: "DONE" 17 | } 18 | }) 19 | assert v.lookup(None).value == "BUSY" 20 | assert v.lookup(1).value == "DONE" 21 | 22 | def test_predicate(self): 23 | v = ValueMap({ 24 | 'map': { 25 | "lambda v: v >= 0": "+", 26 | "lambda v: v < 0": "-", 27 | } 28 | }) 29 | assert v.lookup(5).value == "+" 30 | assert v.lookup(-3).value == "-" 31 | 32 | def test_fstring(self): 33 | v = ValueMap({ 34 | 'map': { 35 | "A": "Alpha {input}", 36 | "B": "Bravo {input}", 37 | } 38 | }) 39 | assert v.lookup('A').value == "Alpha A" 40 | assert v.lookup('B').value == "Bravo B" 41 | 42 | def test_regex(self): 43 | v = ValueMap({ 44 | 'map': { 45 | "re:(?P.*) (?P.*)": "{input}: [{first}] ({last})", 46 | } 47 | }) 48 | assert v.lookup('Mr X').value == "Mr X: [Mr] (X)" 49 | 50 | def test_jsonpath(self): 51 | v = ValueMap({ 52 | 'jsonpath': '$.key' 53 | }) 54 | assert v.lookup('{"key": "value"}').value == "value" 55 | 56 | def test_jsonpath_map(self): 57 | v = ValueMap({ 58 | 'jsonpath': '$.action', 59 | 'map': { 60 | 'RUN': 'launch', 61 | } 62 | }) 63 | assert v.lookup('{"action": "RUN"}').value == "launch" 64 | 65 | def test_format(self): 66 | v = ValueMap({ 67 | 'map': { 68 | 'lambda v: True': '{value:.3f}' 69 | } 70 | }) 71 | assert v.lookup(1/3).value == "0.333" 72 | -------------------------------------------------------------------------------- /mqtt_kube/k8s/listener.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import mqtt_kube.k8s.text 4 | import mqtt_kube.k8s.workload 5 | 6 | 7 | class Listener: 8 | def __init__(self, api_client): 9 | logging.debug('API Client: %s', api_client.configuration.host) 10 | self._api_client = api_client 11 | self._delegate_map = {} 12 | 13 | def open(self): 14 | for delegate in self._delegate_map.values(): 15 | delegate.open() 16 | 17 | def _resource_key(self, namespace, resource): 18 | return f'{namespace}:{resource}' 19 | 20 | def _delegate(self, namespace, resource): 21 | key = self._resource_key(namespace, resource) 22 | if key not in self._delegate_map: 23 | self._delegate_map[key] = ListenerDelegate(self._api_client, namespace, resource) 24 | return self._delegate_map[key] 25 | 26 | def patch(self, namespace, resource, name): 27 | return self._delegate(namespace, resource).patch(name) 28 | 29 | def watch(self, namespace, resource, name, on_watch): 30 | return self._delegate(namespace, resource).watch(name, on_watch) 31 | 32 | 33 | class ListenerDelegate: 34 | def __init__(self, api_client, namespace, resource): 35 | resource = mqtt_kube.k8s.text.camel_to_snake(resource) 36 | if resource in ('deployment', 'daemon_set', 'job'): 37 | self._workload = mqtt_kube.k8s.workload.Workload(api_client, namespace, resource) 38 | else: 39 | self._workload = None 40 | logging.error('Unsupported resource type "%s"', resource) 41 | self._name_onwatch_map = {} 42 | 43 | def open(self): 44 | for name, on_watches in self._name_onwatch_map.items(): 45 | if not self._workload.exists(name): 46 | for on_watch in on_watches: 47 | on_watch(None, deleted=True) 48 | 49 | def patch(self, name): 50 | if not self._workload: 51 | logging.warning('Not patching unsupported resource type') 52 | return None 53 | return self._workload.patch(name) 54 | 55 | def watch(self, name, on_watch): 56 | if not self._workload: 57 | logging.warning('Not watching unsupported resource type') 58 | return 59 | if not self._name_onwatch_map: 60 | self._workload.watch(self._on_watch) 61 | if name not in self._name_onwatch_map: 62 | self._name_onwatch_map[name] = [] 63 | self._name_onwatch_map[name].append(on_watch) 64 | 65 | def _on_watch(self, obj, deleted): 66 | try: 67 | for on_watch in self._name_onwatch_map[obj.metadata.name]: 68 | on_watch(obj, deleted=deleted) 69 | except KeyError: 70 | pass 71 | -------------------------------------------------------------------------------- /mqtt_kube/config.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import logging 3 | import os 4 | import random 5 | import sys 6 | 7 | from typing import Any, IO 8 | 9 | import yaml 10 | 11 | # Prevent YAML loader from interpreting 'on', 'off', 'yes', 'no' as bool 12 | from yaml.resolver import Resolver 13 | 14 | for ch in "OoYyNn": 15 | Resolver.yaml_implicit_resolvers[ch] = [x for x in 16 | Resolver.yaml_implicit_resolvers[ch] 17 | if x[0] != 'tag:yaml.org,2002:bool'] 18 | 19 | 20 | class Loader(yaml.Loader): # pylint: disable=R0901 21 | """YAML Loader with custom constructors""" 22 | 23 | def __init__(self, stream: IO) -> None: 24 | try: 25 | self._base = os.path.split(stream.name)[0] 26 | except AttributeError: 27 | self._base = os.path.curdir 28 | 29 | super().__init__(stream) 30 | 31 | @classmethod 32 | def construct_relpath(cls, loader, node: yaml.Node) -> Any: 33 | """Relative path""" 34 | 35 | return os.path.abspath(os.path.join(loader._base, loader.construct_scalar(node))) # pylint: disable=W0212 36 | 37 | 38 | yaml.add_constructor('!relpath', Loader.construct_relpath, Loader) 39 | 40 | 41 | def default(item, key, value): 42 | if key not in item: 43 | item[key] = value 44 | 45 | 46 | class Config: 47 | def __init__(self, config_file): 48 | logging.info('Config file: %s', config_file) 49 | try: 50 | with open(config_file, 'rt', encoding="utf8") as fh: 51 | self._d = yaml.load(fh, Loader=Loader) 52 | except yaml.parser.ParserError: 53 | logging.exception('Loading %s', config_file) 54 | sys.exit(1) 55 | 56 | self._hash = hashlib.md5(str(random.random()).encode('utf-8')).hexdigest() 57 | 58 | self._d['mqtt']['client-id'] += f'-{self._hash[8:]}' 59 | 60 | @property 61 | def log_level(self): 62 | try: 63 | level = self._d['logging']['level'].upper() 64 | return { 65 | 'DEBUG': logging.DEBUG, 66 | 'INFO': logging.INFO, 67 | 'WARNING': logging.WARNING, 68 | 'WARN': logging.WARNING, 69 | 'ERROR': logging.ERROR, 70 | }[level] 71 | except KeyError: 72 | return logging.DEBUG 73 | 74 | @property 75 | def http(self): 76 | return self._d['http'] 77 | 78 | @property 79 | def mqtt(self): 80 | return self._d['mqtt'] 81 | 82 | @property 83 | def bindings(self): 84 | for item in self._d['bindings']: 85 | yield item 86 | 87 | 88 | def plural(items): 89 | if not items: 90 | return 91 | if isinstance(items, list): 92 | yield from items 93 | else: 94 | yield items 95 | -------------------------------------------------------------------------------- /mqtt_kube/binding/binding.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import mqtt_kube.k8s.locus 3 | import mqtt_kube.k8s.listener 4 | 5 | from mqtt_kube.binding.actioner import Actioner 6 | from mqtt_kube.binding.patcher import Patcher 7 | from mqtt_kube.binding.valuemap import ValueMap 8 | from mqtt_kube.binding.watcher import Watcher 9 | from mqtt_kube.config import plural 10 | 11 | 12 | class Binding: 13 | def __init__(self, kube_client, mqtt, bindings): 14 | self._kube_client = kube_client 15 | self._mqtt = mqtt 16 | self._bindings = [] 17 | self._api_listener = mqtt_kube.k8s.listener.Listener(self._kube_client) 18 | self._init(bindings) 19 | 20 | def _init(self, bindings): 21 | for binding in bindings: 22 | try: 23 | for patch in plural(binding.get('patch', None)): 24 | locus = mqtt_kube.k8s.locus.Locus(self._api_listener, binding['namespace'], 25 | binding['resource'], binding['name'], 26 | patch['locus']) 27 | 28 | valuemap = ValueMap(patch.get('values', {})) 29 | patcher = Patcher(locus, 30 | self._mqtt, patch['topic'], valuemap) 31 | self._bindings.append(patcher) 32 | 33 | for watch in plural(binding.get('watch', None)): 34 | locus = mqtt_kube.k8s.locus.Locus(self._api_listener, binding['namespace'], 35 | binding['resource'], binding['name'], 36 | watch['locus']) 37 | valuemap = ValueMap(watch.get('values', {})) 38 | watcher = Watcher(locus, 39 | self._mqtt, watch['topic'], valuemap, 40 | watch.get('retain', False), watch.get('qos', 0)) 41 | self._bindings.append(watcher) 42 | 43 | for action in plural(binding.get('action', None)): 44 | workload = mqtt_kube.k8s.workload.Workload(self._kube_client, binding['namespace'], binding['resource']) 45 | valuemap = ValueMap(action.get('values', {})) 46 | actioner = Actioner(workload, 47 | self._mqtt, action['topic'], valuemap, 48 | binding, action) 49 | self._bindings.append(actioner) 50 | 51 | except (KeyError, ValueError) as ex: 52 | logging.error('Loading binding "%s: %s": %s', ex.__class__.__name__, ex, repr(binding)) 53 | 54 | def open(self): 55 | for binding in self._bindings: 56 | binding.open() 57 | self._api_listener.open() 58 | -------------------------------------------------------------------------------- /mqtt_kube/k8s/locus.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from datetime import datetime, timedelta 4 | from textwrap import shorten 5 | 6 | import jsonpath_ng 7 | 8 | import mqtt_kube.k8s.jsonpathadaptor 9 | import mqtt_kube.k8s.text 10 | import mqtt_kube.k8s.workload 11 | 12 | 13 | class Locus: 14 | def __init__(self, api_listener, namespace, resource, name, jsonpath): 15 | self._api_listener = api_listener 16 | self._namespace = namespace 17 | self._resource = resource 18 | self._name = name 19 | 20 | self._id = self._object_id(namespace, resource, name) 21 | self._last_watch_value = None 22 | self._last_watch = None 23 | 24 | jsonpath = mqtt_kube.k8s.text.camel_to_snake(jsonpath) 25 | try: 26 | self._jsonpath = jsonpath_ng.parse(jsonpath) 27 | except AttributeError: 28 | raise ValueError(f"Invalid JSON Path: '{jsonpath}'") from None 29 | self._on_change = None 30 | 31 | @classmethod 32 | def _object_id(cls, namespace, resource, name): 33 | resource = mqtt_kube.k8s.text.camel_to_snake(resource) 34 | return f'{namespace}:{resource}/{name}' 35 | 36 | def write(self, value): 37 | with self._api_listener.patch(self._namespace, self._resource, self._name) as obj: 38 | if obj is None: 39 | return 40 | self._jsonpath.update(mqtt_kube.k8s.jsonpathadaptor.JsonPathAdaptor(obj), value) 41 | 42 | values = self._jsonpath.find(mqtt_kube.k8s.jsonpathadaptor.JsonPathAdaptor(obj)) 43 | if values: 44 | final_value = [v.value.raw for v in values] 45 | final_value = final_value[0] # Just use first occurrence TODO: improve 46 | 47 | if final_value == value: 48 | logging.debug('Set %s::%s: %s', self._id, self._jsonpath, value) 49 | else: 50 | logging.warning('Not set %s::%s: %s != %s', self._id, self._jsonpath, value, final_value) 51 | else: 52 | logging.warning('Value not at %s::%s %s', self._id, self._jsonpath, value) 53 | 54 | def watch(self, on_change): 55 | self._on_change = on_change 56 | logging.debug('Watch %s::%s', self._id, self._jsonpath) 57 | self._api_listener.watch(self._namespace, self._resource, self._name, self._on_watch) 58 | 59 | def _notify_watch_change(self, payload): 60 | now = datetime.now() 61 | try: 62 | if payload != self._last_watch_value: 63 | return True 64 | if not self._last_watch: 65 | return True 66 | if self._last_watch < (now - timedelta(seconds=10)): 67 | return True 68 | return False 69 | finally: 70 | self._last_watch_value = payload 71 | self._last_watch = now 72 | 73 | def _on_watch(self, obj, deleted): 74 | if deleted is True: 75 | self._on_change(None, deleted=deleted) 76 | return 77 | 78 | object_id = self._object_id(obj.metadata.namespace, obj.kind, obj.metadata.name) 79 | if object_id != self._id: 80 | return 81 | 82 | values = self._jsonpath.find(mqtt_kube.k8s.jsonpathadaptor.JsonPathAdaptor(obj)) 83 | if values: 84 | value = [v.value.raw for v in values] 85 | if len(value) == 1: 86 | value = value[0] 87 | 88 | if self._notify_watch_change(value): 89 | logging.info('Update %s::%s: %s', self._id, self._jsonpath, shorten(str(value), width=30, placeholder="...")) 90 | self._on_change(value, deleted=deleted) 91 | -------------------------------------------------------------------------------- /mqtt_kube/binding/valuemap.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import re 4 | 5 | import jsonpath_ng 6 | 7 | 8 | class ValueMap: 9 | def __init__(self, values): 10 | self._transform = None 11 | if 'transform' in values: 12 | try: 13 | self._transform = eval(values['transform']) # pylint: disable=W0123 14 | except SyntaxError as ex: 15 | logging.warning('Failed to parse transform "%s" because: %s: %s', values['transform'], ex.__class__.__name__, ex) 16 | raise ValueError('Bad Configuration') # pylint: disable=W0707 17 | 18 | self._jsonpath = None 19 | if 'jsonpath' in values: 20 | self._jsonpath = jsonpath_ng.parse(values['jsonpath']) 21 | 22 | self._value_map = [] 23 | if 'map' in values: 24 | for match, value in values['map'].items(): 25 | value = ValueMatcher(match, value) 26 | self._value_map.append(value) 27 | 28 | self._format = values.get('format', None) 29 | 30 | def _value(self, value, ctx): 31 | try: 32 | if isinstance(value, str): 33 | return value.format(**ctx) 34 | return value 35 | except Exception as ex: # pylint: disable=W0703 36 | logging.warning('Failed to format value "%s" because: %s:%s', value, ex.__class__.__name__, ex) 37 | return None 38 | 39 | def lookup(self, needle): 40 | ctx = {'input': needle} 41 | if self._transform: 42 | try: 43 | needle = self._transform(needle) 44 | except Exception as ex: # pylint: disable=W0703 45 | logging.warning('Failed to transform value "%s" because: %s:%s', needle, ex.__class__.__name__, ex) 46 | return None 47 | 48 | if self._jsonpath: 49 | try: 50 | needle = json.loads(needle) 51 | ctx['json'] = needle 52 | except json.JSONDecodeError: 53 | logging.warning('JSON loads failed for {key}') 54 | return None 55 | found = self._jsonpath.find(needle) 56 | if found: 57 | needle = found[0].value # take first value 58 | else: 59 | logging.warning('JSONPath lookup failed with {self._jsonpath} in {key}') 60 | return None 61 | ctx['value'] = needle 62 | if self._value_map: 63 | for value in self._value_map: 64 | ret, m_ctx = value.match(needle) 65 | if m_ctx is None: 66 | continue 67 | ctx.update(m_ctx) 68 | needle = ret 69 | break 70 | else: 71 | return None 72 | return ValueMatch(self._render_value(needle, ctx), ctx) 73 | 74 | @staticmethod 75 | def _render_value(value, ctx): 76 | try: 77 | if isinstance(value, str): 78 | return value.format(**ctx) 79 | return value 80 | except Exception as ex: # pylint: disable=W0703 81 | logging.warning('Failed to format value "%s" because: %s:%s', value, ex.__class__.__name__, ex) 82 | return None 83 | 84 | 85 | class ValueMatcher: 86 | def __init__(self, match, value): 87 | self._match = match 88 | self._value = value 89 | 90 | if isinstance(match, str): 91 | if match.startswith('re:'): 92 | try: 93 | self._match = re.compile(match[3:]) 94 | except re.error as ex: 95 | logging.warning("Ignoring regex match: \"%s\" because %s", match, ex) 96 | raise ValueError(f'Ignoring regex match: "{match}"') from ex 97 | elif match.startswith('lambda '): 98 | try: 99 | self._match = eval(match) # pylint: disable=W0123 100 | except Exception as ex: # pylint: disable=W0703 101 | logging.warning("Ignoring lambda match: \"%s\" because %s", match, ex) 102 | raise ValueError(f'Ignoring lambda match: "{match}" because {ex}') from ex 103 | 104 | def match(self, needle): 105 | 106 | if callable(self._match): 107 | try: 108 | m = self._match(needle) 109 | if m: 110 | return self._value, {} 111 | except Exception: # pylint: disable=W0703 112 | pass 113 | elif isinstance(self._match, re.Pattern): 114 | if not isinstance(needle, str): 115 | return None, None 116 | m = self._match.match(needle) 117 | if m: 118 | return self._value, m.groupdict() 119 | else: 120 | if self._match == needle: 121 | return self._value, {} 122 | return None, None 123 | 124 | 125 | class ValueMatch: 126 | def __init__(self, value, ctx): 127 | self._value = value 128 | self._ctx = ctx 129 | 130 | @property 131 | def value(self): 132 | return self._value 133 | 134 | @property 135 | def context(self): 136 | return self._ctx 137 | -------------------------------------------------------------------------------- /mqtt_kube/k8s/workload.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logging 3 | 4 | from contextlib import contextmanager 5 | 6 | import gevent 7 | 8 | import kubernetes 9 | import kubernetes.config 10 | import kubernetes.client 11 | import kubernetes.stream 12 | import kubernetes.watch 13 | 14 | import mqtt_kube.k8s.text 15 | 16 | 17 | class Workload: 18 | def __init__(self, api_client, namespace, resource): 19 | self._api_client = api_client 20 | self._resource = mqtt_kube.k8s.text.camel_to_snake(resource) 21 | assert self._resource in ('deployment', 'daemon_set', 'job') 22 | self._namespace = namespace 23 | self._o = {} 24 | self._watch_greenlet = None 25 | self._on_watch = None 26 | 27 | def _get_api(self): 28 | if self._resource in ('deployment', 'daemon_set'): 29 | return kubernetes.client.AppsV1Api(self._api_client) 30 | if self._resource in ('job'): 31 | return kubernetes.client.BatchV1Api(self._api_client) 32 | raise NotImplementedError(f'Resource type not supported: {self._resource}') 33 | 34 | def _api_op(self, op): 35 | api = self._get_api() 36 | return getattr(api, f'{op}_{self._resource}') 37 | 38 | def exists(self, name): 39 | try: 40 | self._o[name] = self._api_op('read_namespaced')(name=name, namespace=self._namespace) 41 | logging.debug('{%s:%s} Object exists "%s"', self._namespace, self._resource, name) 42 | return True 43 | except kubernetes.client.exceptions.ApiException as ex: 44 | if ex.reason == 'Not Found' and ex.status == 404: 45 | return False 46 | raise 47 | 48 | def delete(self, name): 49 | try: 50 | propagation_policy = 'Foreground' 51 | self._o[name] = self._api_op('delete_namespaced')(name=name, namespace=self._namespace, 52 | orphan_dependents=None, 53 | propagation_policy=propagation_policy) 54 | logging.info('{%s:%s} Deleted object "%s"', self._namespace, self._resource, name) 55 | except kubernetes.client.exceptions.ApiException as ex: 56 | logging.error('{%s:%s} Deleting object "%s" failed: %s: %s', self._namespace, self._resource, name, ex.__class__.__name__, ex) 57 | 58 | @contextmanager 59 | def patch(self, name): 60 | # if not self._watch_greenlet or not self._o: 61 | # print('## get latest', name) 62 | try: 63 | self._o[name] = self._api_op('read_namespaced')(name=name, namespace=self._namespace) 64 | except kubernetes.client.exceptions.ApiException as ex: 65 | if ex.reason == "Not Found" and ex.status == 404: 66 | logging.info('{%s:%s} Not patching object "%s" because it was not found', self._namespace, self._resource, name) 67 | else: 68 | logging.error('{%s:%s} Fetching object "%s" failed: %s: %s', self._namespace, self._resource, name, ex.__class__.__name__, ex) 69 | yield None 70 | return 71 | 72 | yield self._o[name] 73 | 74 | logging.debug('{%s:%s} Patching object "%s"', self._namespace, self._resource, name) 75 | 76 | try: 77 | self._o[name] = self._api_op('patch_namespaced')(name=name, namespace=self._namespace, body=self._o[name]) 78 | except kubernetes.client.exceptions.ApiException as ex: 79 | logging.error('{%s:%s} Patching object "%s" failed: %s: %s', self._namespace, self._resource, name, ex.__class__.__name__, ex) 80 | 81 | def watch(self, on_watch): 82 | if not self._watch_greenlet: 83 | self._on_watch = on_watch 84 | self._watch_greenlet = gevent.spawn(self._watch_forever) 85 | 86 | def close(self): 87 | if self._watch_greenlet: 88 | self._watch_greenlet.kill() 89 | self._watch_greenlet = None 90 | 91 | def _watch_forever(self): 92 | while True: 93 | try: 94 | self._watch() 95 | except kubernetes.client.exceptions.ApiException as ex: 96 | logging.warning('{%s:%s} Watch failed: %s: %s', self._namespace, self._resource, ex.__class__.__name__, ex) 97 | if ex.reason.startswith('Expired: too old resource version'): 98 | pass 99 | self._o = {} 100 | except Exception: # pylint: disable=W0703 101 | logging.exception('{%s:%s} Watch failed', self._namespace, self._resource,) 102 | time.sleep(30) 103 | 104 | def _watch(self): 105 | w = kubernetes.watch.Watch() 106 | 107 | logging.debug('{%s:%s} Watching ...', self._namespace, self._resource) 108 | for item in w.stream( 109 | self._api_op('list_namespaced'), 110 | namespace=self._namespace, 111 | limit=1, 112 | resource_version=self._resource_version, 113 | timeout_seconds=0, 114 | watch=True 115 | ): 116 | watch_type = item['type'] 117 | 118 | if watch_type == 'ERROR': 119 | logging.error('{%s:%s} Watch ERROR: %s %s (%s): %s', self._namespace, self._resource, item['raw_object']['status'], item['raw_object']['reason'], item['raw_object']['code'], item['raw_object']['message']) 120 | self._o = {} 121 | return 122 | if watch_type in ('ADDED', 'MODIFIED'): 123 | obj = item['object'] 124 | self._o[obj.metadata.name] = obj 125 | logging.debug('{%s:%s} Watch %s %s %s', self._namespace, self._resource, watch_type, obj.metadata.name, obj.metadata.resource_version) 126 | self._on_watch(obj, deleted=False) 127 | elif watch_type in ('DELETED',): 128 | obj = item['object'] 129 | self._o[obj.metadata.name] = None 130 | logging.debug('{%s:%s} Watch %s %s %s', self._namespace, self._resource, watch_type, obj.metadata.name, obj.metadata.resource_version) 131 | self._on_watch(obj, deleted=True) 132 | else: 133 | logging.warning('{%s:%s} Watch %s unhandled', self._namespace, self._resource, watch_type) 134 | 135 | @property 136 | def _resource_version(self): 137 | version = 0 138 | if not self._o: 139 | return 0 140 | for obj in self._o.values(): 141 | if obj is None: 142 | continue 143 | if obj.metadata is None: 144 | continue 145 | version = max(version, int(obj.metadata.resource_version)) 146 | return version 147 | 148 | def _ensure(self, obj, path, value): 149 | try: 150 | for p in path[:-1]: 151 | obj = obj[p] 152 | p = path[-1] 153 | if obj[p] != value: 154 | logging.warning('Forcing value %s=%s to %s', '.'.join(path), obj[p], value) 155 | obj[p] = value 156 | except KeyError: 157 | pass 158 | 159 | def launch_template(self, path, context): 160 | launch_object = mqtt_kube.k8s.template.load_yaml_template(path, context) 161 | if launch_object is None: 162 | return 163 | if mqtt_kube.k8s.text.camel_to_snake(launch_object['kind']) != self._resource: # pylint: disable=E1136 164 | logging.warning('Did not launch object, because resource does not match the binding') 165 | # self._ensure(launch_object, ('kind',), self._resource) 166 | # self._ensure(launch_object, ('metadata', 'name'), self._name) 167 | self._ensure(launch_object, ('metadata', 'namespace'), self._namespace) 168 | try: 169 | kubernetes.utils.create_from_dict(self._api_client, 170 | data=launch_object, 171 | namespace=self._namespace) 172 | logging.info("{%s:%s} Launched '%s'", self._namespace, launch_object['kind'], 173 | launch_object['metadata']['name']) 174 | except kubernetes.utils.FailToCreateError as ex: 175 | logging.error('Failed to launch object: %s', ex) 176 | -------------------------------------------------------------------------------- /mqtt_kube/mqtt.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os.path 3 | import re 4 | import ssl 5 | import sys 6 | 7 | import paho.mqtt.client 8 | import gevent 9 | 10 | 11 | class Mqtt: 12 | def __init__(self, config): 13 | self._subscribe_map = {} 14 | self._c = config 15 | self._topic_prefix = self._c.get('topic-prefix', None) or '' 16 | self._watchers = [] 17 | self._client = None 18 | self.connect_timestamp = None 19 | 20 | def watch_online(self, on_change): 21 | self._watchers.append(on_change) 22 | 23 | def _notify_watchers(self, online): 24 | for watcher in self._watchers: 25 | gevent.spawn(watcher, online) 26 | 27 | def open(self): 28 | logging.info("Open (%s)", self._topic_prefix) 29 | self._client = paho.mqtt.client.Client(client_id=self._c['client-id']) 30 | self._client.start = paho.mqtt.client.time_func() 31 | 32 | self._client.on_connect = self._on_connect 33 | self._client.on_disconnect = self._on_disconnect 34 | self._client.on_message = self._on_message 35 | 36 | auth = self._c.get('auth', None) 37 | if auth: 38 | auth_type = auth.get('type', 'none') 39 | 40 | if auth_type == 'basic': 41 | logging.info('Using basic auth') 42 | username = auth.get('username', None) 43 | password = auth.get('password', None) 44 | self._client.username_pw_set(username, password) 45 | 46 | elif auth_type == 'mtls': 47 | logging.info('Using mTLS auth') 48 | 49 | mtls_context = ssl.create_default_context() 50 | mtls_context.set_alpn_protocols(auth.get("protocols", [])) 51 | 52 | cafile = auth.get("cafile", None) 53 | if cafile: 54 | mtls_context.load_verify_locations(cafile=cafile) 55 | 56 | certfile = auth.get("certfile", None) 57 | keyfile = auth.get("keyfile", None) 58 | keyfile_password = auth.get("keyfile_password", None) 59 | if certfile and keyfile: 60 | mtls_context.load_cert_chain( 61 | certfile=certfile, 62 | keyfile=keyfile, 63 | password=keyfile_password, 64 | ) 65 | 66 | self._client.tls_set_context(mtls_context) 67 | 68 | else: 69 | logging.warning('Ignoring unknown auth type: %s', auth_type) 70 | 71 | connect = { 72 | 'host': self._c.get('host', '127.0.0.1'), 73 | 'port': self._c.get('port', 1833), 74 | 'keepalive': self._c.get('keepalive', 60), 75 | 'bind_address': self._c.get('bind_address', ''), 76 | 'bind_port': self._c.get('bind_port', 0), 77 | 'properties': self._c.get('properties', None), 78 | } 79 | logging.info('Connecting to %s:%s', connect['host'], connect['port']) 80 | self._client.connect(**connect) 81 | 82 | def _on_connect(self, _client, _userdata, flags, rc): 83 | if rc != 0: 84 | logging.error("Bad Connect: %s rc=%s", flags, rc) 85 | return 86 | 87 | logging.info("On Connect: %s", flags) 88 | self.connect_timestamp = paho.mqtt.client.time_func() 89 | 90 | self._notify_watchers(True) 91 | 92 | for matcher in self._subscribe_map: 93 | result, _mid = self._client.subscribe(matcher.topic) 94 | error = paho.mqtt.client.error_string(result) 95 | if error != 'No error.': 96 | logging.warning('Subscribe "%s" with %s', matcher.topic, error) 97 | else: 98 | logging.debug('Subscribe "%s"', matcher.topic) 99 | 100 | def run(self): 101 | return gevent.spawn(self._run_loop) 102 | 103 | def _run_loop(self): 104 | while True: 105 | try: 106 | self.open() 107 | self._client.loop_forever() 108 | except Exception as ex: # pylint: disable=W0703 109 | logging.error('Run loop exception %s: %s', ex.__class__.__name__, ex) 110 | gevent.sleep(10) 111 | 112 | def close(self): 113 | logging.info("Close") 114 | if self._client: 115 | client = self._client 116 | self._client = None 117 | client.disconnect() 118 | 119 | def _on_disconnect(self, _client, _userdata, _rc): 120 | logging.info("On Disconnect") 121 | self._notify_watchers(False) 122 | 123 | if self._client: 124 | self.close() 125 | self.open() 126 | 127 | def _on_message(self, _client, _userdata, message): 128 | if self._topic_prefix and not message.topic.startswith(self._topic_prefix): 129 | logging.warning('Dropped message: %s', message) 130 | return 131 | 132 | try: 133 | payload = message.payload.decode() 134 | retained = ' (retained)' if message.retain else '' 135 | logging.debug("Received %s: %s%s", message.topic, payload, retained) 136 | for listener in self._iter_matching_listeners(message.topic): 137 | try: 138 | listener(payload, message.timestamp) 139 | except Exception: # pylint: disable=W0703 140 | logging.exception('Handling MQTT message') 141 | sys.exit(1) 142 | except KeyError: 143 | logging.exception('_on_message') 144 | 145 | def _topic(self, *p): 146 | if not p[0]: 147 | return None 148 | return os.path.join(self._topic_prefix, *p) 149 | 150 | def _iter_matching_listeners(self, topic): 151 | for matcher, listeners in self._subscribe_map.items(): 152 | if matcher.match(topic): 153 | yield from listeners 154 | 155 | def subscribe(self, topic, on_payload): 156 | topic = self._topic(topic) 157 | matcher = TopicMatcher(topic) 158 | if topic not in self._subscribe_map: 159 | self._subscribe_map[matcher] = [] 160 | self._subscribe_map[matcher].append(on_payload) 161 | 162 | def publish(self, topic, payload, retain=False, qos=0): 163 | if not topic: 164 | logging.warning('Ignoring empty topic') 165 | return 166 | topic = self._topic(topic) 167 | logging.debug("Publish %s: %s", topic, payload) 168 | try: 169 | if not isinstance(payload, (str, bytearray, int, float, type(None))): 170 | payload = str(payload) 171 | self._client.publish(topic, payload=payload, qos=qos, retain=retain) 172 | except TypeError as ex: 173 | logging.error('%s Value "%s" is a %s', ex, payload, type(payload)) 174 | 175 | 176 | class TopicMatcher: 177 | plus_pattern = r'(?:^|(?<=/))\+(?:$|(?=/))' # find all `+` symbols like `+/...`, `.../+/...`, `.../+` 178 | hash_pattern = r'(?:^|(?<=/))#$' # find all `#` symbols like `#`, `.../#` 179 | 180 | def __init__(self, topic): 181 | self._topic = topic 182 | self._re = self._topic_to_regex(topic) 183 | 184 | def __hash__(self): 185 | return hash(self._topic) 186 | 187 | @property 188 | def topic(self): 189 | return self._topic 190 | 191 | def match(self, topic): 192 | if self._re: 193 | try: 194 | return self._re.match(topic) is not None 195 | except re.error as ex: 196 | logging.error('Matching topic: %s because %s: %s', topic, ex.__class__.__name__, ex) 197 | return self._topic == topic 198 | 199 | @classmethod 200 | def _topic_to_regex(cls, topic): 201 | re_topic = topic 202 | try: 203 | if re.search(cls.plus_pattern, topic): 204 | re_topic = re.sub(cls.plus_pattern, '[^/]+', re_topic) 205 | if re.search(cls.hash_pattern, topic): 206 | re_topic = re.sub(cls.hash_pattern, '.+$', re_topic) 207 | if '#' in re_topic: 208 | logging.warning("Bad MQTT topic pattern, unexpected '#': %s", topic) 209 | return None 210 | if re_topic != topic: 211 | return re.compile(re_topic) 212 | except re.error: 213 | logging.warning('Bad MQTT topic pattern, could not parse: %s', topic) 214 | return None 215 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | MQTT Kube 2 | === 3 | 4 | ***Kubernetes API to MQTT connector service*** 5 | 6 | A service that maps MQTT topics to Kubernetes objects, configurable via YAML. 7 | 8 | This capability presents many possible use cases. An example is to switch a media server on and off as though it were an appliance or launch a Job workload based on some demand. 9 | 10 | - [Installation](#installation) 11 | - [Docker](#docker) 12 | - [Kubernetes](#kubernetes) 13 | - [MQTT Infrastructure](#mqtt-infrastructure) 14 | - [Configuration](#configuration) 15 | - [MQTT](#mqtt) 16 | - [MQTT - Basic Auth](#mqtt---basic-auth) 17 | - [MQTT - mTLS Auth](#mqtt---mtls-auth) 18 | - [Web Server](#web-server) 19 | - [Logging](#logging) 20 | - [Bindings](#bindings) 21 | - [MQTT Associations](#mqtt-associations) 22 | - [Watch](#watch) 23 | - [Patch](#patch) 24 | - [Action](#action) 25 | - [Action Templates](#action-templates) 26 | - [Value Mapping](#value-mapping) 27 | - [Examples](#examples) 28 | - [Deployment as an On/Off Appliance](#deployment-as-an-onoff-appliance) 29 | - [Launch/delete a Job](#launchdelete-a-job) 30 | - [Contribution](#contribution) 31 | - [Development](#development) 32 | - [License](#license) 33 | 34 | # Installation 35 | Prebuilt container images are available on [Docker Hub](https://hub.docker.com/r/sourcesimian/mqtt-kube). 36 | ## Docker 37 | Run 38 | ``` 39 | docker run -n mqtt-kube -d -it --rm -p 8080:8080 \ 40 | --volume my-config.yaml:/config.yaml:ro \ 41 | --volume $HOME/.kube/config:/kubeconfig:ro \ 42 | --env KUBECONFIG=/kubeconfig \ 43 | sourcesimian/mqtt-kube:latest 44 | ``` 45 | 46 | ## Kubernetes 47 | When running **mqtt-kube** on the same Kubernetes cluster that you wish to interact with **mqtt-kube** can make use of [Service Accounts](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/). Typically this would be configured as: 48 | 49 | ``` 50 | apiVersion: rbac.authorization.k8s.io/v1 51 | kind: ClusterRoleBinding 52 | metadata: 53 | name: iot-mqtt-kube 54 | roleRef: 55 | apiGroup: rbac.authorization.k8s.io 56 | kind: ClusterRole 57 | name: cluster-admin 58 | subjects: 59 | - kind: ServiceAccount 60 | name: mqtt-kube 61 | namespace: iot 62 | ``` 63 | 64 | ``` 65 | apiVersion: v1 66 | kind: ServiceAccount 67 | metadata: 68 | name: mqtt-kube 69 | namespace: iot 70 | ``` 71 | 72 | And a typical Deployment would look something like: 73 | ``` 74 | apiVersion: apps/v1 75 | kind: Deployment 76 | metadata: 77 | name: mqtt-kube 78 | namespace: iot 79 | spec: 80 | template: 81 | spec: 82 | serviceAccountName: mqtt-kube 83 | volumes: 84 | - name: config 85 | configMap: 86 | name: mqtt-kube-config 87 | containers: 88 | - name: mqtt-kube 89 | image: sourcesimian/mqtt-kube:latest 90 | command: ["/usr/local/bin/mqtt-kube"] 91 | args: ["/config/config.yaml"] 92 | volumeMounts: 93 | - name: config 94 | mountPath: /config 95 | livenessProbe: 96 | initialDelaySeconds: 30 97 | periodSeconds: 30 98 | httpGet: 99 | path: /api/health 100 | port: 8080 101 | ``` 102 | 103 | ## MQTT Infrastructure 104 | An installation of **mqtt-kube** will need a MQTT broker to connect to. There are many possibilities available. [Eclipse Mosquitto](https://github.com/eclipse/mosquitto/blob/master/README.md) is a great self hosted option with many ways of installation including pre-built containers on [Docker Hub](https://hub.docker.com/_/eclipse-mosquitto). 105 | 106 | To compliment your MQTT infrastructure you may consider the following other microservices: 107 | | Service | Description | 108 | |---|---| 109 | | [mqtt-panel](https://github.com/sourcesimian/mqtt-panel/blob/main/README.md) | A self hostable service that connects to a MQTT broker and serves a progressive web app panel. | 110 | | [mqtt-ical](https://github.com/sourcesimian/mqtt-ical/blob/main/README.md) | Publishes values to MQTT topics based on events in an iCal Calendar. | 111 | | [NodeRED](https://nodered.org/) | A flow-based visual programming tool for wiring together devices, with built in MQTT integration and many others available. Can easily be used to add higher level behaviours. | 112 | 113 | # Configuration 114 | **mqtt-kube** consumes a [YAML](https://yaml.org/) file. To start off you can copy [config-basic.yaml](./config-basic.yaml) 115 | 116 | ## MQTT 117 | ``` 118 | mqtt: 119 | host: # optional: MQTT broker host, default: 127.0.0.1 120 | port: # optional: MQTT broker port, default 1883 121 | client-id: mqtt-gpio # MQTT client identifier, often brokers require this to be unique 122 | topic-prefix: # optional: Scopes the MQTT topic prefix 123 | auth: # optional: Defines the authentication used to connect to the MQTT broker 124 | type: # Auth type: none|basic|mtls, default: none 125 | ... ( specific options) 126 | ``` 127 | 128 | ### MQTT - Basic Auth 129 | ``` 130 | type: basic 131 | username: # MQTT broker username 132 | password: # MQTT broker password 133 | ``` 134 | 135 | ### MQTT - mTLS Auth 136 | ``` 137 | type: mtls 138 | cafile: # CA file used to verify the server 139 | certfile: # Certificate presented by this client 140 | keyfile: # Private key presented by this client 141 | keyfile_password: # optional: Password used to decrypt the `keyfile` 142 | protocols: 143 | - # optional: list of ALPN protocols to add to the SSL connection 144 | ``` 145 | 146 | ## Web Server 147 | ``` 148 | http: 149 | bind: # optional: Interface on which web server will listen, default 0.0.0.0 150 | port: # Port on which web server will listen, default 8080 151 | max-connections: # optional: Limit the number of concurrent connections, default 100 152 | ``` 153 | 154 | The web server exposes the following API: 155 | * `/api/health` - Responds with 200 if service is healthy 156 | 157 | ## Logging 158 | ``` 159 | logging: 160 | level: INFO # optional: Logging level, default DEBUG 161 | ``` 162 | 163 | ## Bindings 164 | A binding is a functional element which is used to connect a Kubernetes object to one or more MQTT topics and payloads. 165 | 166 | Bindings are defined under the `bindings` key: 167 | ``` 168 | bindings: 169 | - ... 170 | ``` 171 | All bindings have the following form: 172 | ``` 173 | - namespace: # The Kubernetes namespace in which the object resides 174 | resource: # The Kubernetes object type: Deployment|DaemonSet|Job 175 | name: # The name of the Kubernetes object 176 | (MQTT associations ...) 177 | ``` 178 | 179 | ### MQTT Associations 180 | Bindings are associated with MQTT topics by adding further sections: 181 | #### Watch 182 | A watch association allows you to define a `locus` within the object to watch. On change the value will be published to the `topic`. 183 | ``` 184 | watch: 185 | - locus: # Value location in Kubernetes object 186 | topic: # MQTT topic where payload is published 187 | qos: [0 | 1 | 2] # optional: MQTT QoS to use, default: 1 188 | retain: [False | True] # optional: Publish with MQTT retain flag, default: False 189 | values: # optional: Transform the value published to MQTT 190 | map: # Key to value mapping. Kubernetes to MQTT 191 | : 192 | ... 193 | ... 194 | ``` 195 | [JSON Path](https://goessner.net/articles/JsonPath/) is a way of referencing locations within a JSON document. 196 | 197 | This example will watch the `availableReplicas` in our object. If the value is `1` then `ON` will be published to the topic, and if the value is `None` then `OFF` will be published to the topic. 198 | 199 | ``` 200 | watch: 201 | - locus: "status.availableReplicas" 202 | topic: jellyfin/power/state 203 | qos: 1 204 | retain: true 205 | values: 206 | 1: "ON" 207 | ~: "OFF" 208 | ``` 209 | 210 | In YAML the '`~`' character is used to represent `None`. 211 | 212 | The values map can be extended to support transforms, regular expressions and lambda tests functions. See [Value Mapping](#value-mapping) for more detail. 213 | 214 | 215 | #### Patch 216 | A patch association allows you to listen on a MQTT topic for certain values and then patch a `locus` within the Kubernetes object. 217 | ``` 218 | patch: 219 | - topic: # MQTT topic where payload is published 220 | locus: # Value location in Kubernetes object 221 | values: # Mapping of MQTT payloads to object values 222 | map: 223 | : # Key to value mapping: MATT to Kubernetes 224 | ... 225 | ... 226 | ``` 227 | 228 | This example will patch the `replicas` of the kubernetes object to `1` when `ON` is recieved on the topic, and to `0` when `OFF` is received on the topic. 229 | ``` 230 | patch: 231 | - topic: jellyfin/power/set 232 | locus: "spec.replicas" 233 | values: 234 | map: 235 | "ON": 1 236 | "OFF": 0 237 | ``` 238 | 239 | #### Action 240 | An action association allows you to listen on a MQTT topic for certain values and then launch or delete a Kubernetes object. 241 | ``` 242 | action: 243 | - topic: # MQTT topic where payload is published 244 | launch: # YAML template file describing the object 245 | values: # Mapping of MQTT payloads to actions 246 | map: 247 | "RUN": "launch" 248 | "STOP": "delete" 249 | ... 250 | ``` 251 | 252 | This example will create a Kubernetes Job when `RUN` is recieved on the topic, and delete the Job when `STOP` is received. 253 | ``` 254 | action: 255 | - topic: webhook/cmd 256 | launch: !relpath job-foo.yaml 257 | values: 258 | map: 259 | "RUN": "launch" 260 | "STOP": "create" 261 | ``` 262 | ##### Action Templates 263 | The template file for the launch object should be in the usual Kubernetes YAML format. Values can be templated using [Mako template](https://docs.makotemplates.org/en/latest/syntax.html) syntax. The base keys included in the template context are: 264 | | Key | Description | 265 | |---|---| 266 | | `binding` | The config tree for the binding | 267 | | `action` | The config tree for the action | 268 | | `uid` | A unique 8 character string that can be used to compose Kubernetes identifiers | 269 | | `match` | The context from the value matching process. If JSONPath was used there will be a `.json` sub-key containing the full blob from the pauyload. If a regular expression was used with named groups they will be present as sub keys. The original input value as `.input` | 270 | 271 | ### Value Mapping 272 | Value mapping is applicable to all MQTT associations. In each case there is a direction. Watches map from a value within a Kubernetes object to a MQTT payload. Patches and Actions map from a MQTT payload to a value or action in Kubernetes. 273 | 274 | ``` 275 | values: 276 | transform: # Modify the inbound value 277 | jsonpath: # Select a a value from a Json blob 278 | map: 279 | : # Translate the value to its final form 280 | ... 281 | ``` 282 | 283 | The values map `` expressions can take on the following forms: 284 | | Form | Description | 285 | |---|---| 286 | | Literal Value | An input expression can simply be a literal string, integer of float that is matched with the source value. Or be used as the output value. | 287 | | Regular Expressions | When the input expression is prefixed with `re:` then the remainder of the string is used as a regular expression match. If groups are specified they will be available by name in the [Format String](#format-string) context. | 288 | | Lambda Expression | When the input expression is prefixed with `lambda ` then this will be used as a unary match function. | 289 | | Format String | When the output expression contains Python [f-string](https://docs.python.org/3/reference/lexical_analysis.html#f-strings) replacement fields it will be rendered using the context from the matching process. | 290 | 291 | # Examples 292 | ## Deployment as an On/Off Appliance 293 | An existing deployment of a media server (in this case Jellyfin) can be controlled as though it were an appliance by mapping `status.availableReplicas` and `spec.replicas` to and from topics, as follows: 294 | 295 | ``` 296 | - namespace: media 297 | resource: Deployment 298 | name: jellyfin 299 | 300 | patch: 301 | - topic: jellyfin/power/set 302 | locus: "spec.replicas" 303 | values: 304 | map: 305 | "ON": 1 306 | "OFF": 0 307 | 308 | watch: 309 | - locus: "status.availableReplicas" 310 | topic: jellyfin/power/state 311 | retain: true 312 | qos: 1 313 | values: 314 | map: 315 | 1: "ON" 316 | ~: "OFF" 317 | ``` 318 | The above would translate into the following MQTT topics: 319 | * `jellyfin/power/set` that accepts values of `ON` or `OFF`, 320 | * `jellyfin/power/state` that publishes values of `ON` or `OFF`. 321 | 322 | ## Launch/delete a Job 323 | A Job or other workload can be launched or deleted, as follows: 324 | ``` 325 | - namespace: sync 326 | resource: Job 327 | name: syncer 328 | 329 | action: 330 | - topic: syncer/cmd 331 | launch: !relpath syncer-job.yaml 332 | values: 333 | map: 334 | "RUN": "launch" 335 | "STOP": "delete" 336 | 337 | watch: 338 | - locus: "$.status" 339 | topic: syncer/status 340 | ``` 341 | The above would translate into the following MQTT topics: 342 | * `syncer/cmd` that accepts values of `RUN` or `STOP`, 343 | * `syncer/status` that publishes the Job status. 344 | 345 | Additional watches can be added to provide more targeted status, e.g. see [config-basic.yaml](./config-basic.yaml) 346 | 347 | 348 | # Contribution 349 | Yes sure! And please. I built **mqtt-kube** as a building block in my MQTT centric home automation. I want it to be a project that is quick and easy to use and makes DIY home automation more fun and interesting. 350 | 351 | Before pushing a PR please ensure that `make check` and `make test` are clean and please consider adding unit tests. 352 | 353 | ## Development 354 | Setup the virtualenv: 355 | 356 | ``` 357 | python3 -m venv virtualenv 358 | . ./virtualenv/bin/activate 359 | python3 ./setup.py develop 360 | ``` 361 | 362 | Run the service: 363 | ``` 364 | export KUBECONFIG=$HOME/.kube/config 365 | mqtt-kube ./config-demo.yaml 366 | ``` 367 | 368 | # License 369 | 370 | In the spirit of the Hackers of the [Tech Model Railroad Club](https://en.wikipedia.org/wiki/Tech_Model_Railroad_Club) from the [Massachusetts Institute of Technology](https://en.wikipedia.org/wiki/Massachusetts_Institute_of_Technology), who gave us all so very much to play with. The license is [MIT](LICENSE). 371 | --------------------------------------------------------------------------------