├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── bin └── n2kparser ├── conf.json ├── main.py ├── n2kparser ├── __init__.py └── n2kparser.py ├── setup.cfg └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/python 2 | # Edit at https://www.gitignore.io/?templates=python 3 | 4 | ### Python ### 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | pip-wheel-metadata/ 28 | share/python-wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | MANIFEST 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .nox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | .python-version 88 | 89 | # pipenv 90 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 91 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 92 | # having no cross-platform support, pipenv may install dependencies that don’t work, or not 93 | # install all needed dependencies. 94 | #Pipfile.lock 95 | 96 | # celery beat schedule file 97 | celerybeat-schedule 98 | 99 | # SageMath parsed files 100 | *.sage.py 101 | 102 | # Environments 103 | .env 104 | .venv 105 | env/ 106 | venv/ 107 | ENV/ 108 | env.bak/ 109 | venv.bak/ 110 | 111 | # Spyder project settings 112 | .spyderproject 113 | .spyproject 114 | 115 | # Rope project settings 116 | .ropeproject 117 | 118 | # mkdocs documentation 119 | /site 120 | 121 | # mypy 122 | .mypy_cache/ 123 | .dmypy.json 124 | dmypy.json 125 | 126 | # Pyre type checker 127 | .pyre/ 128 | 129 | # VS-Code Editor Setting 130 | .vscode/ 131 | 132 | # End of https://www.gitignore.io/api/python -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 iotfablab, BIBA - Bremer Institut für Produktion und Logistik GmbH 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include conf.json -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # n2kparser 2 | Python3.x CLI to parser incoming JSON data from `actisense-serial` and `analyzer` binaries from [CANBOAT](https://github.com/canboat/canboat) and store 3 | values into InfluxDB using UDP 4 | 5 | ## Installation and Dependencies 6 | ### Installation 7 | 8 | Install the `CANBOAT` repository binaries in order to use `actisense-serial` and `analyzer` binaries. 9 | 10 | Clone the repository and then install the code: 11 | 12 | pip install . 13 | 14 | ### Development 15 | 16 | develop using `venv`: 17 | 18 | python -m venv venv 19 | 20 | activate the virtual environment and then 21 | 22 | pip install -e . 23 | 24 | 25 | ## Usage 26 | Path to the `conf.json` (see File in repository for Structure) can be set via argument `--config` 27 | 28 | ### Configuration File 29 | 30 | $ n2kparser --config ./conf.json 31 | 32 | The PGNs are configurable via the `conf.json` file in the repository. Follow the structure mentioned in the file. 33 | 34 | A snippet of the PGN is as follows: 35 | 36 | ```json 37 | "pgnConfigs": { 38 | "130311": { 39 | "for": "Environmental Parameters", 40 | "fieldLabels": [ 41 | "Temperature", 42 | "Atmospheric Pressure" 43 | ], 44 | "topics": [ 45 | "environment/nmea2k/temperature", 46 | "environment/nmea2k/pressure" 47 | ] 48 | }, 49 | "127250": { 50 | "for": "Vessel Heading", 51 | "fieldLabels": [ 52 | "Heading" 53 | ], 54 | "topics": [ 55 | "control/nmea2k/heading" 56 | ] 57 | }, 58 | "127501": { 59 | "for": "Binary Switch Bank", 60 | "fromSource": 1, 61 | "fieldLabels": [ 62 | "Indicator1", 63 | "Indicator2" 64 | ], 65 | "topics": [ 66 | "input/nmea2k/switchbank" 67 | ] 68 | } 69 | } 70 | ``` 71 | __NOTE__: A single PGN can measure a different values 72 | * `for` key is for Human-Readable Description for the PGN. (Optional) 73 | * `fieldLabels` key is an array of all the relevant keys for which the values should be saved 74 | to InfluxDB. Example, for Rudder (same PGN) one can choose to store only `Position` value or 75 | both `Direction Order` and `Position` (Required) 76 | * `fromSource` keys is a filter key to store information from only distinct source (e.g. Engine, Rudder). 77 | If there are two Engines/Rudders and the value of Engine/Rudder 1 is to be stored then use `fromSource: 1`. 78 | (Optional) 79 | * `topics`: A list of topics that are combined with `deviceID` and published in accordance with the `fieldLabels` (Required) 80 | 81 | 82 | ### Topics 83 | 84 | /// 85 | 86 | the payload is in the form of line protocol strings for each `topic` 87 | 88 | ## Maintainer 89 | 90 | * Shantanoo Desai (des@biba.uni-bremen.de) -------------------------------------------------------------------------------- /bin/n2kparser: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import n2kparser 4 | 5 | if __name__ == "__main__": 6 | n2kparser.main() 7 | -------------------------------------------------------------------------------- /conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "deviceID": "UMG/0001", 3 | "influx": { 4 | "host": "localhost", 5 | "port": 8086 6 | }, 7 | "mqtt": { 8 | "broker": "localhost", 9 | "port": 1883 10 | }, 11 | "n2k": { 12 | "port": "/dev/ttyS10", 13 | "baudrate": 115200, 14 | "udp_port": 8095, 15 | "pgnConfigs": { 16 | "130311": { 17 | "for": "Environmental Parameters", 18 | "fieldLabels": [ 19 | "Temperature", 20 | "Atmospheric Pressure" 21 | ], 22 | "topics": [ 23 | "environment/nmea2k/temperature", 24 | "environment/nmea2k/pressure" 25 | ] 26 | }, 27 | "127250": { 28 | "for": "Vessel Heading", 29 | "fieldLabels": [ 30 | "Heading" 31 | ], 32 | "topics": [ 33 | "control/nmea2k/heading" 34 | ] 35 | }, 36 | "127251": { 37 | "for": "Rate of Turn", 38 | "fieldLabels": [ 39 | "control/nmea2k/rateOfTurn" 40 | ] 41 | }, 42 | "129025": { 43 | "for": "GPS Navigation", 44 | "fieldLabels": [ 45 | "Latitude", 46 | "Longitude" 47 | ], 48 | "topics": [ 49 | "location/nmea2k/latitude", 50 | "location/nmea2k/longitude" 51 | ] 52 | }, 53 | "127245": { 54 | "for": "Rudder", 55 | "fieldLabels": [ 56 | "Direction Order", 57 | "Position" 58 | ], 59 | "topics": [ 60 | "control/nmea2k/rudder" 61 | ] 62 | }, 63 | "127501": { 64 | "for": "Binary Switch Bank", 65 | "fromSource": 1, 66 | "fieldLabels": [ 67 | "Indicator1", 68 | "Indicator2" 69 | ], 70 | "topics": [ 71 | "input/nmea2k/switchbank" 72 | ] 73 | }, 74 | "127488": { 75 | "for": "Engine Speed", 76 | "fieldLabel": [ 77 | "Engine Speed" 78 | ], 79 | "topics": [ 80 | "engine/nmea2k/speed" 81 | ] 82 | }, 83 | "127489": { 84 | "for": "Engine Parameters", 85 | "fieldLabels": [ 86 | "Engine oil pressure", 87 | "Engine temp.", 88 | "Fuel rate" 89 | ], 90 | "topics": [ 91 | "engine/nmea2k/oilPressure", 92 | "engine/nmea2k/temperature", 93 | "engine/nmea2k/fuelRate" 94 | ] 95 | }, 96 | "129026": { 97 | "for": "SOG/COG", 98 | "fieldLabels": [ 99 | "SOG", 100 | "COG" 101 | ], 102 | "topics": [ 103 | "groundVelocity/nmea2k/sog", 104 | "groundVelocity/nmea2k/cog" 105 | ] 106 | }, 107 | "127257": { 108 | "for": "Attitude(Yaw,Pitch,Roll)", 109 | "fieldLabels": [ 110 | "Yaw", 111 | "Pitch", 112 | "Roll" 113 | ], 114 | "topics": [ 115 | "orientation/nmea2k/yaw", 116 | "orientation/nmea2k/pitch", 117 | "orientation/nmea2k/roll" 118 | ] 119 | } 120 | } 121 | } 122 | } -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from n2kparser.n2kparser import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /n2kparser/__init__.py: -------------------------------------------------------------------------------- 1 | from .n2kparser import main -------------------------------------------------------------------------------- /n2kparser/n2kparser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from subprocess import Popen, PIPE 4 | import sys 5 | import os 6 | import json 7 | import concurrent.futures 8 | import time 9 | import logging 10 | import argparse 11 | 12 | from influxdb import InfluxDBClient, line_protocol 13 | import paho.mqtt.publish as publish 14 | 15 | logging.basicConfig(level=logging.DEBUG) 16 | logger = logging.getLogger(__name__) 17 | 18 | handler = logging.FileHandler("/var/log/n2kparser.log") 19 | handler.setLevel(logging.ERROR) 20 | 21 | formatter = logging.Formatter('%(asctime)s-%(name)s-%(message)s') 22 | handler.setFormatter(formatter) 23 | logger.addHandler(handler) 24 | 25 | DEVICE = '' 26 | client = None 27 | n2k_conf = dict() 28 | mqtt_conf = dict() 29 | analyzer_process = None 30 | 31 | 32 | def save_to_db(measurments): 33 | """ Save NMEA2000 values to InfluxDB""" 34 | global client 35 | try: 36 | client.send_packet(measurments) 37 | return True 38 | except Exception as e: 39 | logger.error('Error while UDP sending: {}'.format(e)) 40 | return False 41 | 42 | 43 | def publish_data(pgn_value, lineprotocol_data): 44 | lp_array = lineprotocol_data.split('\n') 45 | lp_array.pop() # remove last newline 46 | publish_msgs = [] 47 | global n2k_conf 48 | global mqtt_conf 49 | for topic in n2k_conf['pgnConfigs'][str(pgn_value)]['topics']: 50 | mqtt_msg = { 51 | 'topic': DEVICE + '/' + topic, 52 | 'payload': lp_array[n2k_conf['pgnConfigs'][str(pgn_value)]['topics'].index(topic)], 53 | 'qos': 1, 54 | 'retain': False 55 | } 56 | publish_msgs.append(mqtt_msg) 57 | logger.debug(publish_msgs) 58 | try: 59 | publish.multiple( 60 | publish_msgs, 61 | hostname=mqtt_conf['broker'], 62 | port=mqtt_conf['port'] 63 | ) 64 | return True 65 | except Exception as e: 66 | logger.error('Publish Error: {}'.format(e)) 67 | return False 68 | 69 | 70 | def read_nmea2k(): 71 | """Read the actisense-serial -r {device} | analyzer -json for given NMEA2000 NGT-1 device port""" 72 | # Actisense-Serial 73 | actisense_process = Popen(['actisense-serial', '-r', n2k_conf['port']], stdout=PIPE) 74 | 75 | # Analyzer Stream for output in JSON 76 | global analyzer_process 77 | analyzer_process = Popen(['analyzer', '-json'], stdin=actisense_process.stdout, stdout=PIPE, stderr=PIPE) 78 | PGNs = list(map(int, n2k_conf['pgnConfigs'].keys())) 79 | logger.debug('PGNs: {}'.format(PGNs)) 80 | 81 | while True: 82 | incoming_json = analyzer_process.stdout.readline().decode('utf-8') 83 | try: 84 | incoming_data = json.loads(incoming_json) 85 | 86 | if incoming_data['pgn'] in PGNs: 87 | # remove unnecessary keys 88 | del incoming_data['dst'] 89 | del incoming_data['prio'] 90 | 91 | # check if the configuration for the PGN has the `fromSource` Key 92 | if 'fromSource' in list(n2k_conf['pgnConfigs'][str(incoming_data['pgn'])].keys()): 93 | logger.info('PGN Source Filter Check') 94 | if incoming_data['src'] != n2k_conf['pgnConfigs'][str(incoming_data['pgn'])]['fromSource']: 95 | logger.info('PGN: {} with src: {}'.format(incoming_data['pgn'], incoming_data['src'])) 96 | logger.info('Skipping data for: {}'.format(incoming_data['description'])) 97 | continue 98 | 99 | measurement = { 100 | "tags": { 101 | "source": "nmea2k", 102 | "PGN": incoming_data['pgn'], 103 | "src": incoming_data['src'] 104 | }, 105 | "points": [] 106 | } 107 | 108 | # Create a set of all available fields from the incoming frame 109 | incoming_fields = set(incoming_data['fields'].keys()) 110 | fields_from_conf = set(n2k_conf['pgnConfigs'][str(incoming_data['pgn'])]['fieldLabels']) 111 | logger.debug('Fields To Log: {f}'.format(f=fields_from_conf.intersection(incoming_fields))) 112 | 113 | # Get all the Fields necessary to be stored into InfluxDB 114 | for selected_field in fields_from_conf.intersection(incoming_fields): 115 | # Measurement name is the profile type name e.g. control/environment/engine etc available 116 | # as the first level for the mqtt topics 117 | meas_name = n2k_conf['pgnConfigs'][str(incoming_data['pgn'])]['topics'][0].split('/')[0] 118 | point = { 119 | "measurement": meas_name, 120 | "time": int(time.time() * 1e9), 121 | "fields": {} 122 | } 123 | point['fields'][selected_field.replace(" ", "")] = incoming_data['fields'][selected_field] 124 | measurement['points'].append(point) 125 | # logger.debug(line_protocol.make_lines(measurement)) 126 | 127 | with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: 128 | if executor.submit(save_to_db, measurement).result(): 129 | logger.info('saved to InfluxDB') 130 | if executor.submit(publish_data, incoming_data['pgn'], line_protocol.make_lines(measurement)).result(): 131 | logger.info('Published data successfully') 132 | time.sleep(0.05) 133 | 134 | except Exception as e: 135 | logger.exception(e) 136 | 137 | 138 | def file_path(path_to_file): 139 | """Check if Path and File exist for Configuration""" 140 | 141 | if os.path.exists(path_to_file): 142 | if os.path.isfile(path_to_file): 143 | logger.info('File Exists') 144 | with open(path_to_file) as conf_file: 145 | return json.load(conf_file) 146 | else: 147 | logger.error('Configuration File Not Found') 148 | raise FileNotFoundError(path_to_file) 149 | else: 150 | logger.error('Path to Configuration File Not Found') 151 | raise NotADirectoryError(path_to_file) 152 | 153 | 154 | def parse_args(): 155 | """Parse Arguments for configuration file""" 156 | 157 | parser = argparse.ArgumentParser(description='CLI to store Actisense-NGT Gateway values to InfluxDB and publish via MQTT') 158 | parser.add_argument('--config', type=str, required=True, help='Provide the configuration conf.json file with path') 159 | return parser.parse_args() 160 | 161 | 162 | def main(): 163 | """Step ups for Clients and Configuration """ 164 | args = parse_args() 165 | 166 | CONF = file_path(args.config) 167 | global DEVICE 168 | DEVICE = CONF['deviceID'] 169 | global n2k_conf 170 | n2k_conf = CONF['n2k'] 171 | influx_conf = CONF['influx'] 172 | 173 | global mqtt_conf 174 | mqtt_conf = CONF['mqtt'] 175 | 176 | logger.info('Creating InfluxDB client') 177 | logger.debug('Client for {h}@{p} with udp:{ud}'.format(h=influx_conf['host'], p=influx_conf['port'], ud=n2k_conf['udp_port'])) 178 | global client 179 | client = InfluxDBClient( 180 | host=influx_conf['host'], 181 | port=influx_conf['port'], 182 | use_udp=True, 183 | udp_port=n2k_conf['udp_port']) 184 | 185 | logger.info('Checking for InfluxDB Connectivity') 186 | try: 187 | if client.ping(): 188 | logger.info('Connected to InfluxDB') 189 | else: 190 | logger.error('Cannot connect to InfluxDB. Check Configuration/Connectivity') 191 | except Exception as connect_e: 192 | logger.error(connect_e) 193 | client.close() 194 | sys.exit(1) 195 | 196 | try: 197 | read_nmea2k() 198 | except KeyboardInterrupt: 199 | print('CTRL+C pressed for script') 200 | analyzer_process.stdout.flush() 201 | analyzer_process.terminate() 202 | analyzer_process.wait() 203 | client.close() 204 | 205 | 206 | if __name__ == '__main__': 207 | main() 208 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 140 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='n2kparser', 5 | version=1.4, 6 | description='Extract values from Actisense-Serial PGNs and save them to InfluxDB and publish them to MQTT', 7 | url='https://github.com/iotfablab/n2kparser', 8 | author='Shantanoo Desai', 9 | author_email='des@biba.uni-bremen.de', 10 | license='MIT', 11 | packages=['n2kparser'], 12 | scripts=['bin/n2kparser'], 13 | install_requires=[ 14 | 'influxdb', 15 | 'paho-mqtt' 16 | ], 17 | include_data_package=True, 18 | zip_safe=False 19 | ) 20 | --------------------------------------------------------------------------------