├── .gitignore ├── .readthedocs.yml ├── CHANGELOG.md ├── LICENSE.txt ├── README.md ├── docs ├── .gitignore ├── CHANGELOG.md ├── Makefile ├── _static │ └── wide_page_table.css ├── _templates │ └── page.html ├── api.rst ├── cli.rst ├── cli_write_support.rst ├── conf.py ├── generate_registry.py ├── images │ ├── app_android_2.500_battery-annotated.png │ ├── app_android_2.500_household-annotated.png │ ├── app_android_2.500_inverter-annotated.png │ ├── app_android_2.500_main_overview-annotated.png │ ├── app_android_2.500_power_grid-annotated.png │ ├── app_android_2.500_power_switch_sensor-annotated.png │ └── app_android_2.500_solar_generator-annotated.png ├── index.rst ├── inverter_app_mapping.rst ├── inverter_bitfields.rst ├── inverter_faults.rst ├── inverter_registry.rst ├── protocol_event_table.rst ├── protocol_overview.rst ├── protocol_timeseries.rst ├── simulator.rst ├── tools.rst └── usage.rst ├── requirements.txt ├── requirements_dev.txt ├── setup.cfg ├── setup.py ├── src └── rctclient │ ├── __init__.py │ ├── cli.py │ ├── exceptions.py │ ├── frame.py │ ├── py.typed │ ├── registry.py │ ├── simulator.py │ ├── types.py │ └── utils.py ├── tests ├── __init__.py ├── test_bytecode_generation.py ├── test_decode_value.py ├── test_receiveframe.py └── test_sendframe.py └── tools ├── .gitignore ├── README.md ├── csv2influxdb.py ├── read_pcap.py └── timeseries2csv.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | devtools/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # Environments 92 | .env 93 | .venv 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | .dmypy.json 113 | dmypy.json 114 | 115 | # Pyre type checker 116 | .pyre/ 117 | 118 | # Swap 119 | [._]*.s[a-v][a-z] 120 | [._]*.sw[a-p] 121 | [._]s[a-rt-v][a-z] 122 | [._]ss[a-gi-z] 123 | [._]sw[a-p] 124 | 125 | # Session 126 | Session.vim 127 | 128 | # Temporary 129 | .netrwhist 130 | *~ 131 | # Auto-generated tag files 132 | tags 133 | # Persistent undo 134 | [._]*.un~ 135 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | 4 | build: 5 | os: "ubuntu-22.04" 6 | tools: 7 | python: "3.11" 8 | 9 | sphinx: 10 | configuration: docs/conf.py 11 | 12 | python: 13 | install: 14 | - method: pip 15 | path: . 16 | extra_requirements: 17 | - docs 18 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## Release 0.0.5 - unreleased 6 | 7 | ### Documentation 8 | 9 | - Add page to document inverter bitfields and enumerations. 10 | - Add more fault codes and a warning that the added codes overflow the bitfield. 11 | - Update description for ``battery.bat_status`` in the App mapping after RCT added a proper name to it, also add a description to the field in the Registry. 12 | 13 | ## Release 0.0.4 - 2023-10-05 14 | 15 | ### Documentation 16 | 17 | - Add a page showing the overview screens of the RCT Power app and which OIDs are used to display the values. 18 | - Mention that some behaviours and the entire content of the registry is an implementation detail of the vendor and are 19 | not mandated by the protocol itself and as such may not apply to other implementations. 20 | - New page `Faults` for interpreting the `fault[*].flt` responses in RCT devices. 21 | - Split documentation by moving the vendor-specific (aka RCT Inverter) parts to their own section (Registry, Faults and 22 | App mapping). 23 | - Protocol overview: 24 | - Add section about lack of protocol-level error handling. 25 | - Add note about `EXTENSION` commands, their structure is unknown. 26 | - Document what is known about `READ_PERIODICALLY` and how it is supposed to work with vendor devices. 27 | - Mention checksum algorithm. 28 | - List all known commands as well as reserved ones. 29 | - Describe plant communication, which has not been tested yet. 30 | - CLI invocation: Bash-completion activation changed with newer versions of `Click`. 31 | - CLI: Document why there is now write support in the CLI. 32 | 33 | ### Features 34 | 35 | - `Command` has new functions `is_write` and `is_response` to help working with received frames. 36 | - `Command` learned about `READ_PERIODICALLY`, but it has not been tested yet. 37 | - `Command` learned about the `PLANT_` equivalents of the other commands. 38 | 39 | ### Bugfixes 40 | 41 | - CLI: Implement support for `Click 8.1` caused by API changes related to custom completions (Issue #17, PR #18). 42 | - Make setuptools happy again by adhering to `PEP-508`. 43 | 44 | ### Dependency changes 45 | 46 | - `Click`: Version `7.0` is the new minimum, it supported custom completion functions for the first time 47 | - `Click`: Version `8.1` is the new maximum, to guard against API changes during unconditionally updating the 48 | dependencies. 49 | 50 | ## Release 0.0.3 - 2021-05-22 51 | 52 | ### Breaking changes 53 | 54 | #### Receiving of frames has been completely reworked 55 | 56 | It now uses a streaming approach where parts are decoded (almost) as soon as they are received instead of waiting for 57 | the entire frame to be received. This was done in order to allow for more flexible handling. The correctness of the 58 | data still cannot be determined before the entire frame has been received and the CRC16 checksum indicating correct 59 | reception. 60 | 61 | Except for invalid or unsupported (EXTENSION) commands, which will raise an exception and abort consumption, the 62 | properties of `ReceiveFrame` are now populated as soon as possible and no longer raise an exception when accessed 63 | before the frame is `complete()`. 64 | 65 | **Rationale**: The main use case is the detection and handling of frames with invalid length field. As the correctness 66 | of the frame could previously only be determined after it the advertised amount of data was received, frames that 67 | advertise an abnormal amount of data consumed tens or hundreds of valid frames with no way for the application to 68 | determine what was wrong. With the change, the application can now check for command and length, and abort the frame if 69 | it seems reasonable. For example when it detects that a frame that carries a `DataType.UINT8` field wants to consume 70 | 100 bytes, which is far larger than what is needed to transport such a small type, it can abort the frame and skip past 71 | the beginning of the broken frame, as an alternative to keeping track of buffer contents over multiple TCP packets, in 72 | order to loop back once it is clear that the current frame is broken. 73 | 74 | The exception `FrameNotComplete` was removed as it was not used any more, and `InvalidCommand` was added in its place. 75 | Furthermore, if the parser detects that it overshot (which hints at a programming error), it raises 76 | `FrameLengthExceeded`, enabling calling code to abort the frame. 77 | 78 | ### Known issues 79 | 80 | - Time stamps in the output of tool `timeseries2csv.py` are off by one or two (during DST) hours. 81 | 82 | ### Features 83 | 84 | - Registry: Update with new OIDs from OpenWB. 85 | - Tool `read_pcap.py` now makes an attempt to decode frames that are complete but have an incorrect checksum to try to 86 | give a better insight into what's going on. 87 | - Tool `read_pcap.py` prints the time stamp encoded in the dump with each packet. 88 | - `ReceiveFrame`: Add a flag to allow decoding the content of a frame even if the CRC checksum does not match. This is 89 | intended as a debug measure and not to be used in normal operation. 90 | - Added type hints for `decode_value` and `encode_value`. Requires `typing_extensions` for Python version 3.7 and 91 | below. 92 | - Mention that tool `csv2influx.py` is written with InfluxDB version 1.x in mind (Issue #10). 93 | - Debugging `ReceiveFrame` now happens using the Python logging framework, using the ``rctclient.frame.ReceiveFrame`` 94 | logger, the ``debug()`` method has been removed. 95 | - New CLI flag ``--frame-debug``, which enables debug output for frame parsing if ``--debug`` is set as well. 96 | - Tool `timeseries2csv.py` can now output different header formats (none, the original header, and InfluxDB 2.x 97 | compatible headers). The command line switch ``--no-headers`` was replaced by ``--header-format``. 98 | 99 | ### Documentation 100 | 101 | - Disable Smartquotes (https://docutils.sourceforge.io/docs/user/smartquotes.html) which renders double-dash strings as 102 | a single hyphen character, and the CLI documentation can't be copy-pasted to a terminal any more without manually 103 | editing it before submitting. (Issue #5). 104 | 105 | ### Bugfixes 106 | 107 | - CLI: Fix incomplete example in `read-value` help output (Issue #5). 108 | - CLI: Change output for OIDs of type `UNKNOWN` to a hexdump. This works around the problem of some of them being 109 | marked as being strings when instead they carry complex data that can't easily be represented as textual data. 110 | - Registry: Mark some OIDs that are known to contain complex data that hasn't been decoded yet as being of type 111 | `UNKNOWN` instead of `STRING`. Most of them cannot be decoded to a valid string most of the time, and even then the 112 | content would not make sense. This change allows users to filter these out, e.g. when printing their content. 113 | - Simulator: If multiple requests were sent in the same TCP packet, the simulator returned the answer for the first 114 | frame that it got for all of the requests in the buffer. 115 | - Tool `read_pcap.py` now drops a frame if it ran over the segment boundary (next TCP packet) if the new segment looks 116 | like it starts with a new frame (`0x002b`). This way invalid frames with very high length fields are caught earlier, 117 | only losing the rest of the segment instead of consuming potentially hundreds of frames only to error out on 118 | CRC-check. 119 | - Tool `csv2influx.py` had a wrong `--resolution` parameter set. It has been adapted to the one used in 120 | `timeseries2csv.py`. Note that the table name is made up from the parameters value and changes with it (Issue #8). 121 | - `ReceiveFrame` used to extract the address in plant frames at the wrong point in the buffer, effectively swapping 122 | address and OID (PR #11). 123 | 124 | ## Release 0.0.2 - 2021-02-17 125 | 126 | ### Features 127 | 128 | - New tool `timeseries2csv.py`: Reads time series data from the device and outputs CSV data for other tools to consume. 129 | - New tool `csv2influx.py`: Takes a CSV generated from `timeseries2csv.py` and writes it to an InfluxDB database. 130 | - Refactored frame generation: The raw byte-stream generation for sending a frame was factored out of class `SendFrame` 131 | and put into its own function `make_frame`. Internally, `SendFrame` calls `make_frame`. 132 | - CLI: Implement simple handling of time series data. The data is returned as a CSV table and the start timestamp is 133 | always the current time. 134 | - CLI: Implement simple handling of the event table. The data is returned as a CSV table and the start timestamp is 135 | always the current time. The data is printed as hexadecimal strings, as the meaning of most of the types is now known 136 | yet. 137 | - `Registry`: Add handling of enum value mappings. 138 | - Tool `read_pcap.py`: learned to output enum values as text (from mapping in `Registry`). 139 | - Setup: The `rctclient` CLI is only installed if the `cli` dependencies are installed. 140 | - Tests: Some unit-tests were added for the encoding and decoding of frames. 141 | - Tests: Travis was set up to run the unit-tests. 142 | 143 | ### Documentation 144 | 145 | - New tools `timeseries2csv.py` and `csv2influx.py` added. 146 | - Enum-mappings were added to the Registry documentation. 147 | - Event table: document recent findings. 148 | - Protocol: documentation of the basic protocol has been enhanced. 149 | - Added this changelog file and wired it into the documentation generation. 150 | 151 | ### Bugfixes 152 | 153 | - Encoding/Decoding: `ENUM` data types are now correctly encoded/decoded the same as `UINT8`. 154 | - Simulator: Fix mocking of `BOOL` and `STRING`. 155 | 156 | ## Release 0.0.1 - 2020-10-07 157 | 158 | Initial release. 159 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rctclient - Python implementation of the RCT Power GmbH "Serial Communication Protocol" 2 | 3 | This Python module implements the "Serial Communication Protocol" by RCT Power GmbH, used in their line of solar 4 | inverters. Appart from the API, it also includes a registry of object IDs and a command line tool. For development, a 5 | simple simulator is included. 6 | 7 | This project is not in any way affiliated with or supported by RCT Power GmbH. 8 | 9 | ## Documentation 10 | 11 | Below is a quickstart guide, the project documentation is on [Read the Docs](https://rctclient.readthedocs.io/). 12 | 13 | ## Installing 14 | 15 | Install and update using [pip](https://pip.pypa.io/en/stable/quickstart/): 16 | 17 | ``` 18 | $ pip install -U rctclient 19 | ``` 20 | 21 | To install the dependencies required for the CLI tool: 22 | 23 | ``` 24 | $ pip install -U rctclient[cli] 25 | ``` 26 | 27 | ## Example 28 | 29 | Let's read the current battery state of charge: 30 | ```python 31 | 32 | import socket, select, sys 33 | from rctclient.frame import ReceiveFrame, make_frame 34 | from rctclient.registry import REGISTRY as R 35 | from rctclient.types import Command 36 | from rctclient.utils import decode_value 37 | 38 | # open the socket and connect to the remote device: 39 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 40 | sock.connect(('192.168.0.1', 8899)) 41 | 42 | # query information about an object ID (here: battery.soc): 43 | object_info = R.get_by_name('battery.soc') 44 | 45 | # construct a frame that will send a read command for the object ID we want, and send it 46 | send_frame = make_frame(command=Command.READ, id=object_info.object_id) 47 | sock.send(send_frame) 48 | 49 | # loop until we got the entire response frame 50 | frame = ReceiveFrame() 51 | while True: 52 | ready_read, _, _ = select.select([sock], [], [], 2.0) 53 | if sock in ready_read: 54 | # receive content of the input buffer 55 | buf = sock.recv(256) 56 | # if there is content, let the frame consume it 57 | if len(buf) > 0: 58 | frame.consume(buf) 59 | # if the frame is complete, we're done 60 | if frame.complete(): 61 | break 62 | else: 63 | # the socket was closed by the device, exit 64 | sys.exit(1) 65 | 66 | # decode the frames payload 67 | value = decode_value(object_info.response_data_type, frame.data) 68 | 69 | # and print the result: 70 | print(f'Response value: {value}') 71 | ``` 72 | 73 | ## Reading values from the command line 74 | 75 | The module installs the `rctclient` command (requires `click`). The subcommand `read-values` reads a single value from 76 | the device and returns its output. Here is a call example using the object ID with verbose output: 77 | 78 | ``` 79 | $ rctclient read-value --verbose --host 192.168.0.1 --id 0x959930BF 80 | #413 0x959930BF battery.soc SOC (State of charge) 0.29985150694847107 81 | ``` 82 | 83 | Without `--verbose`, the only thing that's printed is the received value. This is demonstrated below, where the 84 | `--name` parameter is used instead of the `--id`: 85 | ``` 86 | $ rctclient read-value --host 192.168.0.1 --name battery.soc 87 | 0.2998138964176178 88 | ``` 89 | This makes it suitable for use with scripting environments where one just needs some values. If `--debug` is added 90 | before the subcommands name, the log level is set to DEBUG and all log messages are sent to `stderr`, which allows for 91 | scripts to continue processing the value on stdout, while allowing for observations of the inner workings of the code. 92 | 93 | ## Generating the documentation 94 | 95 | The documentation is generated using Sphinx, and requires that the software be installed to the local environment (e.g. 96 | via virtualenv). With a local clone of the repository, do the following (activate your virtualenv before if so 97 | desired): 98 | ``` 99 | $ pip install -e .[docs,cli] 100 | $ cd docs 101 | $ make clean html 102 | ``` 103 | The documentation is put into the `docs/_build/html` directory, simply point your browser to the `index.html` file. 104 | 105 | The documentation is also auto-generated after every commit and can be found at 106 | [https://rctclient.readthedocs.io/](https://rctclient.readthedocs.io/). 107 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | *.csv 2 | -------------------------------------------------------------------------------- /docs/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ../CHANGELOG.md -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile html clean 16 | 17 | CSV_FILES := objectgroup_acc_conv.csv objectgroup_adc.csv objectgroup_bat_mng_struct.csv objectgroup_battery.csv objectgroup_buf_v_control.csv objectgroup_can_bus.csv objectgroup_cs_map.csv objectgroup_cs_neg.csv objectgroup_db.csv objectgroup_dc_conv.csv objectgroup_display_struct.csv objectgroup_energy.csv objectgroup_fault.csv objectgroup_flash_param.csv objectgroup_flash_rtc.csv objectgroup_grid_lt.csv objectgroup_grid_mon.csv objectgroup_g_sync.csv objectgroup_hw_test.csv objectgroup_io_board.csv objectgroup_iso_struct.csv objectgroup_line_mon.csv objectgroup_logger.csv objectgroup_modbus.csv objectgroup_net.csv objectgroup_nsm.csv objectgroup_others.csv objectgroup_power_mng.csv objectgroup_p_rec.csv objectgroup_prim_sm.csv objectgroup_rb485.csv objectgroup_switch_on_cond.csv objectgroup_temperature.csv objectgroup_wifi.csv 18 | 19 | html: $(CSV_FILES) 20 | @$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | 22 | clean: 23 | @$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 24 | rm -f objectgroup_*.csv 25 | 26 | %.csv: ../src/rctclient/registry.py 27 | ./generate_registry.py 28 | 29 | ../src/rctclient/registry.py: 30 | @: 31 | 32 | # Catch-all target: route all unknown targets to Sphinx using the new 33 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 34 | %: Makefile 35 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 36 | -------------------------------------------------------------------------------- /docs/_static/wide_page_table.css: -------------------------------------------------------------------------------- 1 | /* taken from https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html and 2 | * https://github.com/readthedocs/sphinx_rtd_theme/issues/295 3 | */ 4 | 5 | /* override table width restrictions */ 6 | @media screen and (min-width: 767px) { 7 | .wy-nav-content { 8 | max-width: 1500px !important; 9 | } 10 | 11 | .wy-table-responsive table td { 12 | /* !important prevents the common CSS stylesheets from overriding 13 | * this as on RTD they are loaded after this stylesheet 14 | */ 15 | white-space: normal !important; 16 | } 17 | 18 | .wy-table-responsive { 19 | overflow: visible !important; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /docs/_templates/page.html: -------------------------------------------------------------------------------- 1 | {% extends "!page.html" %} 2 | {%- if meta.wide_page|default(false) %} 3 | {%- set css_files = css_files + ['_static/wide_page_table.css'] %} 4 | {%- endif %} 5 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _api: 3 | 4 | ### 5 | API 6 | ### 7 | 8 | Types 9 | ***** 10 | 11 | .. autoclass:: rctclient.types.Command 12 | :members: 13 | 14 | .. autoclass:: rctclient.types.ObjectGroup 15 | :members: 16 | :undoc-members: 17 | 18 | .. autoclass:: rctclient.types.FrameType 19 | :members: 20 | 21 | .. autoclass:: rctclient.types.DataType 22 | :members: 23 | 24 | .. autoclass:: rctclient.types.EventEntry 25 | :members: 26 | 27 | 28 | Exceptions 29 | ********** 30 | 31 | .. autoclass:: rctclient.exceptions.RctClientException 32 | 33 | .. autoclass:: rctclient.exceptions.FrameError 34 | 35 | .. autoclass:: rctclient.exceptions.FrameCRCMismatch 36 | 37 | .. autoclass:: rctclient.exceptions.FrameLengthExceeded 38 | 39 | .. autoclass:: rctclient.exceptions.InvalidCommand 40 | 41 | Classes 42 | ******* 43 | 44 | .. autoclass:: rctclient.registry.ObjectInfo 45 | :members: 46 | 47 | .. autoclass:: rctclient.registry.Registry 48 | :members: 49 | 50 | .. autoclass:: rctclient.frame.ReceiveFrame 51 | :members: 52 | 53 | .. autoclass:: rctclient.frame.SendFrame 54 | :members: 55 | 56 | Functions 57 | ********* 58 | 59 | .. autofunction:: rctclient.frame.make_frame 60 | 61 | .. autofunction:: rctclient.utils.decode_value 62 | 63 | .. autofunction:: rctclient.utils.encode_value 64 | 65 | .. autofunction:: rctclient.utils.CRC16 66 | -------------------------------------------------------------------------------- /docs/cli.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _cli: 3 | 4 | ### 5 | CLI 6 | ### 7 | 8 | The library comes with a CLI tool called ``rctclient`` that offers some useful subcommands. 9 | 10 | The tool is only installed if the `click `_ module's present. If installing from 11 | `pip`, the requirement can be pulled in by specifying ``rctclient[cli]``. 12 | 13 | For certain parameters, the tool supports shell completion. Depending on your version of *Click*, there are different 14 | ways to enable the completion: 15 | 16 | * ``< 8.0.0``: ``eval "$(_RCTCLIENT_COMPLETE=source_bash rctclient)"`` (e.g. Debian 11, Ubuntu 20.04) 17 | * ``>= 8.0.0``: ``eval "$(_RCTCLIENT_COMPLETE=bash_source rctclient)"`` (e.g. Arch, Fedora 35+, Gentoo) 18 | 19 | Read more about this at the `click documentation `_. 20 | 21 | .. click:: rctclient.cli:cli 22 | :prog: rctclient 23 | :nested: full 24 | -------------------------------------------------------------------------------- /docs/cli_write_support.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _cli_write_support: 3 | 4 | ############# 5 | Write support 6 | ############# 7 | 8 | The CLI does **not** include support for changing values. This will not change in the foreseeable future. 9 | 10 | Rationale 11 | ********* 12 | .. warning:: 13 | 14 | TL;DR: It's just too dangerous. 15 | 16 | The tool is intended to be used by end users, and it does not look like the devices being able to protect themselves 17 | against invalid data. 18 | 19 | The protocol lacks any measure to communicate failure of any sorts. While the device has a rich set of error conditions 20 | that can be queried (``fault[*].flt``), it does not have a feature to communicate "this value is invalid". It isn't 21 | known how a device will react. It may simply ignore the value or send a specific value back that is not part of the 22 | documentation, or simply apply it somehow. What would happen if the trailing ``\0`` on a string was missing, or if the 23 | payload was far larger than what the device supports, for example? Would it run over the intended length of the string 24 | into its own adjacent memory, causing it to… do what, crash and reset, or would it become bricked? There have been 25 | instances where a padded string payload was received, containing garbled data after the trailing ``\0`` that looked 26 | like another frame, hinting at less-than-ideal memory management in that particular version of the control software. Or 27 | the network interface card, it's not known how the data is passed around. It is also unknown how to recover, though 28 | ultimately replacing components is a valid recovery strategy, albeit potentially costly. 29 | 30 | It is also known that certain values must be within bounds, such as a maximum or minimum value for an integer, or a 31 | maximum length for a string. But what these bounds are is not documented by the vendor, although the Android app seems 32 | to have knowledge of them. In order to prevent accidentally causing damage, the client application would need to have 33 | knowledge of the bounds, which is not possible at the moment. It is unknown how the devices deal with invalid data. 34 | From various reports, it is safe to assume that (at the time of writing) there are little to no checks, such as 35 | corrupting the display. 36 | 37 | Some OIDs are not meant to be written to, yet the device does allow writing to at least some of them and reports the 38 | new value in further read-requests. The documentation does not include information on which are writeable or supposed 39 | to be written to, or which of these are persisted to flash. Yet the client would need to know this to prevent 40 | overwriting important information. 41 | 42 | The vendor provides zero support for usage of the device by means other than their App or portal. Others who tried to 43 | implement the protocol in their own applications have reported that while they got access to a -- presumably stripped 44 | down or even simulated -- device to test against, the support did not answer any of their questions regarding specifics 45 | related to the protocol, let alone how the devices react or how one is supposed to react to certain behaviours that the 46 | device shows. 47 | 48 | Potential for (physical) damage: With the above, imagine that the device simply applies the data. This may potentially 49 | lead to impact in the real world, e.g. by over-stressing its own components, impacting the power grid or cause damage 50 | to electronics within the house. The devices seem to rely on the App to prevent invalid data from being send. 51 | 52 | Testing is non-trivial, as a device would be required. Most devices are in use (installed into domestic houses or 53 | factories), so testing on them is out of the question. At this moment, most "testing" is done by looking at *tcpdumps* 54 | to extract WRITE calls issued by the official app and compare it with the output of functions like 55 | :func:`~rctclient.frame.make_frame`. This is very limited as well, as only a very small subset of OIDs can be changed 56 | while the device is in use (without causing trouble). 57 | 58 | Do it yourself! 59 | *************** 60 | If you want to implement this on your own, here are some tips: 61 | 62 | Values appear to be ephemeral by default, meaning their values will be reset when the device reboots. To persist them, 63 | you need to do the COM-dance by writing ``0x05`` to ``com_service`` and then set it back to ``0x00`` after a couple of 64 | seconds. 65 | 66 | If this was a success is unknown, one knows when the next reboot happens and the old value shows up. Also, this wears 67 | the flash down, these chips typically have a very limited lifetime (anywhere from a few dozen to a couple hundred 68 | writes, depending on the actual chip and how it is implemented), so one does not want to write to flash other than 69 | absolutely necessary, and the device most likely breaks when the flash is dead. There are certain counters that can be 70 | queried that seem to report the number of write-cycles though, but it's not specifically documented and how many it can 71 | sustain is unknown. 72 | 73 | That being said, the client does support writing: The ``timeseries`` data is queried by issuing ``WRITE``-commands with 74 | the current unix timestamp to the device and it answers with a ``timestamp: value-table``. That was, by the way, 75 | reverse-engineered by staring at hexdumps, as the format is not in the documentation. But all the code is there, just 76 | not wired up the way you want. 77 | 78 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | 15 | # -- Read-the-docs specifics ------------------------------------------------- 16 | 17 | on_rtd = os.environ.get('READTHEDOCS') == 'True' 18 | if on_rtd: 19 | import subprocess 20 | subprocess.run('./generate_registry.py') 21 | 22 | # -- Project information ----------------------------------------------------- 23 | 24 | project = 'rctclient' 25 | copyright = '2020-2021, Peter Oberhofer, 2020-2023 Stefan Valouch' 26 | author = 'Stefan Valouch' 27 | 28 | # The full version, including alpha/beta/rc tags 29 | release = '0.0.5' 30 | 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 36 | # ones. 37 | extensions = [ 38 | 'sphinx.ext.autodoc', 39 | 'sphinx.ext.todo', 40 | 'sphinx_autodoc_typehints', 41 | 'sphinx_click.ext', 42 | 'recommonmark', 43 | ] 44 | 45 | # Add any paths that contain templates here, relative to this directory. 46 | templates_path = ['_templates'] 47 | 48 | # List of patterns, relative to source directory, that match files and 49 | # directories to ignore when looking for source files. 50 | # This pattern also affects html_static_path and html_extra_path. 51 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 52 | 53 | # Disable automatic quotes, which converts '--' to '—' (among others) which 54 | # breaks the CLI examples where the source is a python docstring and can't be 55 | # changed without interfering with CLI --help output. 56 | smartquotes = False 57 | 58 | # -- Options for HTML output ------------------------------------------------- 59 | 60 | # The theme to use for HTML and HTML Help pages. See the documentation for 61 | # a list of builtin themes. 62 | # 63 | import sphinx_rtd_theme 64 | html_theme = 'sphinx_rtd_theme' 65 | 66 | # Add any paths that contain custom static files (such as style sheets) here, 67 | # relative to this directory. They are copied after the builtin static files, 68 | # so a file named "default.css" will overwrite the builtin "default.css". 69 | html_static_path = ['_static'] 70 | 71 | # -- Options for todo extension ---------------------------------------------- 72 | 73 | # If true, `todo` and `todoList` produce output, else they produce nothing. 74 | todo_include_todos = True 75 | 76 | # -- Options for autodoc extension ------------------------------------------- 77 | 78 | # autodoc_mock_imports = ['rctclient.settings'] 79 | -------------------------------------------------------------------------------- /docs/generate_registry.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ''' 4 | Generator for CSV files from REGISTRY content. 5 | ''' 6 | 7 | import csv 8 | from typing import Dict 9 | 10 | from rctclient.registry import REGISTRY as R 11 | from rctclient.types import DataType, ObjectGroup 12 | 13 | 14 | def generate_registry_csv() -> None: 15 | ''' 16 | Generates the registry csv files from the REGISTRY instance. 17 | ''' 18 | files: Dict[ObjectGroup, Dict] = dict() 19 | for ogroup in ObjectGroup: 20 | files[ogroup] = dict() 21 | files[ogroup]['file'] = open(f'objectgroup_{ogroup.name.lower()}.csv', 'wt') 22 | files[ogroup]['csv'] = csv.writer(files[ogroup]['file']) 23 | files[ogroup]['csv'].writerow(['OID', 'Request Type', 'Response Type', 'Unit', 'Name', 'Description']) 24 | 25 | for oinfo in sorted(R.all()): 26 | description = oinfo.description if oinfo.description is not None else '' 27 | if oinfo.request_data_type == DataType.ENUM: 28 | if len(description) > 0 and description[-1] != '.': 29 | description += '.' 30 | if oinfo.enum_map is not None: 31 | # list the mappings in the description 32 | enum = ', '.join([f'"{v}" = ``0x{k:02X}``' for k, v in oinfo.enum_map.items()]) 33 | description += f' ENUM values: {enum}' 34 | else: 35 | description += ' (No enum mapping defined)' 36 | 37 | files[oinfo.group]['csv'].writerow([f'``0x{oinfo.object_id:X}``', oinfo.request_data_type.name, 38 | oinfo.response_data_type.name, oinfo.unit, f'``{oinfo.name}``', 39 | description]) 40 | 41 | for val in files.values(): 42 | val['file'].close() 43 | 44 | 45 | if __name__ == '__main__': 46 | generate_registry_csv() 47 | -------------------------------------------------------------------------------- /docs/images/app_android_2.500_battery-annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/svalouch/python-rctclient/9300ea947e6f45a100e6b598b8146c2e59a9d646/docs/images/app_android_2.500_battery-annotated.png -------------------------------------------------------------------------------- /docs/images/app_android_2.500_household-annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/svalouch/python-rctclient/9300ea947e6f45a100e6b598b8146c2e59a9d646/docs/images/app_android_2.500_household-annotated.png -------------------------------------------------------------------------------- /docs/images/app_android_2.500_inverter-annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/svalouch/python-rctclient/9300ea947e6f45a100e6b598b8146c2e59a9d646/docs/images/app_android_2.500_inverter-annotated.png -------------------------------------------------------------------------------- /docs/images/app_android_2.500_main_overview-annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/svalouch/python-rctclient/9300ea947e6f45a100e6b598b8146c2e59a9d646/docs/images/app_android_2.500_main_overview-annotated.png -------------------------------------------------------------------------------- /docs/images/app_android_2.500_power_grid-annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/svalouch/python-rctclient/9300ea947e6f45a100e6b598b8146c2e59a9d646/docs/images/app_android_2.500_power_grid-annotated.png -------------------------------------------------------------------------------- /docs/images/app_android_2.500_power_switch_sensor-annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/svalouch/python-rctclient/9300ea947e6f45a100e6b598b8146c2e59a9d646/docs/images/app_android_2.500_power_switch_sensor-annotated.png -------------------------------------------------------------------------------- /docs/images/app_android_2.500_solar_generator-annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/svalouch/python-rctclient/9300ea947e6f45a100e6b598b8146c2e59a9d646/docs/images/app_android_2.500_solar_generator-annotated.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ##################################### 2 | Welcome to rctclient's documentation! 3 | ##################################### 4 | 5 | This documentation covers the `rctclient` python library, used to interact with solar inverters by `RCT Power GmbH` by 6 | implementing their proprietary "RCT Power Serial Communication Protocol". It also serves as documentation of the 7 | protocol itself. As such, the main focus is on supporting the vendors devices, but the library can be used for own 8 | implementations just as well. 9 | 10 | **Disclaimer**: Neither the python library nor this documentation is in any way affiliated with or supported by RCT 11 | Power GmbH in any way. Do **not** ask them for support regarding anything concerning the content of the material 12 | provided by this project. 13 | 14 | **2nd Disclaimer**: Use the material provided by this project (code, documentation etc.) at your own risk. None of the 15 | contributors can be held liable for any damage that may occur by using the information provided. See also the `LICENSE` 16 | file for further information. 17 | 18 | Target audience 19 | *************** 20 | * The :ref:`cli` is meant for end users who wish to easily extract values without having to write a program. 21 | * The :ref:`library ` is intended for developers who wish to enable their software to interact with the devices. 22 | * The documentation is also mostly targeted at developers, but some of the information may be of use for everyone else, 23 | too. 24 | 25 | The CLI is pretty low-level, however, and you may wish to use a ready-to-use solution for integrating the devices into 26 | your own home automation or to monitor them. In this case, here's a randomly sorted list open source tools that may be 27 | of interest for you: 28 | 29 | * `home-assistant-rct-power-integration `__ by 30 | `@weltenwort `__ integrates an inverter into Home Assistant (using this library). 31 | * `solaranzeige.de `__ (German) is a complete solution running on a 32 | Raspberry Pi. 33 | * `rctmon `__ exposes a Prometheus endpoint and fills an InfluxDB (using this 34 | library). 35 | * `OpenWB `__ (German) is an open source wallbox. 36 | * Integrations for Node-Red, Homematic and a lot of other systems can be found in their respective communities. 37 | 38 | .. toctree:: 39 | :maxdepth: 2 40 | :caption: Module contents 41 | 42 | usage 43 | cli 44 | cli_write_support 45 | simulator 46 | api 47 | tools 48 | CHANGELOG 49 | 50 | .. toctree:: 51 | :maxdepth: 1 52 | :caption: RCT Inverter Documentation 53 | 54 | inverter_registry 55 | inverter_faults 56 | inverter_bitfields 57 | inverter_app_mapping 58 | 59 | .. toctree:: 60 | :maxdepth: 1 61 | :caption: Protocol 62 | 63 | protocol_overview 64 | protocol_event_table 65 | protocol_timeseries 66 | 67 | History 68 | ******* 69 | The original implementation was done by GitHub user `pob90 `__, including a simulator that 70 | can be used for development and a command line tool to manually query single objects from the device. The simulator 71 | proved to be invaluable when developing software that interfaces with RCT devices and when porting the code to Python 72 | 3. The original code is based on the official documentation by the vendor (version 1.8, later updated to 1.13). 73 | 74 | The original implementation can be found here: ``__ 75 | 76 | The implementation was done for use with the `OpenWB `__ project. The projects goal is to 77 | provide an open source wall box for charging electric cars, and it needs to interface with various devices in order to 78 | optimize its usage of energy, such as only charging when the solar inverter signals an abundance of energy. 79 | 80 | The project was, however, implemented as pure Python 2 code. As Python 2 is officially dead and the first distributions 81 | completed its removal, the first step was to convert everything to Python 3. Once the code worked (again), the 82 | structure was changed to span multiple files instead of one single file, some classes were split, many were renamed and 83 | variable constants were converted to enums and `PEP-484 `__ type hinting was 84 | added. 85 | 86 | See the *CHANGELOG* for the history since. 87 | -------------------------------------------------------------------------------- /docs/inverter_bitfields.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _inverter_bitfields: 3 | 4 | ########################## 5 | Bitfields and Enumerations 6 | ########################## 7 | 8 | As is common with embedded devices, the inverters use bitfields to save space, as a single byte can represent a number 9 | of individual states in 255 possible combinations. The downside is: One cannot interpret these without an explanation. 10 | 11 | ``fault[*].flt`` 12 | **************** 13 | With four 32-bit-OIDs and thus 128 bits, the ``fault[*].flt`` family is so large that it gets its own page at 14 | :ref:`inverter_faults`. 15 | 16 | ``battery.bat_status`` 17 | ********************** 18 | This 32-bit wide value describes the status of the battery stack as a whole. Some of the values have been empirically 19 | identified, but most of it is largely unknown. 20 | 21 | * ``value & 1032 == 0`` causes the App to display ``(calib.)`` [1]_. Note that this sets two bits at the same time. 22 | * ``value & 2048 == 0`` denotes that balancing is in progress (App displays ``(balance)``) [1]_ 23 | 24 | .. [1] Found by "oliverrahner": `weltenwort/home-assistant-rct-power-integration#264 (comment) `__ -------------------------------------------------------------------------------- /docs/inverter_faults.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _inverter_faults: 3 | 4 | ###### 5 | Faults 6 | ###### 7 | 8 | There is a total of four OIDs that make up a 128 bit wide field, with each bit representing a particular fault. Each of 9 | the OIDs contains 32 bits. If no bits are set (i.e. the four OIDs return 0), no fault is present. 10 | 11 | ================ ============== 12 | ``fault[0].flt`` Bits 0 to 31 13 | ``fault[1].flt`` Bits 32 to 63 14 | ``fault[2].flt`` Bits 64 to 95 15 | ``fault[3].flt`` Bots 96 to 127 16 | ================ ============== 17 | 18 | The table lists known faults, if the bit is set in the devices output then the fault is currently present: 19 | 20 | === ========================================================================================= 21 | Bit Description 22 | === ========================================================================================= 23 | 0 TRAP occured 24 | 1 RTC can't be configured 25 | 2 RTC 1Hz signal timeout 26 | 3 Hardware Stop by 3.3V fault 27 | 4 Hardware Stop by PWM Logic 28 | 5 Hardware Stop by ``Uzk`` overvoltage 29 | 6 ``Uzk+`` is above limit 30 | 7 ``Uzk-`` is above limit 31 | 8 Throttle phase ``L1`` overcurrent 32 | 9 Throttle phase ``L2`` overcurrent 33 | 10 Throttle phase ``L3`` overcurrent 34 | 11 Buffer capacitor voltage 35 | 12 Quartz fault 36 | 13 Grid under-voltage phase 1 37 | 14 Grid under-voltage phase 2 38 | 15 Grid under-voltage phase 3 39 | 16 Battery overcurrent 40 | 17 Relais test failed 41 | 18 Board overtemperature 42 | 19 Core overtemperature 43 | 20 (Heat)Sink 1 overtemperature 44 | 21 (Heat)Sink 2 overtemperature 45 | 22 Error in I2C communication with Power Board 46 | 23 Power Board error 47 | 24 PWM output ports defective 48 | 25 Insulation to small or not plausible 49 | 26 ``I`` DC component max (1A) 50 | 27 ``I`` DC component max slow (47mA) 51 | 28 Possible defect in one of the DSD channels (current offset too large) 52 | 29 Error in RS485 communication with Relais Box IGBT L1 BH defective 53 | 30 Phase to phase overvoltage 54 | 31 IGBT L1 BH defective 55 | 32 IGBT L1 BL defective 56 | 33 IGBT L2 BH defective 57 | 34 IGBT L2 BL defective 58 | 35 IGBT L3 BH defective 59 | 36 IGBT L3 BL defective 60 | 37 Long-term overvoltage phase 1 61 | 38 Long-term overvoltage phase 2 62 | 39 Long-term overvoltage phase 3 63 | 40 Overvoltage phase 1, level 1 64 | 41 Overvoltage phase 1, level 2 65 | 42 Overvoltage phase 2, level 1 66 | 43 Overvoltage phase 2, level 2 67 | 44 Overvoltage phase 3, level 1 68 | 45 Overvoltage phase 3, level 2 69 | 46 Overfrequency, level 1 70 | 47 Overfrequency, level 2 71 | 48 Undervoltage phase 1, level 1 72 | 49 Undervoltage phase 1, level 2 73 | 50 Undervoltage phase 2, level 1 74 | 51 Undervoltage phase 2, level 2 75 | 52 Undervoltage phase 3, level 1 76 | 53 Undervoltage phase 3, level 2 77 | 54 Underfrequency, level 1 78 | 55 Underfrequency, level 2 79 | 56 CPU exception NMI 80 | 57 CPU exception HardFault 81 | 58 CPU exception MemManage 82 | 59 CPU exception BusFault 83 | 60 CPU exception UsageFault 84 | 61 RTC Power on reset 85 | 62 RTC Oscillation stops 86 | 63 RTC Supply voltage drop 87 | 64 Jump of RCD current ``DC + AC > 30mA`` was noticed 88 | 65 Jump of RCD current ``DC > 60mA`` was noticed 89 | 66 Jump of RCD current ``AC > 150mA`` was noticed 90 | 67 RCD current ``> 300mA`` was notices 91 | 68 Incorrect ``+5V`` was noticed 92 | 69 Incorrect ``-9V`` was noticed 93 | 70 Incorrect ``+9V`` was noticed 94 | 71 Incorrect ``+3V3`` was noticed 95 | 72 Failure of RDC calibration was noticed 96 | 73 Failure of I2C was noticed 97 | 74 afi frequency generator failure 98 | 75 (Heat)sink temperature too high 99 | 76 ``Uzk`` is over limit 100 | 77 ``Usg A`` is over limit 101 | 78 ``Usg B`` is over limit 102 | 79 Switching On Conditions ``Umin`` phase 1 103 | 80 Switching On Conditions ``Umax`` phase 1 104 | 81 Switching On Conditions ``Fmin`` phase 1 105 | 82 Switching On Conditions ``Fmax`` phase 1 106 | 83 Switching On Conditions ``Umin`` phase 2 107 | 84 Switching On Conditions ``Umax`` phase 2 108 | 85 Battery current sensor defective 109 | 86 Battery booster damaged 110 | 87 Switching On Conditions ``Umin`` phase 3 111 | 88 Switching On Conditions ``Umax`` phase 3 112 | 89 Voltage surge or average offset is too large on AC-terminals (phase failure detected) 113 | 90 Inverter is disconnected from the household grid 114 | 91 Difference of the measured ``+9V`` between DSP and PIC is too large 115 | 92 ``1.5V`` error 116 | 93 ``2.5V`` error 117 | 94 ``1.5V`` measurement difference 118 | 95 ``2.5V`` measurement difference 119 | 96 The battery voltage is outside of the expected range 120 | 97 Unable to start the main PIC software 121 | 98 PIC bootloader detected unexpectedly 122 | 99 Phase position error (not 120° as expected) 123 | 100 Battery overvoltage 124 | 101 Throttle current is unstable 125 | 102 Difference between internally and externally measured grid voltage is too large in phase 1 126 | 103 Difference between internally and externally measured grid voltage is too large in phase 2 127 | 104 Difference between internally and externally measured grid voltage is too large in phase 3 128 | 105 External emergency turn-off signal is active 129 | 106 Battery is empty, not enough energy for standby 130 | 107 CAN communication timeout with battery 131 | 108 Timing problem 132 | 109 Battery IGBT's heat sink overtemperature 133 | 110 Battery heat sink temperature too high 134 | 111 Internal relais box error 135 | 112 Relais box PE off error 136 | 113 Relais box PE on error 137 | 114 Internal battery error 138 | 115 Parameter changed 139 | 116 3 attempts of island building are failing 140 | 117 Phase to phase undervoltage 141 | 118 System reset detected 142 | 119 Update detected 143 | 120 FRT overvoltage 144 | 121 FRT undervoltage 145 | 122 IGBT L1 free-wheeling diode defective 146 | 123 IGBT L2 free-wheeling diode defective 147 | 124 IGBT L3 free-wheeling diode defective 148 | 125 Single-phase mode is activated but not allowed for this device class (e.g. 10K) 149 | 126 Island detected 150 | === ========================================================================================= 151 | 152 | Additionally, recent versions list three additional bits: Number 127 makes sense, but the last two (128+129) can not 153 | be represented by the bit fields, so this might be an error. 154 | 155 | === ================================ 156 | Bit Description 157 | === ================================ 158 | 127 Neutral fault 159 | 128 Battery calibration failed 160 | 129 Power switch relay not confirmed 161 | === ================================ -------------------------------------------------------------------------------- /docs/inverter_registry.rst: -------------------------------------------------------------------------------- 1 | 2 | :wide_page: true 3 | 4 | .. _registry: 5 | 6 | ######## 7 | Registry 8 | ######## 9 | 10 | This page lists all known OIDs for use with a device by RCT Power GmbH. With the protocol only specifying the length of 11 | the OID as four bytes, it is an implementation detail of the particular application. This list here is bundled with the 12 | application, as interfacing with vendor devices is its main use-case. If you were to implement your own application, 13 | you'd simply define your own OIDs and use them instead of the bundled ones. 14 | 15 | The registry as part of the library is a data structure that maintains a list of all known 16 | :class:`~rctclient.registry.ObjectInfo`. It is implemented as :class:`~rctclient.registry.Registry`, please head to the 17 | API documentation for means to query it for object ID information. As the registry is a heavy object, it is maintained 18 | as an instance ``REGISTRY`` in the ``registry`` module and can be imported where needed. 19 | 20 | The following list is a complete index of all the object IDs currently maintained by the registry for use with official 21 | vendor devices. 22 | 23 | .. the following tables are generated from the registry in registry.py using generate_registry.py 24 | 25 | acc_conv 26 | ======== 27 | 28 | .. csv-table:: 29 | :header-rows: 1 30 | :widths: 10, 5, 5, 5, 15, 40 31 | :file: objectgroup_acc_conv.csv 32 | 33 | adc 34 | === 35 | 36 | .. csv-table:: 37 | :header-rows: 1 38 | :widths: 10, 5, 5, 5, 15, 40 39 | :file: objectgroup_adc.csv 40 | 41 | bat_mng_struct 42 | ============== 43 | 44 | .. csv-table:: 45 | :header-rows: 1 46 | :widths: 10, 5, 5, 5, 15, 40 47 | :file: objectgroup_bat_mng_struct.csv 48 | 49 | battery 50 | ======= 51 | 52 | .. csv-table:: 53 | :header-rows: 1 54 | :widths: 10, 5, 5, 5, 15, 40 55 | :file: objectgroup_battery.csv 56 | 57 | buf_v_control 58 | ============= 59 | 60 | .. csv-table:: 61 | :header-rows: 1 62 | :widths: 10, 5, 5, 5, 15, 40 63 | :file: objectgroup_buf_v_control.csv 64 | 65 | can_bus 66 | ======= 67 | 68 | .. csv-table:: 69 | :header-rows: 1 70 | :widths: 10, 5, 5, 5, 15, 40 71 | :file: objectgroup_can_bus.csv 72 | 73 | cs_map 74 | ====== 75 | 76 | .. csv-table:: 77 | :header-rows: 1 78 | :widths: 10, 5, 5, 5, 15, 40 79 | :file: objectgroup_cs_map.csv 80 | 81 | cs_neg 82 | ====== 83 | 84 | .. csv-table:: 85 | :header-rows: 1 86 | :widths: 10, 5, 5, 5, 15, 40 87 | :file: objectgroup_cs_neg.csv 88 | 89 | db 90 | == 91 | 92 | .. csv-table:: 93 | :header-rows: 1 94 | :widths: 10, 5, 5, 5, 15, 40 95 | :file: objectgroup_db.csv 96 | 97 | dc_conv 98 | ======= 99 | 100 | .. csv-table:: 101 | :header-rows: 1 102 | :widths: 10, 5, 5, 5, 15, 40 103 | :file: objectgroup_dc_conv.csv 104 | 105 | display_struct 106 | ============== 107 | 108 | .. csv-table:: 109 | :header-rows: 1 110 | :widths: 10, 5, 5, 5, 15, 40 111 | :file: objectgroup_display_struct.csv 112 | 113 | energy 114 | ====== 115 | 116 | .. csv-table:: 117 | :header-rows: 1 118 | :widths: 10, 5, 5, 5, 15, 40 119 | :file: objectgroup_energy.csv 120 | 121 | fault 122 | ===== 123 | 124 | .. csv-table:: 125 | :header-rows: 1 126 | :widths: 10, 5, 5, 5, 15, 40 127 | :file: objectgroup_fault.csv 128 | 129 | flash_param 130 | =========== 131 | 132 | .. csv-table:: 133 | :header-rows: 1 134 | :widths: 10, 5, 5, 5, 15, 40 135 | :file: objectgroup_flash_param.csv 136 | 137 | flash_rtc 138 | ========= 139 | .. csv-table:: 140 | :header-rows: 1 141 | :widths: 10, 5, 5, 5, 15, 40 142 | :file: objectgroup_flash_rtc.csv 143 | 144 | grid_lt 145 | ======= 146 | 147 | .. csv-table:: 148 | :header-rows: 1 149 | :widths: 10, 5, 5, 5, 15, 40 150 | :file: objectgroup_grid_lt.csv 151 | 152 | grid_mon 153 | ======== 154 | 155 | .. csv-table:: 156 | :header-rows: 1 157 | :widths: 10, 5, 5, 5, 15, 40 158 | :file: objectgroup_grid_mon.csv 159 | 160 | g_sync 161 | ====== 162 | 163 | .. csv-table:: 164 | :header-rows: 1 165 | :widths: 10, 5, 5, 5, 15, 40 166 | :file: objectgroup_g_sync.csv 167 | 168 | hw_test 169 | ======= 170 | 171 | .. csv-table:: 172 | :header-rows: 1 173 | :widths: 10, 5, 5, 5, 15, 40 174 | :file: objectgroup_hw_test.csv 175 | 176 | io_board 177 | ======== 178 | 179 | .. csv-table:: 180 | :header-rows: 1 181 | :widths: 10, 5, 5, 5, 15, 40 182 | :file: objectgroup_io_board.csv 183 | 184 | iso_struct 185 | ========== 186 | 187 | .. csv-table:: 188 | :header-rows: 1 189 | :widths: 10, 5, 5, 5, 15, 40 190 | :file: objectgroup_iso_struct.csv 191 | 192 | line_mon 193 | ======== 194 | 195 | .. csv-table:: 196 | :header-rows: 1 197 | :widths: 10, 5, 5, 5, 15, 40 198 | :file: objectgroup_line_mon.csv 199 | 200 | logger 201 | ====== 202 | The `logger` group contains time series data and the event log. These are special, compound data structures that 203 | require a bit of work to parse. They generally work by writing the timestamp of the newest element of interest to them 204 | and respond with the entries or events **older** than that time stamp. For more details, take a look at the 205 | :ref:`protocol-event-table` and :ref:`protocol-timeseries` pages. 206 | 207 | .. csv-table:: 208 | :header-rows: 1 209 | :widths: 10, 5, 5, 5, 15, 40 210 | :file: objectgroup_logger.csv 211 | 212 | modbus 213 | ====== 214 | 215 | .. csv-table:: 216 | :header-rows: 1 217 | :widths: 10, 5, 5, 5, 15, 40 218 | :file: objectgroup_modbus.csv 219 | 220 | net 221 | === 222 | 223 | .. csv-table:: 224 | :header-rows: 1 225 | :widths: 10, 5, 5, 5, 15, 40 226 | :file: objectgroup_net.csv 227 | 228 | nsm 229 | === 230 | 231 | .. csv-table:: 232 | :header-rows: 1 233 | :widths: 10, 5, 5, 5, 15, 40 234 | :file: objectgroup_nsm.csv 235 | 236 | others 237 | ====== 238 | 239 | .. csv-table:: 240 | :header-rows: 1 241 | :widths: 10, 5, 5, 5, 15, 40 242 | :file: objectgroup_others.csv 243 | 244 | power_mng 245 | ========= 246 | 247 | .. csv-table:: 248 | :header-rows: 1 249 | :widths: 10, 5, 5, 5, 15, 40 250 | :file: objectgroup_power_mng.csv 251 | 252 | p_rec 253 | ===== 254 | 255 | .. csv-table:: 256 | :header-rows: 1 257 | :widths: 10, 5, 5, 5, 15, 40 258 | :file: objectgroup_p_rec.csv 259 | 260 | prim_sm 261 | ======= 262 | 263 | .. csv-table:: 264 | :header-rows: 1 265 | :widths: 10, 5, 5, 5, 15, 40 266 | :file: objectgroup_prim_sm.csv 267 | 268 | rb485 269 | ===== 270 | 271 | .. csv-table:: 272 | :header-rows: 1 273 | :widths: 10, 5, 5, 5, 15, 40 274 | :file: objectgroup_rb485.csv 275 | 276 | switch_on_cond 277 | ============== 278 | 279 | .. csv-table:: 280 | :header-rows: 1 281 | :widths: 10, 5, 5, 5, 15, 40 282 | :file: objectgroup_switch_on_cond.csv 283 | 284 | temperature 285 | =========== 286 | 287 | .. csv-table:: 288 | :header-rows: 1 289 | :widths: 10, 5, 5, 5, 15, 40 290 | :file: objectgroup_temperature.csv 291 | 292 | wifi 293 | ==== 294 | 295 | .. csv-table:: 296 | :header-rows: 1 297 | :widths: 10, 5, 5, 5, 15, 40 298 | :file: objectgroup_wifi.csv 299 | -------------------------------------------------------------------------------- /docs/protocol_event_table.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _protocol-event-table: 3 | 4 | ########### 5 | Event Table 6 | ########### 7 | 8 | The event table (OID ``0x6F3876BC``) comes as a long response. It is queried by **writing** a UNIX timestamp to the 9 | OID, and the device will respond with a list of entries that occured *earlier* than that timestamp. The amount of 10 | entries varies, so in order to receive a larger amount multiple queries are required, by using the timestamp of the 11 | "oldest" entry as timestamp for the next query. 12 | 13 | Similar to the time series metrics, the first element is the UNIX timestamp that was used in the request, then a table 14 | follows. Each table row consists of five UINT32 values, though not all of them are used with all entry types. Each row 15 | represents a single entry in the table. The first element (element 0) is the **identifier** of the event, it appears to 16 | only contain a single byte. The next element is a UNIX timestamp that denotes either the precise moment the event 17 | occured for some types, or the begin of an event. The meaning of the other three elements can be read from the table 18 | below, as they are dependant on the type of event. 19 | 20 | Data format 21 | *********** 22 | All elements are 4 bytes wide. The response looks like this: 23 | 24 | +--------+-------------------------------------------------------------------------------------------------------+ 25 | | Number | Meaning | 26 | +========+=======================================================================================================+ 27 | | 0 | Query timestamp, repeated from the write request. | 28 | +--------+-------------------------------------------------------------------------------------------------------+ 29 | | 1 | ASCII character denoting the type of the entry (see table below). | 30 | +--------+-------------------------------------------------------------------------------------------------------+ 31 | | 2 | Timestamp denoting the start of the event (for ranged events) or the point in time the event occured. | 32 | +--------+-------------------------------------------------------------------------------------------------------+ 33 | | 3 | Type-specific value. This is the end timestamp for ranged types, but also carries other information. | 34 | +--------+-------------------------------------------------------------------------------------------------------+ 35 | | 4 | Type-specific value. | 36 | +--------+-------------------------------------------------------------------------------------------------------+ 37 | | 5 | Type specific value. | 38 | +--------+-------------------------------------------------------------------------------------------------------+ 39 | | 6 | ASCII character for the type of the next entry | 40 | +--------+-------------------------------------------------------------------------------------------------------+ 41 | | 7 | Timestamp of the event. | 42 | +--------+-------------------------------------------------------------------------------------------------------+ 43 | | ... | ... | 44 | +--------+-------------------------------------------------------------------------------------------------------+ 45 | 46 | Unless an error occurs, which may happen when the device receives another command while working on this request, 47 | resulting in a correct CRC but incomplete data, the structure is always `` * 5 + 1`` 4-byte 48 | sequences, the extra element is the timestamp at the very beginning. 49 | 50 | Event overview 51 | ************** 52 | 53 | Not all events have been observed yet, and as such the table is incomplete. Gaps in events may or may not actually 54 | contain event data at all. 55 | 56 | +----------+---------------+--------------+-------------+----------------------+ 57 | | ID (Hex) | Element 2 | Element 3 | Element 4 | Name | 58 | +==========+===============+==============+=============+======================+ 59 | | ``0x31`` | End timestamp | Float value | unknown | ``UL_UNDER_L1_LV2`` | 60 | +----------+---------------+--------------+-------------+----------------------+ 61 | | ``0x32`` | | | | | 62 | +----------+---------------+--------------+-------------+----------------------+ 63 | | ``0x33`` | | | | | 64 | +----------+---------------+--------------+-------------+----------------------+ 65 | | ``0x34`` | | | | | 66 | +----------+---------------+--------------+-------------+----------------------+ 67 | | ``0x35`` | | | | | 68 | +----------+---------------+--------------+-------------+----------------------+ 69 | | ``0x36`` | | | | | 70 | +----------+---------------+--------------+-------------+----------------------+ 71 | | ``0x37`` | | | | | 72 | +----------+---------------+--------------+-------------+----------------------+ 73 | | ``0x38`` | | | | | 74 | +----------+---------------+--------------+-------------+----------------------+ 75 | | ``0x39`` | | | | | 76 | +----------+---------------+--------------+-------------+----------------------+ 77 | | ``0x3a`` | | | | | 78 | +----------+---------------+--------------+-------------+----------------------+ 79 | | ``0x3b`` | | | | | 80 | +----------+---------------+--------------+-------------+----------------------+ 81 | | ``0x3c`` | | | | | 82 | +----------+---------------+--------------+-------------+----------------------+ 83 | | ``0x3d`` | | | | | 84 | +----------+---------------+--------------+-------------+----------------------+ 85 | | ``0x3e`` | | | | | 86 | +----------+---------------+--------------+-------------+----------------------+ 87 | | ``0x3f`` | | | | | 88 | +----------+---------------+--------------+-------------+----------------------+ 89 | | ``0x40`` | | | | | 90 | +----------+---------------+--------------+-------------+----------------------+ 91 | | ``0x41`` | | | | | 92 | +----------+---------------+--------------+-------------+----------------------+ 93 | | ``0x42`` | | | | | 94 | +----------+---------------+--------------+-------------+----------------------+ 95 | | ``0x43`` | | | | | 96 | +----------+---------------+--------------+-------------+----------------------+ 97 | | ``0x44`` | | | | | 98 | +----------+---------------+--------------+-------------+----------------------+ 99 | | ``0x45`` | | | | | 100 | +----------+---------------+--------------+-------------+----------------------+ 101 | | ``0x46`` | | | | | 102 | +----------+---------------+--------------+-------------+----------------------+ 103 | | ``0x47`` | | | | | 104 | +----------+---------------+--------------+-------------+----------------------+ 105 | | ``0x48`` | | | | | 106 | +----------+---------------+--------------+-------------+----------------------+ 107 | | ``0x49`` | | | | | 108 | +----------+---------------+--------------+-------------+----------------------+ 109 | | ``0x4a`` | | | | | 110 | +----------+---------------+--------------+-------------+----------------------+ 111 | | ``0x4b`` | | | | | 112 | +----------+---------------+--------------+-------------+----------------------+ 113 | | ``0x4c`` | | | | | 114 | +----------+---------------+--------------+-------------+----------------------+ 115 | | ``0x4d`` | | | | | 116 | +----------+---------------+--------------+-------------+----------------------+ 117 | | ``0x4e`` | | | | | 118 | +----------+---------------+--------------+-------------+----------------------+ 119 | | ``0x4f`` | End timestamp | Float value | ignored | ``SW_ON_UMIN_L1`` | 120 | +----------+---------------+--------------+-------------+----------------------+ 121 | | ``0x50`` | End timestamp | Float value | ignored | ``SW_ON_UMAX_L1`` | 122 | +----------+---------------+--------------+-------------+----------------------+ 123 | | ``0x52`` | End timestamp | Float value | ignored | ``SW_ON_FMAX_L1`` | 124 | +----------+---------------+--------------+-------------+----------------------+ 125 | | ``0x53`` | End timestamp | Float value | ignored | ``SW_ON_UMIN_L2`` | 126 | +----------+---------------+--------------+-------------+----------------------+ 127 | | ``0x54`` | End timestamp | Float value | ignored | ``SW_ON_UMAX_L2`` | 128 | +----------+---------------+--------------+-------------+----------------------+ 129 | | ``0x55`` | | | | | 130 | +----------+---------------+--------------+-------------+----------------------+ 131 | | ``0x56`` | | | | | 132 | +----------+---------------+--------------+-------------+----------------------+ 133 | | ``0x57`` | End timestamp | Float value | ignored | ``SW_ON_UMIN_L3`` | 134 | +----------+---------------+--------------+-------------+----------------------+ 135 | | ``0x58`` | End timestamp | Float value | ignored | ``SW_ON_UMAX_L3`` | 136 | +----------+---------------+--------------+-------------+----------------------+ 137 | | ``0x59`` | End timestamp | unknown | ignored | ``SURGE`` | 138 | +----------+---------------+--------------+-------------+----------------------+ 139 | | ``0x5a`` | End timestamp | Case number | unknown | ``NO_GRID`` | 140 | +----------+---------------+--------------+-------------+----------------------+ 141 | | ``0x5b`` | | | | | 142 | +----------+---------------+--------------+-------------+----------------------+ 143 | | ``0x5c`` | | | | | 144 | +----------+---------------+--------------+-------------+----------------------+ 145 | | ``0x5d`` | | | | | 146 | +----------+---------------+--------------+-------------+----------------------+ 147 | | ``0x5e`` | | | | | 148 | +----------+---------------+--------------+-------------+----------------------+ 149 | | ``0x5f`` | | | | | 150 | +----------+---------------+--------------+-------------+----------------------+ 151 | | ``0x60`` | | | | | 152 | +----------+---------------+--------------+-------------+----------------------+ 153 | | ``0x61`` | End timestamp | Case number | unknown | ``PHASE_POS`` | 154 | +----------+---------------+--------------+-------------+----------------------+ 155 | | ``0x62`` | | | | | 156 | +----------+---------------+--------------+-------------+----------------------+ 157 | | ``0x63`` | | | | | 158 | +----------+---------------+--------------+-------------+----------------------+ 159 | | ``0x64`` | End timestamp | Float value | | ``BAT_OVERVOLTAGE`` | 160 | +----------+---------------+--------------+-------------+----------------------+ 161 | | ``0x65`` | | | | | 162 | +----------+---------------+--------------+-------------+----------------------+ 163 | | ``0x66`` | | | | | 164 | +----------+---------------+--------------+-------------+----------------------+ 165 | | ``0x67`` | | | | | 166 | +----------+---------------+--------------+-------------+----------------------+ 167 | | ``0x68`` | | | | | 168 | +----------+---------------+--------------+-------------+----------------------+ 169 | | ``0x69`` | | | | | 170 | +----------+---------------+--------------+-------------+----------------------+ 171 | | ``0x6a`` | | | | | 172 | +----------+---------------+--------------+-------------+----------------------+ 173 | | ``0x6b`` | End timestamp | unknown | unknown | ``CAN_TIMEOUT`` | 174 | +----------+---------------+--------------+-------------+----------------------+ 175 | | ``0x6c`` | | | | | 176 | +----------+---------------+--------------+-------------+----------------------+ 177 | | ``0x6d`` | | | | | 178 | +----------+---------------+--------------+-------------+----------------------+ 179 | | ``0x6e`` | | | | | 180 | +----------+---------------+--------------+-------------+----------------------+ 181 | | ``0x6f`` | | | | | 182 | +----------+---------------+--------------+-------------+----------------------+ 183 | | ``0x70`` | | | | | 184 | +----------+---------------+--------------+-------------+----------------------+ 185 | | ``0x71`` | | | | | 186 | +----------+---------------+--------------+-------------+----------------------+ 187 | | ``0x72`` | End timestamp | Error code | | ``BAT_INTERN`` | 188 | +----------+---------------+--------------+-------------+----------------------+ 189 | | ``0x73`` | Object ID | Old value | New value | ``PRM_CHANGE`` | 190 | +----------+---------------+--------------+-------------+----------------------+ 191 | | ``0x74`` | | | | | 192 | +----------+---------------+--------------+-------------+----------------------+ 193 | | ``0x75`` | | | | | 194 | +----------+---------------+--------------+-------------+----------------------+ 195 | | ``0x76`` | End timestamp | unknown | ignored | ``RESET`` | 196 | +----------+---------------+--------------+-------------+----------------------+ 197 | | ``0x77`` | 0 | Old version | New version | ``UPDATE`` | 198 | +----------+---------------+--------------+-------------+----------------------+ 199 | | ``0x78`` | End timestamp | unknown | unknown | ``FRT_OVERVOLTAGE`` | 200 | +----------+---------------+--------------+-------------+----------------------+ 201 | | ``0x79`` | End timestamp | unknown | unknown | ``FRT_UNDERVOLTAGE`` | 202 | +----------+---------------+--------------+-------------+----------------------+ 203 | 204 | Event details 205 | ************* 206 | For each of the event types observed, this section provides a short explanation and an example of the corresponding 207 | message taken from the official app. 208 | 209 | 210 | PHASE_POS 211 | ========= 212 | 213 | A ranged event, containing a "case number" and an unknown element. The app reports this as follows: 214 | 215 | :: 216 | 217 | Duration: - 218 | Phase position error (not 120° as expected) 219 | case 220 | 221 | 222 | BAT_OVERVOLTAGE 223 | =============== 224 | A ranged event with the floating point number of the battery voltage at the time of the event start as only parameter. 225 | The app reports this as follows: 226 | 227 | :: 228 | 229 | Duration: 00:00:22 230 | Battery overvoltage 231 | U = V 232 | 233 | 234 | CAN_TIMEOUT 235 | =========== 236 | Ranged event reporting a timeout in the CAN-bus communication with a component. It is not known yet if there is a 237 | separate RS485 timeout event or if these events are handled here as well. The app reports this as follows: 238 | 239 | :: 240 | 241 | Duration: 00:00:22 242 | "CAN communication timeout with battery" 243 | 244 | 245 | BAT_INTERN 246 | ========== 247 | This ranged event reports an abnormal condition with the battery stack. The payload seems to contain an error code and 248 | another unknown element. The app reports this as follows (example, there are other messages possible): 249 | 250 | :: 251 | 252 | Duration: 00:00:22 253 | Internal battery error () 254 | Battery 0 255 | UI-Board (0x123) 256 | Error class 1: Charge overcurrent 257 | 258 | 259 | PRM_CHANGE 260 | ========== 261 | This event has no end timestamp as it reports a singular event: the change of a parameter. Thus, it carries the object 262 | ID of the changed object as element 2 and the old and new values as elements 3 and 4. The meaning of the values depends 263 | on the object ID, obviously, as the raw values are reported. To make a meaning of them, the values have to be decoded 264 | according to the data types associated with the object ID. 265 | 266 | The app automatically performs a lookup and translates the object ID to its name in many but not all cases. For 267 | parameters that are unlikely to be changed by the user, it reports the name of the object ID. The app does not 268 | interpret the values, so boolean values are reported as ``0`` or ``1``, for example. 269 | 270 | :: 271 | 272 | Duration: - 273 | Parameter changed 274 | "Enable rescan for global MPP on solar generator A": 0 --> 1 275 | 276 | :: 277 | 278 | Duration: - 279 | Parameter changed 280 | "display_struct.variate_contrast": 1 --> 0 281 | 282 | 283 | RESET 284 | ===== 285 | This ranged event reports the reset or restart of the system. This is, for example, done after a firmware update. The 286 | app reports this as follows: 287 | 288 | :: 289 | 290 | Duration: 00:00:22 291 | System start 292 | 293 | 294 | UPDATE 295 | ====== 296 | A non-ranged event, reporting the successful update of the controller software. It includes the old and new software 297 | version as alements 3 and 4, element 2 contains the number ``0``. The app reports this like so: 298 | 299 | :: 300 | 301 | 302 | Duration: - 303 | Update 304 | 305 | 306 | FRT_UNDERVOLTAGE 307 | ================ 308 | A ranged event, containing two unknown parameters that are not shown in the app. 309 | 310 | :: 311 | 312 | Duration: 00:00:22 313 | FRT under-voltage 314 | 315 | 316 | FRT_OVERVOLTAGE 317 | =============== 318 | A ranged event, containing two unknown parameters that are not shown in the app. 319 | 320 | :: 321 | 322 | Duration: 00:00:22 323 | FRT over-voltage 324 | 325 | 326 | SW_ON_UMIN_L1 327 | ============= 328 | Ranged event, containing the voltage as element 3. 329 | 330 | :: 331 | 332 | Duration: 00:00:22 333 | Switching On Conditions Umin phase 1 334 | U = V 335 | 336 | SW_ON_UMAX_L1 337 | ============= 338 | Ranged event, carrying the voltage level as element 3. 339 | 340 | :: 341 | 342 | Duration: 00:00:22 343 | Switching On Conditions Umax phase 1 344 | U = V 345 | 346 | 347 | SW_ON_FMAX_L1 348 | ============= 349 | Ranged event, caryying the frequency as element 3. This seems to be the only frequency level event, as there is no room 350 | in the type list for FMAX events for the other two phases. Also, some inverters are capable of putting all power into 351 | one single phase if so desired. 352 | 353 | :: 354 | 355 | Duration: 00:00:22 356 | Switching On Conditions Fmax phase 1 357 | f = Hz 358 | 359 | SW_ON_UMIN_L2 360 | ============= 361 | See ``SW_ON_UMIN_L1``. 362 | 363 | SW_ON_UMAX_L2 364 | ============= 365 | See ``SW_ON_UMAX_L1``. 366 | 367 | SW_ON_UMIN_L3 368 | ============= 369 | See ``SW_ON_UMIN_L1``. 370 | 371 | SW_ON_UMAX_L3 372 | ============= 373 | See ``SW_ON_UMIN_L1``. 374 | 375 | 376 | SURGE 377 | ===== 378 | A ranged event reporting a surge event. An unknown value is transported in element 3. The app reports this as follows: 379 | 380 | :: 381 | 382 | Duration: 00:00:01 383 | Phase failure detected 384 | 385 | NO_GRID 386 | ======= 387 | A ranged event reporting a loss of the power grid. The elements 3 and 4 are the same as for ``PHASE_POS``, but the case 388 | number is not shown by the app. 389 | 390 | :: 391 | 392 | Duration: 00:00:00 393 | Reserved 394 | -------------------------------------------------------------------------------- /docs/protocol_overview.rst: -------------------------------------------------------------------------------- 1 | 2 | ######## 3 | Overview 4 | ######## 5 | 6 | The protocol is based on TCP and is based around commands targeting object IDs that work similar to OIDs in SNMP. 7 | Messages are protected against transfer failures by means of a 2-byte CRC checksum. The protocol in itself isn't very 8 | complicated, but there are some things like escaping, length calculation and plant communication to look out for. 9 | 10 | There is no method to tell that a communication was not understood at the protocol level, meaning that it is up to the 11 | implementor to figure out a way to let the client know that something was not understood. Vendor devices may simply 12 | fail to respond or respond with an empty payload, or do something else. It is advised to pay attention to the checksums 13 | and errors during payload decoding. Vendor devices do facilitate special OIDs that contain information about their 14 | state, but that is an implementation detail of the specific devices. If implementing an application, one could specify 15 | some OIDs that can be used to report if the previous command resulted in an error, for example. 16 | 17 | Protocol Data Units 18 | ******************* 19 | The Protocol Data Units (PDU) are comprised of: 20 | 21 | #. A start token (``+``) 22 | #. The command byte 23 | #. The length of the ID and data payload 24 | #. For plant communication, the address, omitted for standard frames 25 | #. The object ID 26 | #. Payload (optional) 27 | #. CRC16 checksum 28 | 29 | Data is encoded as big endian, a leading ``0x00`` before the start token is allowed. 30 | 31 | +----------------+-------+---------------------------------------+ 32 | | Element | Bytes | Remarks | 33 | +================+=======+=======================================+ 34 | | Start token | 1 | ``+`` character, ``0x2b``. | 35 | +----------------+-------+---------------------------------------+ 36 | | Command | 1 | | 37 | +----------------+-------+---------------------------------------+ 38 | | Payload length | 1 | All but long read / long write | 39 | | +-------+---------------------------------------+ 40 | | | 2 | Long read / long write | 41 | +----------------+-------+---------------------------------------+ 42 | | Address | 4 | For plant communication | 43 | | +-------+---------------------------------------+ 44 | | | 0 | Omitted for standard frames | 45 | +----------------+-------+---------------------------------------+ 46 | | Object ID | 4 | | 47 | +----------------+-------+---------------------------------------+ 48 | | Payload | N | Payload is optional for some commands | 49 | +----------------+-------+---------------------------------------+ 50 | | CRC16 | 2 | | 51 | +----------------+-------+---------------------------------------+ 52 | 53 | .. hint:: 54 | 55 | The OIDs are actually an implementation detail of the device. The protocol only defines that they are 4 bytes in 56 | length. All other details like the data type, whether a payload can be used and so on are up to the implementer, so 57 | in order to implement your own application, you would simply define your own OIDs with associated data types instead 58 | of using the ones in the :ref:`registry` or the examples. 59 | 60 | Escaping 61 | ======== 62 | 63 | Certain characters are escaped by inserting the escape token ``-`` (``0x2d``) into the stream before the byte that 64 | requires escaping. When the start token (``+``) or escape token is encountered in the data stream (unless it's the very 65 | first byte for the start token), the escape token is inserted before the token that it escapes. On decoding, when the 66 | escape token is encountered, the next character is interpreted as data and not as start token or escape token. Thus, if 67 | the task is to encode a plus sign (usually in a string), then a minus is added *before* the plus sign to escape it. 68 | 69 | Checksum 70 | ======== 71 | The checksum algorithm used is a special version of CRC16 using a CCITT polynom (``0x1021``). It varies from other 72 | implementation by appending a NULL byte to the input if its length is uneven before commencing with the calculation. 73 | 74 | 75 | Commands 76 | ******** 77 | There are two groups of commands: *Standard* communication commands that are sent to a device and the device replies, 78 | as well as *Plant* communication commands, which are standard commands ORed with ``0x40``. 79 | 80 | Commands not listed here are either not known or are reserved, and should not be used with the devices as it is not 81 | known what effect this could have. 82 | 83 | ======================= ============= ====================================================== 84 | Command Value Description 85 | ======================= ============= ====================================================== 86 | READ ``0x01`` Request the current value of an object ID. No payload. 87 | WRITE ``0x02`` Write the payload to the object ID. 88 | LONG_WRITE ``0x03`` When writing "long" payloads. 89 | *RESERVED* ``0x04`` 90 | RESPONSE ``0x05`` Normal response to a read or write command. 91 | LONG_RESPONSE ``0x06`` Response with a "long" payload. 92 | *RESERVED* ``0x07`` 93 | READ_PERIODICALLY ``0x08`` Request automatic, periodic sending of an OIDs value. 94 | *Reserved* ``0x09-0x40`` 95 | PLANT_READ ``0x41`` *READ* for plant communication. 96 | PLANT_WRITE ``0x42`` *WRITE* for plant communication. 97 | PLANT_LONG_WRITE ``0x43`` *LONG_WRITE* for plant communication. 98 | *RESERVED* ``0x44`` 99 | PLANT_RESPONSE ``0x45`` *RESPONSE* for plant communication. 100 | PLANT_LONG_RESPONSE ``0x46`` *LONG_RESPONSE* for plant communication. 101 | *RESERVED* ``0x47`` 102 | PLANT_READ_PERIODICALLY ``0x48`` *READ_PERIODICALLY* for plant communication. 103 | EXTENSION ``0x3c`` Unknown, see below. 104 | ======================= ============= ====================================================== 105 | 106 | The EXTENSION command 107 | ===================== 108 | EXTENSION does not follow the semantics of other commands and cannot be parsed by *rctclient*. It is believed to be a 109 | single-byte payload; a frame often observed is ``0x2b3ce1``, which is sent by the official app uppon connecting to a 110 | device to "switch to COM protocol". In this case, ``0xe1`` is the commands payload, and a normal frame follows 111 | immediately after, which leads to the conclusion of this command always being three bytes in length. 112 | 113 | READ_PERIODICALLY 114 | ================= 115 | Registers a OID for being sent periodically. The device will send the current value of the OID at an interval defined 116 | in ``pas.period`` (see :ref:`registry`). Up to 64 OIDs can be registered with vendor devices, but the protocol does not 117 | impose a limit, and all registered OIDs will be served at the same interval setting. When sending this command, the 118 | device immediately responds with the current value of the OID, and will then periodically send the current value. 119 | 120 | To disable, set ``pas.period`` to 0, which clears the list of registered OIDs, effectively disabling the feature. No 121 | method exists for removing a single OID, one has to clear it, then set ``pas.period`` and re-register all desired OIDs. 122 | 123 | .. warning:: 124 | 125 | The implementation has not been tested yet, please don't hesitate to open an issue if you run into problems or have 126 | more insight into the matter. 127 | 128 | Frame length 129 | ************ 130 | 131 | The frame length is 1 byte for all commands except *LONG_RESPONSE* and *LONG_WRITE* and their *PLANT_* counterparts, 132 | which use 2 bytes (most siginificant byte first). The length denotes how many bytes of data follow it. Escape tokens 133 | are not counted, and it does not include the two-byte header before it (start token and command) and does also not 134 | include the two-byte CRC16 at the end of the frame. In order to fully receive a frame, after reversing any escaping, 135 | the buffer should therefor hold ``2 + length + 2`` bytes. 136 | 137 | Plant communication 138 | ******************* 139 | With plant communication, one device acts as plant leader and relays commands addressed at subordinate devices to them 140 | and their responses back to the client. Vendor devices need to be linked together, each has its own ``address``. 141 | 142 | To use plant communication, use the ``PLANT_*`` variations of the normal commands (``READ`` → ``PLANT_READ`` and so on) 143 | and supply the ``address`` property. The leader device forwards the command to the device that has the address set, all 144 | other devices ignore the frame. The answer from the addressed device is then sent back to the client by the leader, 145 | with a ``PLANT_*`` response command and the ``address`` set to that of the addressed device. 146 | 147 | .. warning:: 148 | 149 | Plant communication has not been tested and the implementation simply follows what is known. The authors do not have 150 | a setup to test this kind of communication. We'd greatly appreciate traffic dumps of actual plant communication or 151 | feedback if it works or not. 152 | 153 | Frame by example 154 | **************** 155 | The following is a dissection of a frame sent to the device (read request) and its response from the device. 156 | 157 | Request 158 | ======= 159 | 160 | Setting: 161 | 162 | * *READ* request, so command is ``0x01`` 163 | * The OID ``battery.soc`` is ``0x959930BF`` 164 | * No payload and no address and nothing to escape. 165 | 166 | :: 167 | 168 | Data: 2b 01 04 959930bf 0d65 169 | 170 | ID: 1 2 3 4 5 171 | 172 | == ============ ========================================================= 173 | ID Bytes Description 174 | == ============ ========================================================= 175 | 1 ``2b`` Start token 176 | 2 ``01`` Command: *READ* 177 | 3 ``04`` Length of the data that follows, it's the OID of 4 bytes. 178 | 4 ``959930bf`` Data, which in this example consists of the OID only. 179 | 5 ``0d65`` CRC16 checksum. 180 | == ============ ========================================================= 181 | 182 | Response 183 | ======== 184 | 185 | The response for the command (read battery state of charge) is disected below. The string has been split up for ease of 186 | reading, but it is a single byte stream. 187 | 188 | The raw response looks like this (in hexadecimal): ``002b0508959930bf3e97b1919c86`` 189 | :: 190 | 191 | Data: 00 2b 05 08 959930bf 3e97b191 9c86 192 | 193 | ID: 1 2 3 4 5 6 7 194 | 195 | == ============ ============================================================================== 196 | ID Bytes Description 197 | == ============ ============================================================================== 198 | 1 ``00`` Data before the start of the command. It is ignored. 199 | 2 ``2b`` Start token, all data before this is ignored. 200 | 3 ``05`` Command, this is a `RESPONSE`. 201 | 4 ``08`` Length field, 4 byte OID and 4 byte payload. 202 | 5 ``959930bf`` The OID this response carries. 203 | 6 ``3e97b191`` Payload data, as per the OID this is a big endian float value of roughly 0.296 204 | 7 ``9x86`` CRC16 checksum. 205 | == ============ ============================================================================== 206 | 207 | The payload in this example is a big endian floating point number. The data type can be looked up in the 208 | :ref:`Registry`. 209 | -------------------------------------------------------------------------------- /docs/protocol_timeseries.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _protocol-timeseries: 3 | 4 | ########## 5 | Timeseries 6 | ########## 7 | 8 | The ``logger`` group of object IDs (with the exception of the event log) returns time series data (values with an 9 | associated timestamp taken at regular intervals) as a long response. Similar to th event log, the data is queried by 10 | writing a UNIX timestamp to the object ID, uppon which the device returns data that was logged **before** that 11 | timestamp. The amount of data varies similar to the event log, so in order to get a full days of data for a single time 12 | series, an average of 7 queries are required. 13 | 14 | As with the event table, the first element is a unix timestamp, repeating the value written to the object ID. Then 15 | follows a list of pairs, first a UNIX timestamp and then the floating point value. 16 | 17 | Thus, the request data type is ``INT32`` for the timestamp, and the response data type is the special ``TIMESTAMP`` 18 | data type to cause the :func:`~rctclient.utils.decode_value` function to correctly parse the data into a data 19 | structure. 20 | 21 | Data resolution 22 | *************** 23 | The data resolution varies between object IDs. Object IDs with ``minutes`` in their name, such as 24 | ``logger.day_egrid_load_log_ts`` have a resolution of 5 minutes. 25 | 26 | +-------------+------------+ 27 | | Time part | Resolution | 28 | +=============+============+ 29 | | ``minutes`` | 5 minutes | 30 | +-------------+------------+ 31 | | ``day`` | | 32 | +-------------+------------+ 33 | | ``month`` | | 34 | +-------------+------------+ 35 | | ``year`` | | 36 | +-------------+------------+ 37 | 38 | Data storage 39 | ************ 40 | The devices use some sort of ring buffer to manage their data, meaning that old elements are overwritten as new data is 41 | stored. For ``minutes`` graphs, this leads to ~90 days of history. 42 | 43 | Data format 44 | *********** 45 | All elements are always 4-bytes long and are either UINT32 for the timestamp or FLOAT for the value. 46 | 47 | +--------+-----------------------------------------------------+ 48 | | Number | Meaning | 49 | +========+=====================================================+ 50 | | 0 | Query timestamp, repeated from the write request. | 51 | +--------+-----------------------------------------------------+ 52 | | 1 | Timestamp 0, associated with the following element. | 53 | +--------+-----------------------------------------------------+ 54 | | 2 | Value 0. | 55 | +--------+-----------------------------------------------------+ 56 | | 3 | Timestamp 1. | 57 | +--------+-----------------------------------------------------+ 58 | | 4 | Value 1. | 59 | +--------+-----------------------------------------------------+ 60 | | 5 | Timestamp 2. | 61 | +--------+-----------------------------------------------------+ 62 | | ... | ... | 63 | +--------+-----------------------------------------------------+ 64 | 65 | Unless an error occurs, the structure is always `` * 2 + 1`` 4-byte sequences, the extra sequence 66 | being the timestamp at the very beginning. 67 | -------------------------------------------------------------------------------- /docs/simulator.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _simulator: 3 | 4 | ######### 5 | Simulator 6 | ######### 7 | 8 | To aid in developing tools or this module itself, a simulator offers an easy way to safely test code without 9 | interfacing with real hardware and potentially causing problems. The simulator can be run by using the subcommand 10 | ``simulator`` of the :ref:`cli`. 11 | 12 | Starting the simulator 13 | ********************** 14 | The simulator will, by default, bind to `localhost` on port `8899`, which is the default port for the protocol. These 15 | can be changed using the ``--host`` and ``--port`` options. To stop the simulator, press `Ctrl+c` on the terminal. 16 | 17 | Without ``--verbose``, the simulator won't output anything to the terminal. Adding the flag yields output: 18 | 19 | .. code-block:: shell-session 20 | 21 | $ rctclient simulator --verbose 22 | INFO:rctclient.simulator:Waiting for client connections 23 | INFO:rctclient.simulator:connection accepted: ('127.0.0.1', 55038) 24 | INFO:rctclient.simulator.socket_thread.55038:Read : #394 0x90B53336 temperature.sink_temp_power_reduction -> 2b050890b53336000000006157 25 | 26 | For a better view on the inner workings, supply the ``--debug`` parameter to ``rctclient``, but be warned: this will 27 | print a lot of text: 28 | 29 | .. code-block:: shell-session 30 | 31 | $ rctclient --debug simulator 32 | 2020-10-02 12:14:09,935 - rctclient.cli - INFO - rctclient CLI starting 33 | 2020-10-02 12:14:09,936 - rctclient.simulator - INFO - Waiting for client connections 34 | 2020-10-02 12:14:12,486 - rctclient.simulator - DEBUG - connection accepted: ('127.0.0.1', 55044) 35 | 2020-10-02 12:14:12,486 - rctclient.simulator.socket_thread.55044 - DEBUG - Read 9 bytes: 2b010490b533361775 36 | 2020-10-02 12:14:12,486 - rctclient.simulator.socket_thread.55044 - DEBUG - Frame consumed 9 bytes 37 | 2020-10-02 12:14:12,486 - rctclient.simulator.socket_thread.55044 - DEBUG - Complete frame: 38 | 2020-10-02 12:14:12,486 - rctclient.simulator.socket_thread.55044 - INFO - Read : #394 0x90B53336 temperature.sink_temp_power_reduction -> 2b050890b53336000000006157 39 | 2020-10-02 12:14:12,486 - rctclient.simulator.socket_thread.55044 - DEBUG - Sending frame with 13 bytes 0x2b050890b53336000000006157 40 | 2020-10-02 12:14:12,487 - rctclient.simulator.socket_thread.55044 - DEBUG - Closing connection 41 | 2020-10-02 12:14:16,063 - rctclient.simulator - DEBUG - Keyboard interrupt, shutting down 42 | 43 | How it works 44 | ************ 45 | The simulator acts as a server for the protocol. It receives the commands, decodes them and then creates a new frame as 46 | answer to the command. So far, only read-requests for non-plant communication are implemented. 47 | 48 | Using the :class:`~rctclient.registry.Registry`, it looks up information about the object ID received from the client 49 | software and uses the ``sim_data`` property to craft a valid response. For trivial types, a sensible default is used: 50 | 51 | * Boolean types return ``False`` 52 | * Floating point types return ``0.0`` 53 | * Integer types return ``0`` 54 | * Strings return ``ASDFGH`` 55 | 56 | Additionally, when creating the :class:`~rctclient.registry.ObjectInfo` objects, a value can be set using the 57 | ``sim_data`` keyword argument to provide a more fitting result. 58 | 59 | Limitations 60 | *********** 61 | The simulator won't return invalid data, that means that it always calculates the correct checksum and won't send 62 | incomplete data. When developing against the simulator, bear in mind the limitations of the real world device: 63 | 64 | If a request is received while serving another request, the response that the device is currently prepairing is cut at 65 | the current point and a CRC checksum is computed. This results in frames that can't be decoded, but have a valid 66 | checksum for the incomplete payload. 67 | 68 | For now, it only returns normal replies, even though some replies should be long replies. This may be fixed in the 69 | future. 70 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _usage: 3 | 4 | ##### 5 | Usage 6 | ##### 7 | 8 | This page describes the basic concepts of the module and how to use the different parts. See also the short 9 | :ref:`security` section below. 10 | 11 | Parts overview 12 | ************** 13 | 14 | The module offers a range of classes and methods that allow for a relatively modular workflow. This makes it possible 15 | to write a client application (the main goal of the module), but also a :ref:`simulator` to develop against. With a 16 | little extra work, it is even possible to read from a *pcap* file and check past communication captured using *tcpdump* 17 | or *wireshark*. 18 | 19 | One thing that is not provided is methods for network communication, it is a 20 | `sans I/O `_ library. This effectively means that the user has to bring in their own 21 | code for doing the network communication (though with the :ref:`CLI` there's some optional support for doing simple 22 | calls). This might sound odd at first, but it allows for the library to be used in a very modular way, by not dictating 23 | how network-communication is handled, and it frees the developers of this library from having to support different 24 | communication schemes. There are some examples further down that actually deal with network communication. 25 | 26 | Object IDs (OIDs) 27 | ================= 28 | The protocol revolves around Object IDs (OIDs) that work similar to OIDs in *snmp*, in that they are an address that is 29 | targeted by a command (read value from OID, send value to OID) and that is referenced in the response to such a 30 | command. To deepen the similarities, a :ref:`registry` is provided that acts like a MIB definition file and enriches 31 | the raw OIDs with human-readable names as well as data types for decoding/encoding and so on. 32 | 33 | This information is kept in :class:`~rctclient.registry.ObjectInfo` objects inside a 34 | :class:`~rctclient.registry.Registry` instance conveniently provided as `rctclient.registry.REGISTRY` for easier 35 | consumption. 36 | 37 | Looking at the information objects, they contain a ``name`` such as ``battery.soc``, an optional description ``SOC 38 | (State of charge)`` and most importantly, the ``request_data_type`` and ``response_data_type`` fields. These fields are 39 | used to specify how to encode or decode values for the particular OID. In most cases, the response type is the same as 40 | the request type, but there are a few exceptions: :ref:`protocol-timeseries` and :ref:`protocol-event-table`. 41 | 42 | Registry 43 | ======== 44 | The :class:`rctclient.registry.Registry` class maintains a list of OIDs for communicating with vendor devices. It isn't 45 | required for own implementations of a server of this protocol, where one would simply define own OIDs as needed. 46 | 47 | As the list is quite long and for the users convenience, a module-scope instance is available as ``REGISTRY``. 48 | 49 | Most of the examples will assume an import like the following: 50 | 51 | .. code-block:: python 52 | 53 | from rctclient.registry import REGISTRY as R 54 | 55 | This makes the registry available as ``R``. It provides a set of functions to query 56 | :class:`~rctclient.registry.ObjectInfo` instances that describe OIDs as explained above. A complete list of the OIDs 57 | shipped with the module is available at the :ref:`registry` page. 58 | 59 | The most commonly used functions are :func:`~rctclient.registry.Registry.get_by_id` and 60 | :func:`~rctclient.registry.Registry.get_by_name` that return a `ObjectInfo` instance for the OID or the name, observe: 61 | 62 | .. code-block:: pycon 63 | 64 | >>> from rctclient.registry import REGISTRY as R 65 | >>> oinfo_name = R.get_by_name('battery.soc') 66 | >>> oinfo_name 67 | 68 | >>> oinfo_name.description 69 | 'SOC (State of charge)' 70 | >>> oinfo_id = R.get_by_id(0x959930BF) 71 | >>> oinfo_id 72 | 73 | 74 | For some OIDs, additional information such as a textual description or a unit like ``V`` for volts is available. 75 | 76 | Frames 77 | ====== 78 | Individual requests and responses that are sent to or received from a device are called "Frame". These are the raw 79 | bytes that are exchanged between client and server (device). 80 | 81 | Frames contain a command such as *read* and a OID such as ``0x959930BF``. Some commands (such as *write*) can contain a 82 | payload and there's a way to communicate to a network of devices, called plant communication which has not been tested 83 | with this library yet. The details of the encoding of the mentioned parts is not of relevance here. 84 | 85 | For creating a frame that is to be sent to a device, there's two ways: 86 | 87 | * Creating it directly using :func:`~rctclient.frame.make_frame`, which takes the above mentioned input parameters and 88 | returns the byte stream ready to be sent 89 | * Using the higher-level class :class:`~rctclient.frame.SendFrame` which internally calls ``make_frame``, but stores 90 | the input parameters as well. This is especially useful for checking how things work, as its ``__repr__`` dunder 91 | pretty-prints both input and output. 92 | 93 | For receiving, there's the :class:`~rctclient.frame.ReceiveFrame`, which is fed with raw data from the wire and that 94 | signals when a complete frame is received. 95 | 96 | SendFrame 97 | --------- 98 | :class:`~rctclient.frame.SendFrame` is used to craft the byte stream used to send a request to the device. Uppon 99 | constructing the frame, it automatically crafts the byte stream, which is then available in the ``data`` property and 100 | can be sent to the device. 101 | 102 | .. note:: 103 | 104 | The payload has to be encoded before passing it to ``SendFrame`` e.g. using :func:`~rctclient.utils.encode_value`. 105 | 106 | The following example crafts a read command for the battery state of charge (``battery.soc``). The data that is to be 107 | sent via a network socket can be read from ``frame.data`` in the end: 108 | 109 | .. code-block:: pycon 110 | 111 | >>> from rctclient.registry import REGISTRY as R 112 | >>> from rctclient.frame import SendFrame 113 | >>> from rctclient.types import Command 114 | >>> 115 | >>> oinfo = R.get_by_name('battery.soc') 116 | >>> frame = SendFrame(command=Command.READ, id=oinfo.id) 117 | >>> frame 118 | 119 | >>> frame.data.hex() 120 | '2b0104959930bf0d65' 121 | 122 | make_frame 123 | ---------- 124 | As discussed earlier, :func:`~rctclient.frame.make_frame` is used internally by ``SendFrame``. It basically behaves the 125 | same but does not require object instantiation and all that comes with it, but instead simply returns the generated 126 | bytes to be sent. 127 | 128 | .. code-block:: pycon 129 | 130 | >>> from rctclient.registry import REGISTRY as R 131 | >>> from rctclient.frame import make_frame 132 | >>> from rctclient.types import Command 133 | >>> 134 | >>> oinfo = R.get_by_name('battery.soc') 135 | >>> frame_data = make_frame(command=Command.READ, id=oinfo.id) 136 | >>> frame_data.hex() 137 | '2b0104959930bf0d65' 138 | 139 | ReceiveFrame 140 | ------------ 141 | :class:`rctclient.frame.ReceiveFrame` is used to receive a frame of data from the device. It is designed so that it can 142 | ``consume`` a frame as it is received over the network. The instance signals when a frame has been received 143 | (``complete()`` returns *True*) or raise an exception when an error occurs, such as a checksum mismatch. The 144 | ``consume`` function returns the amount of bytes it consumed, which allows for removing the consumed data from the 145 | buffer and start receiving the next frame immediately, which will become clearer in the examples below. 146 | 147 | If the checksum does not match, an exception (:class:`~rctclient.exceptions.FrameCRCMismatch`) is raised that contains 148 | the received and computed checksums for debugging and also carries the amount of consumed bytes, so one can slice off 149 | those bytes and start with the next frame. Due to the way the devices work, CRC mismatches are not uncommon, and even 150 | a matching checksum does not guarantee that the data in the payload is complete. More on that later. 151 | 152 | In addition to that, if a command that the parser can't work with (such as ``EXTENSION``, or if the frame is broken), a 153 | :class:`~rctclient.exceptions.InvalidCommand` is raised, containing the amount of consumed bytes. 154 | 155 | If the parser notices that it overshot, a :class:`~rctclient.exceptions.FrameLengthExceeded` is raised, again 156 | containing the amount of consumed bytes. 157 | 158 | As an example, we'll read the frame data from the above *SendFrame* example as an input to the ReceiveFrames consume 159 | method. The output above was (in hexadecimal notation) ``2b0104959930bf0d65`` which can be transformed back into a byte 160 | stream using the ``bytearray.fromhex`` method: 161 | 162 | .. code-block:: python 163 | 164 | from rctclient.registry import REGISTRY as R 165 | from rctclient.frame import ReceiveFrame 166 | 167 | frame = ReceiveFrame() 168 | print(frame.complete()) 169 | #> False 170 | 171 | data = bytearray.fromhex('2b0104959930bf0d65') 172 | consumed_bytes = frame.consume(data) 173 | print(f'Consumed: {consumed_bytes}, input length: {len(data)}') 174 | #> Consumed: 9, input length: 9 175 | 176 | print(frame) 177 | #> 178 | print(R.get_by_id(frame.id)) 179 | #> 180 | 181 | (This script is complete, it should run "as is") 182 | 183 | This is a rather constructed use case, as normally the data to parse would be a response frame from the device. But it 184 | shows the modularity of the approach. Now, using the ``read-value`` subcommand to the :ref:`cli` tool, extract the 185 | payload from a real response. This safes us from needing to explain the entire network handling in this section. By 186 | starting the tool in ``--debug`` mode, the payload can be read as hex string: 187 | 188 | .. code-block:: shell-session 189 | 190 | $ rctclient --debug read-value -h 192.168.0.1 --name battery.soc 191 | 2020-10-02 15:11:02,367 - rctclient.cli - INFO - rctclient CLI starting 192 | 2020-10-02 15:11:02,367 - rctclient.cli - DEBUG - Object info by name: 193 | 2020-10-02 15:11:02,367 - rctclient.cli - DEBUG - Connecting to host 194 | 2020-10-02 15:11:02,368 - rctclient.cli - DEBUG - Connected to 192.168.19.13:8899 195 | 2020-10-02 15:11:02,431 - rctclient.cli - DEBUG - Received 14 bytes: 002b0508959930bf3f590f868810 196 | 2020-10-02 15:11:02,432 - rctclient.cli - DEBUG - Frame consumed 14 bytes 197 | 2020-10-02 15:11:02,432 - rctclient.cli - DEBUG - Got frame: 198 | 0.8478931188583374 199 | 200 | The raw byte stream that the device responded with is ``002b0508959930bf3f590f868810`` in hexadecimal notation. The 201 | following example uses it to manually craft a response frame and also demonstrates how to decode the payload: 202 | 203 | .. code-block:: python 204 | 205 | from rctclient.registry import REGISTRY as R 206 | from rctclient.frame import ReceiveFrame 207 | from rctclient.utils import decode_value 208 | 209 | frame = ReceiveFrame() 210 | frame.consume(bytearray.fromhex('002b0508959930bf3f590f868810')) 211 | 212 | # check that the frame is complete 213 | print(frame.complete()) 214 | #> True 215 | 216 | # take a look at the frame 217 | print(frame) 218 | #> 219 | 220 | # get information about the object 221 | oinfo = R.get_by_id(frame.id) 222 | print(oinfo.name, oinfo.response_data_type) 223 | #> battery.soc DataType.FLOAT 224 | 225 | # decode the value using the response data type 226 | value = decode_value(oinfo.response_data_type, frame.data) 227 | print(value) 228 | #> 0.8478931188583374 229 | 230 | (This script is complete, it should run "as is") 231 | 232 | Encoding and decoding data 233 | ========================== 234 | The two functions :func:`rctclient.utils.decode_value` and :func:`rctclient.utils.encode_value` are used to transform 235 | data between high-level data types and byte streams in both directions. 236 | 237 | Each OID (see above) has a data type associated for sending and one for receiving (though they are the same for most 238 | OIDs). To encode a value for sending with a *SendFrame*, supply the ``request_data_type`` as first parameter to 239 | ``encode_value``. For the opposite direction, supply the ``response_data_type`` to ``decode_value`` along with the 240 | content from the ``data`` attribute from the completed *ReceiveFrame*. 241 | 242 | If the data can't be decoded, a ``struct.error`` is raised by the *struct* module. 243 | 244 | .. warning:: 245 | 246 | It is not uncommon for the device to send incomplete payload along with a valid checksum. Always catch the 247 | exceptions raised by the functions. 248 | 249 | Basic workflow 250 | ************** 251 | The most basic workflow involves sending a request to the device and receive the response: 252 | 253 | #. Open a TCP socket to the device. 254 | #. If payload is to be sent (write commands), use :func:`~rctclient.utils.encode_value` to encode the data. 255 | #. Craft a frame (using :class:`~rctclient.frame.SendFrame` or :func:`~rctclient.frame.make_frame`) with the correct 256 | object ID and command set and, if required, include the payload. 257 | #. Send the frame via a TCP socket to the device. 258 | #. Read the response into a :class:`~rctclient.frame.ReceiveFrame` 259 | #. Once complete, decode the response value using :func:`~rctclient.utils.decode_value` 260 | #. Repeat steps 2-6 as long as required. 261 | #. Close the socket to the device. 262 | 263 | Basic example 264 | ************* 265 | Assuming the :ref:`simulator` is running in its default config (listening on ``localhost:8899``) by starting it without 266 | parameters like so: ``rctclient simulator``, the following script can be used to query for the battery state of charge 267 | (SOC) value: 268 | 269 | .. code-block:: python 270 | 271 | #!/usr/bin/env python3 272 | 273 | import socket, select, sys 274 | from rctclient.frame import ReceiveFrame, make_frame 275 | from rctclient.registry import REGISTRY as R 276 | from rctclient.types import Command 277 | from rctclient.utils import decode_value 278 | 279 | # open the socket and connect to the remote device: 280 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 281 | sock.connect(('localhost', 8899)) 282 | 283 | # query information about an object ID (here: battery.soc): 284 | object_info = R.get_by_name('battery.soc') 285 | 286 | # construct a byte stream that will send a read command for the object ID we want, and send it 287 | send_frame = make_frame(command=Command.READ, id=object_info.object_id) 288 | sock.send(send_frame) 289 | 290 | # loop until we got the entire response frame 291 | frame = ReceiveFrame() 292 | while True: 293 | ready_read, _, _ = select.select([sock], [], [], 2.0) 294 | if sock in ready_read: 295 | # receive content of the input buffer 296 | buf = sock.recv(256) 297 | # if there is content, let the frame consume it 298 | if len(buf) > 0: 299 | frame.consume(buf) 300 | # if the frame is complete, we're done 301 | if frame.complete(): 302 | break 303 | else: 304 | # the socket was closed by the device, exit 305 | sys.exit(1) 306 | 307 | # decode the frames payload 308 | value = decode_value(object_info.response_data_type, frame.data) 309 | 310 | # and print the result: 311 | print(f'Response value: {value}') 312 | 313 | (This script is complete, it should run "as is") 314 | 315 | When run against a real device (by exchanging the ``localhost`` above with the address of the device), the result is 316 | like this: 317 | 318 | .. code-block:: shell-session 319 | 320 | $ ./basic-example.py 321 | Response value: 0.6453145742416382 322 | 323 | Obviously, this example lacks any error handling for the sake of simplicity. 324 | 325 | Caveats 326 | ******* 327 | This section leaves the protocol part and hops into the real world, to the real hardware devices. Some things are 328 | important to know as they can lead to confusion. The inverters are embedded devices and take some shortcuts when it 329 | comes to network communication. 330 | 331 | .. _security: 332 | 333 | Security 334 | ======== 335 | 336 | **There is none.** 337 | 338 | The protocol itself has no security primitives such as authentication and encryption. The device itself does not allow 339 | the usage of TLS (Transport Layer Security) or other encryption standards. Whoever can reach the device via the network 340 | (be it via ethernet cable or the WIFI access point the devices create by default) has full control over all settings of 341 | the device. The official app *does* require passwords to access more than just the basics, but that password is only 342 | used to enable features in the app itself and is not sent over the wire ever. It is really important to understand this 343 | when connecting the device to any network. 344 | 345 | .. warning:: 346 | 347 | To re-iterate: There is no security, anyone who can reach the device on the network has full control over it. 348 | 349 | It has been demonstrated that data can be injected into a running TCP communication. If the device was to communicate 350 | over an untrusted network (e.g. the Internet), anyone who could get a hold of the stream can send commands that the 351 | device will apply. 352 | 353 | .. _incomplete-responses: 354 | 355 | Incomplete, incorrect or missing responses 356 | ========================================== 357 | The devices are not meant to communicate with multiple network clients simultaneously. They will interrupt what they 358 | are doing when another request comes in. This results in incomplete frames that have a valid checksum, as the device 359 | may be interrupted while preparing the payload, then calculates the checksum over the partial response and send it over 360 | the wire. This is especially noticable when requesting large OIDs such as strings or the :ref:`protocol-timeseries` or 361 | :ref:`protocol-event-table` OIDs, as they appear to be cut at arbitrary places, yet the attached checksum matches the 362 | calculated checksum. 363 | 364 | Answers from the device may also contain perfectly valid data, but with a wrong checksum attached (the *read_pcap.py* 365 | tool makes an attempt to decode the frames for debugging purposes). In other (rare) cases, the request body from 366 | another client can found in a response's payload (although the checksum has been invalid in all observed cases). 367 | 368 | Sometimes the response can be lost alltogether, this can be seen in the app as timeouts, or when it appears that some 369 | parts of a table (e.g. the battery overview) are initially empty and are filled in after all the other values on the 370 | next poll. 371 | 372 | If the device is communicating with the vendors servers for external control, this communication could be impacted by 373 | having the app open or using another client to query the device. 374 | 375 | When creating programs that communicate with the devices (which is the sole purpose of this module), always take into 376 | account that queries may simply get lost or have incomplete payload, so make sure to implement some sort of retry 377 | mechanism. 378 | 379 | Conclusion 380 | ********** 381 | With the information provided on this page it should be possible to create client applications with ease. The 382 | :ref:`CLI` tool may also give some insights into how things work, they're implemented in the ``cli.py`` file, the 383 | :ref:`simulator` can be found in ``simulator.py``. 384 | 385 | If things are still unclear, of bugs are found or if there are any questions, don't hestitate to get in contact using 386 | the projects issue tracker in GitHub. 387 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click 2 | Sphinx>=2.0 3 | sphinx-autodoc-typehints 4 | sphinx-click 5 | sphinx-rtd-theme 6 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | flake8 2 | mypy 3 | pytest 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | 2 | [flake8] 3 | ignore = E501,E402 4 | max-line-length = 120 5 | exclude = .git,.tox,build,_build,env,venv,__pycache__ 6 | per-file-ignores = 7 | src/rctclient/registry.py:E241 8 | 9 | [tool:pytest] 10 | testpaths = tests 11 | python_files = 12 | test_*.py 13 | *_test.py 14 | tests.py 15 | addopts = 16 | -ra 17 | --strict-markers 18 | --doctest-modules 19 | --doctest-glob=\*.rst 20 | --tb=short 21 | 22 | [coverage:run] 23 | omit = 24 | venv/* 25 | tests/* 26 | 27 | [pylint.FORMAT] 28 | max-line-length = 120 29 | 30 | [pylint.MESSAGES CONTROL] 31 | disable=logging-fstring-interpolation 32 | 33 | [pylint.REPORTS] 34 | output-format = colorized 35 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from setuptools import setup # type: ignore 3 | 4 | with open('README.md', 'rt') as fh: 5 | long_description = fh.read() 6 | 7 | setup( 8 | name='rctclient', 9 | version='0.0.5', 10 | author='Stefan Valouch', 11 | author_email='svalouch@valouch.com', 12 | description='Implementation of the RCT Power communication protocol', 13 | long_description=long_description, 14 | long_description_content_type='text/markdown', 15 | project_urls={ 16 | 'Documentation': 'https://rctclient.readthedocs.io/', 17 | 'Source': 'https://github.com/svalouch/python-rctclient/', 18 | 'Tracker': 'https://github.com/svalouch/python-rctclient/issues', 19 | }, 20 | packages=['rctclient'], 21 | package_data={'rctclient': ['py.typed']}, 22 | package_dir={'': 'src'}, 23 | include_package_data=True, 24 | zip_safe=False, 25 | platforms='any', 26 | url='https://github.com/svalouch/python-rctclient/', 27 | python_requires='>=3.6', 28 | 29 | install_requires=[ 30 | 'typing_extensions ; python_version < "3.8"', 31 | ], 32 | 33 | extras_require={ 34 | 'cli': [ 35 | 'click >=7.0, <8.2', 36 | ], 37 | 'tests': [ 38 | 'mypy', 39 | 'pylint', 40 | 'pytest', 41 | 'pytest-cov', 42 | 'pytest-pylint', 43 | ], 44 | 'docs': [ 45 | 'click >=7.0, <8.2', 46 | 'Sphinx >=2.0', 47 | 'sphinx-autodoc-typehints', 48 | 'sphinx-click', 49 | 'sphinx-rtd-theme', 50 | 'recommonmark >=0.5.0', 51 | ], 52 | }, 53 | entry_points={ 54 | 'console_scripts': [ 55 | 'rctclient=rctclient.cli:cli [cli]', 56 | ], 57 | }, 58 | 59 | classifiers=[ 60 | 'Development Status :: 2 - Pre-Alpha', 61 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 62 | 'Operating System :: OS Independent', 63 | 'Programming Language :: Python', 64 | 'Programming Language :: Python :: 3', 65 | 'Programming Language :: Python :: 3 :: Only', 66 | 'Programming Language :: Python :: 3.6', 67 | 'Programming Language :: Python :: 3.7', 68 | 'Programming Language :: Python :: 3.8', 69 | 'Programming Language :: Python :: 3.9', 70 | 'Programming Language :: Python :: 3.10', 71 | 'Programming Language :: Python :: 3.11', 72 | ], 73 | ) 74 | -------------------------------------------------------------------------------- /src/rctclient/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/svalouch/python-rctclient/9300ea947e6f45a100e6b598b8146c2e59a9d646/src/rctclient/__init__.py -------------------------------------------------------------------------------- /src/rctclient/cli.py: -------------------------------------------------------------------------------- 1 | 2 | ''' 3 | Command line interface implementation. 4 | ''' 5 | 6 | # Copyright 2020, Peter Oberhofer (pob90) 7 | # Copyright 2020-2021, Stefan Valouch (svalouch) 8 | # SPDX-License-Identifier: GPL-3.0-only 9 | 10 | import logging 11 | import select 12 | import socket 13 | import sys 14 | from datetime import datetime 15 | from typing import List, Optional 16 | 17 | try: 18 | import click 19 | except ImportError: 20 | print('"click" not found, commands unavailable', file=sys.stderr) 21 | sys.exit(1) 22 | 23 | from .exceptions import FrameCRCMismatch, FrameLengthExceeded, InvalidCommand 24 | from .frame import ReceiveFrame, make_frame 25 | from .registry import REGISTRY as R 26 | from .simulator import run_simulator 27 | from .types import Command, DataType 28 | from .utils import decode_value, encode_value 29 | 30 | log = logging.getLogger('rctclient.cli') 31 | 32 | 33 | @click.group() 34 | @click.pass_context 35 | @click.version_option() 36 | @click.option('-d', '--debug', is_flag=True, default=False, help='Enable debug output') 37 | @click.option('--frame-debug', is_flag=True, default=False, help='Enables frame debugging (requires --debug)') 38 | def cli(ctx, debug: bool, frame_debug: bool) -> None: 39 | ''' 40 | rctclient toolbox. Please help yourself with the subcommands. 41 | ''' 42 | ctx.ensure_object(dict) 43 | ctx.obj['DEBUG'] = debug 44 | 45 | if debug: 46 | logging.basicConfig( 47 | level=logging.DEBUG, 48 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', 49 | ) 50 | if not frame_debug: 51 | logging.getLogger('rctclient.frame.ReceiveFrame').setLevel(logging.INFO) 52 | log.info('rctclient CLI starting') 53 | 54 | 55 | def autocomplete_registry_name(*args, **kwargs) -> List[str]: # pylint: disable=unused-argument 56 | ''' 57 | Provides autocompletion for the object IDs name parameter. 58 | 59 | :return: A list of names that either start with `incomplete` or all if `incomplete` is empty. 60 | ''' 61 | if 'incomplete' not in kwargs: 62 | kwargs['incomplete'] = '' 63 | return R.prefix_complete_name(kwargs['incomplete']) 64 | 65 | if click.__version__ >= '8.1.0': 66 | autocomp_registry = {'shell_complete': autocomplete_registry_name} 67 | else: 68 | autocomp_registry = {'autocompletion': autocomplete_registry_name} 69 | 70 | 71 | def receive_frame(sock: socket.socket, timeout: int = 2) -> ReceiveFrame: 72 | ''' 73 | Receives a frame from a socket. 74 | 75 | :param sock: The socket to receive from. 76 | :param timeout: Receive timeout in seconds. 77 | :returns: The received frame. 78 | :raises TimeoutError: If the timeout expired. 79 | ''' 80 | frame = ReceiveFrame() 81 | while True: 82 | try: 83 | ready_read, _, _ = select.select([sock], [], [], timeout) 84 | except select.error as exc: 85 | log.error('Error during receive: select returned an error: %s', str(exc)) 86 | raise 87 | 88 | if ready_read: 89 | buf = sock.recv(1024) 90 | if len(buf) > 0: 91 | log.debug('Received %d bytes: %s', len(buf), buf.hex()) 92 | i = frame.consume(buf) 93 | log.debug('Frame consumed %d bytes', i) 94 | if frame.complete(): 95 | if len(buf) > i: 96 | log.warning('Frame complete, but buffer still contains %d bytes', len(buf) - i) 97 | log.debug('Leftover bytes: %s', buf[i:].hex()) 98 | return frame 99 | raise TimeoutError 100 | 101 | 102 | @cli.command('read-value') 103 | @click.pass_context 104 | @click.option('-p', '--port', default=8899, type=click.INT, help='Port at which the device listens, default 8899', 105 | metavar='') 106 | @click.option('-h', '--host', required=True, type=click.STRING, help='Host address or IP of the device', 107 | metavar='') 108 | @click.option('-i', '--id', type=click.STRING, help='Object ID to query, of the form "0xXXXX"', metavar='') 109 | @click.option('-n', '--name', help='Object name to query', type=click.STRING, metavar='', **autocomp_registry) 110 | @click.option('-v', '--verbose', is_flag=True, default=False, help='Enable verbose output') 111 | def read_value(ctx, port: int, host: str, id: Optional[str], name: Optional[str], verbose: bool) -> None: 112 | ''' 113 | Sends a read request. The request is sent to the target "" on the given "" (default: 8899), the 114 | response is returned on stdout. Without "verbose" set, the value is returned on standard out, otherwise more 115 | information about the object is printed with the value. 116 | 117 | Specify either "--id " or "--name ". The ID must be in th decimal notation, such as "0x959930BF", the 118 | name must exactly match the name of a known object ID such as "battery.soc". 119 | 120 | The "" option supports shell autocompletion (if installed). 121 | 122 | If "--debug" is set, log output is sent to stderr, so the value can be read from stdout while still catching 123 | everything else on stderr. 124 | 125 | Timeseries data and the event table will be queried using the current time. Note that the device may send an 126 | arbitrary amount of data. For time series data, The output will be a list of "timestamp=value" pairs separated by a 127 | comma, the timestamps are in isoformat, and they are not altered or timezone-corrected but passed from the device 128 | as-is. Likewise for event table entries, but their values are printed in hexadecimal. 129 | 130 | Examples: 131 | 132 | \b 133 | rctclient read-value --host 192.168.0.1 --name temperature.sink_temp_power_reduction 134 | rctclient read-value --host 192.168.0.1 --id 0x90B53336 135 | \f 136 | :param ctx: Click context 137 | :param port: The port number. 138 | :param host: The hostname or IP address, passed to ``socket.connect``. 139 | :param id: The ID to query. Mutually exclusive with `name`. 140 | :param name: The name to query. Mutually exclusive with `id`. 141 | :param verbose: Prints more information if `True`, or just the value if `False`. 142 | ''' 143 | if (id is None and name is None) or (id is not None and name is not None): 144 | log.error('Please specify either --id or --name', err=True) 145 | sys.exit(1) 146 | 147 | try: 148 | if id: 149 | real_id = int(id[2:], 16) 150 | log.debug('Parsed ID: 0x%X', real_id) 151 | oinfo = R.get_by_id(real_id) 152 | log.debug('Object info by ID: %s', oinfo) 153 | elif name: 154 | oinfo = R.get_by_name(name) 155 | log.debug('Object info by name: %s', oinfo) 156 | except KeyError: 157 | log.error('Could not find requested id or name') 158 | sys.exit(1) 159 | except ValueError as exc: 160 | log.debug('Invalid --id parameter: %s', str(exc)) 161 | log.error('Invalid --id parameter, can\'t parse', err=True) 162 | sys.exit(1) 163 | 164 | log.debug('Connecting to host') 165 | try: 166 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 167 | sock.connect((host, port)) 168 | log.debug('Connected to %s:%d', host, port) 169 | except socket.error as exc: 170 | log.error('Could not connect to host: %s', str(exc)) 171 | sys.exit(1) 172 | 173 | is_ts = oinfo.response_data_type == DataType.TIMESERIES 174 | is_ev = oinfo.response_data_type == DataType.EVENT_TABLE 175 | if is_ts or is_ev: 176 | sock.send(make_frame(command=Command.WRITE, id=oinfo.object_id, 177 | payload=encode_value(DataType.INT32, int(datetime.now().timestamp())))) 178 | else: 179 | sock.send(make_frame(command=Command.READ, id=oinfo.object_id)) 180 | try: 181 | rframe = receive_frame(sock) 182 | except FrameCRCMismatch as exc: 183 | log.error('Received frame CRC mismatch: received 0x%X but calculated 0x%X', 184 | exc.received_crc, exc.calculated_crc) 185 | sys.exit(1) 186 | except InvalidCommand: 187 | log.error('Received an unexpected/invalid command in response') 188 | sys.exit(1) 189 | except FrameLengthExceeded: 190 | log.error('Parser overshot, cannot recover frame') 191 | sys.exit(1) 192 | 193 | log.debug('Got frame: %s', rframe) 194 | if rframe.id != oinfo.object_id: 195 | log.error('Received unexpected frame, ID is 0x%X, expected 0x%X', rframe.id, oinfo.object_id) 196 | sys.exit(1) 197 | 198 | if is_ts or is_ev: 199 | _, table = decode_value(oinfo.response_data_type, rframe.data) 200 | if is_ts: 201 | value = ', '.join({f'{k:%Y-%m-%dT%H:%M:%S}={v:.4f}' for k, v in table.items()}) 202 | else: 203 | value = '' 204 | for entry in table.values(): 205 | e2 = f'0x{entry.element2:x}' if entry.element2 is not None else '' 206 | e3 = f'0x{entry.element3:x}' if entry.element3 is not None else '' 207 | e4 = f'0x{entry.element4:x}' if entry.element4 is not None else '' 208 | value += f'0x{entry.entry_type:x},{entry.timestamp:%Y-%m-%dT%H:%M:%S},{e2},{e3},{e4}\n' 209 | else: 210 | # hexdump if the data type is now known 211 | if oinfo.response_data_type == DataType.UNKNOWN: 212 | value = '0x' + rframe.data.hex() 213 | else: 214 | value = decode_value(oinfo.response_data_type, rframe.data) 215 | 216 | if verbose: 217 | description = oinfo.description if oinfo.description is not None else '' 218 | unit = oinfo.unit if oinfo.unit is not None else '' 219 | click.echo(f'#{oinfo.index:3} 0x{oinfo.object_id:8X} {oinfo.name:{R.name_max_length()}} ' 220 | f'{description:75} {value} {unit}') 221 | else: 222 | click.echo(f'{value}') 223 | 224 | try: 225 | sock.close() 226 | except Exception as exc: # pylint: disable=broad-except 227 | log.error('Exception when disconnecting from the host: %s', str(exc)) 228 | sys.exit(0) 229 | 230 | 231 | @cli.command('simulator') 232 | @click.pass_context 233 | @click.option('-p', '--port', default=8899, type=click.INT, help='Port to bind the simulator to, defaults to 8899', 234 | metavar='') 235 | @click.option('-h', '--host', default='localhost', type=click.STRING, help='IP to bind the simulator to, defaults to ' 236 | 'localhost', metavar='') 237 | @click.option('-v', '--verbose', is_flag=True, default=False, help='Enable verbose output') 238 | def simulator(ctx, port: int, host: str, verbose: bool) -> None: 239 | ''' 240 | Starts the simulator. The simulator returns valid, but useless responses to queries. It binds to the address and 241 | port passed using "" (default: localhost) and "" (default: 8899) and allows up to five concurrent 242 | clients. 243 | 244 | The response values (for read queries) is read from the information associated with the queried object ID if set, 245 | else a sensible default value (such as 0, False or dummy strings) is computed based on the response data type of 246 | the object ID. 247 | \f 248 | :param port: The port to bind to, defaults to 8899. 249 | :param host: The address to bind to, defaults to localhost. 250 | :param verbose: Enables verbose output. 251 | ''' 252 | if not ctx.obj['DEBUG'] and verbose: 253 | logging.basicConfig(level=logging.INFO) 254 | 255 | run_simulator(host=host, port=port, verbose=verbose) 256 | -------------------------------------------------------------------------------- /src/rctclient/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2020, Peter Oberhofer (pob90) 3 | # Copyright 2020-2021, Stefan Valouch (svalouch) 4 | # SPDX-License-Identifier: GPL-3.0-only 5 | 6 | ''' 7 | Exceptions used by the module. 8 | ''' 9 | 10 | 11 | class RctClientException(Exception): 12 | ''' 13 | Base exception for this Python module. 14 | ''' 15 | 16 | 17 | class FrameError(RctClientException): 18 | ''' 19 | Base exception for frame handling code. 20 | ''' 21 | 22 | 23 | class ReceiveFrameError(FrameError): 24 | ''' 25 | Base exception for errors happening in ReceiveFrame. 26 | 27 | :param message: A message describing the error. 28 | :param consumed_bytes: How many bytes were consumed. 29 | ''' 30 | def __init__(self, message: str, consumed_bytes: int = 0) -> None: 31 | super().__init__(message) 32 | self.consumed_bytes = consumed_bytes 33 | 34 | 35 | class FrameCRCMismatch(ReceiveFrameError): 36 | ''' 37 | Indicates that the CRC that was received did not match with the computed value. 38 | 39 | :param received_crc: The CRC that was received with the frame. 40 | :param calculated_crc: The CRC that was calculated based on the received data. 41 | ''' 42 | def __init__(self, message: str, received_crc: int, calculated_crc: int, consumed_bytes: int = 0) -> None: 43 | super().__init__(message) 44 | self.received_crc = received_crc 45 | self.calculated_crc = calculated_crc 46 | self.consumed_bytes = consumed_bytes 47 | 48 | 49 | class InvalidCommand(ReceiveFrameError): 50 | ''' 51 | Indicates that the command is not supported. This means that ``Command`` does not contain a field for it, or that 52 | it is the EXTENSION command that cannot be handled. 53 | 54 | :param command: The command byte. 55 | ''' 56 | def __init__(self, message: str, command: int, consumed_bytes: int = 0) -> None: 57 | super().__init__(message) 58 | self.command = command 59 | self.consumed_bytes = consumed_bytes 60 | 61 | 62 | class FrameLengthExceeded(ReceiveFrameError): 63 | ''' 64 | Indicates that more data was consumed by ReceiveFrame than it should have. This usually indicates a bug in the 65 | parser code and should be reported. 66 | ''' 67 | -------------------------------------------------------------------------------- /src/rctclient/frame.py: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2020, Peter Oberhofer (pob90) 3 | # Copyright 2020,2021 Stefan Valouch (svalouch) 4 | # SPDX-License-Identifier: GPL-3.0-only 5 | 6 | import logging 7 | import struct 8 | from typing import Union 9 | 10 | from .exceptions import FrameCRCMismatch, InvalidCommand, FrameLengthExceeded 11 | from .types import Command, FrameType 12 | from .utils import CRC16 13 | 14 | #: Token that starts a frame 15 | START_TOKEN = b'+' 16 | #: Token that escapes the next value 17 | ESCAPE_TOKEN = b'-' 18 | #: Length of the header 19 | FRAME_LENGTH_HEADER = 1 20 | #: Length of a command 21 | FRAME_LENGTH_COMMAND = 1 22 | #: Length of the length information 23 | FRAME_LENGTH_LENGTH = 2 24 | #: Length of a frame, contains 1 byte header, 1 byte command and 2 bytes length 25 | FRAME_HEADER_WITH_LENGTH = FRAME_LENGTH_HEADER + FRAME_LENGTH_COMMAND + FRAME_LENGTH_LENGTH 26 | #: Length of the CRC16 checkum 27 | FRAME_LENGTH_CRC16 = 2 28 | 29 | #: Amount of bytes we need to have a command 30 | BUFFER_LEN_COMMAND = 2 31 | 32 | 33 | def make_frame(command: Command, id: int, payload: bytes = b'', address: int = 0, 34 | frame_type: FrameType = FrameType.STANDARD) -> bytes: 35 | ''' 36 | Crafts the byte-stream representing the input values. The result of this function can be sent as-is to the target 37 | device. 38 | 39 | `payload` is ignored for ``READ`` commands. and the `address` is ignored for ``STANDARD`` frames. 40 | 41 | For a variant which stores the input values as well as the output, see :class:`~rctclient.frame.SendFrame`. 42 | 43 | .. versionadded:: 0.0.2 44 | 45 | :param command: The command to transmit. 46 | :param id: The object ID to target. 47 | :param payload: The payload to be transmitted. Use :func:`~rctclient.utils.encode_value` to generate valid 48 | payloads. 49 | :param address: Address for plant communication (untested, ignored for standard communication). 50 | :param frame_type: The type if frame to transmit (standard or plant). 51 | 52 | :return: byte object ready to be sent to a device. 53 | ''' 54 | # start with the command 55 | buf = bytearray(struct.pack('B', command)) 56 | 57 | # add frame type and length of payload 58 | if command in [Command.LONG_WRITE, Command.LONG_RESPONSE]: 59 | buf += struct.pack('>H', frame_type + len(payload)) # 2 bytes 60 | else: 61 | buf += struct.pack('>B', frame_type + len(payload)) # 1 byte 62 | 63 | # add address for plants 64 | if frame_type == FrameType.PLANT: 65 | buf += struct.pack('>I', address) # 4 bytes 66 | 67 | # add the ID 68 | buf += struct.pack('>I', id) # 4 bytes 69 | 70 | # add the payload unless it's a READ 71 | if command != Command.READ: 72 | buf += payload # N bytes 73 | 74 | # calculate and append the checksum 75 | crc16 = CRC16(buf) 76 | buf += struct.pack('>H', crc16) 77 | 78 | data = bytearray(struct.pack('c', START_TOKEN)) 79 | 80 | # go over the buffer and inject escape tokens 81 | 82 | for byt in buf: 83 | byte = bytes([byt]) 84 | if byte in [START_TOKEN, ESCAPE_TOKEN]: 85 | data += ESCAPE_TOKEN 86 | 87 | data += byte 88 | 89 | return data 90 | 91 | 92 | class SendFrame: 93 | ''' 94 | A container for data to be transmitted to the target device. Instances of this class keep the input values so they 95 | can be retrieved later, if that is not a requirement it's easier to use :func:`~rctclient.frame.make_frame` which 96 | is called by this class internally to generate the byte stream. The byte stream stored by this class is generated on 97 | initialization and can be retrieved at any time using the ``data`` property. 98 | 99 | The frame byte stream that is generated by this class is meant to be sent to a device. The receiving side is 100 | implemented in :class:`ReceiveFrame`. 101 | 102 | `payload` needs to be encoded before it can be transmitted. See :func:`~rctclient.utils.encode_value`. It is 103 | ignored for ``READ`` commands. 104 | 105 | `address` is used for ``PLANT`` frames and otherwise ignored, when queried later using the ``address`` property, 0 106 | is returned for non-PLANT frames. 107 | 108 | :param command: The command to transmit. 109 | :param id: The message id. 110 | :param payload: Optional payload (ignored for read commands). 111 | :param address: Address for plant communication (untested, ignored for non-PLANT frame types). 112 | :param frame_type: Type of frame (standard or plant). 113 | ''' 114 | 115 | _command: Command 116 | _id: int 117 | _address: int 118 | _frame_type: FrameType 119 | _payload: bytes 120 | 121 | _data: bytes 122 | 123 | def __init__(self, command: Command, id: int, payload: bytes = b'', address: int = 0, 124 | frame_type: FrameType = FrameType.STANDARD) -> None: 125 | self._command = command 126 | self._id = id 127 | self._frame_type = frame_type 128 | 129 | self._payload = payload if command != Command.READ else b'' 130 | self._address = address if frame_type == FrameType.PLANT else 0 131 | 132 | self._data = make_frame(self._command, self._id, self._payload, self._address, self._frame_type) 133 | 134 | def __repr__(self) -> str: 135 | return f'' 136 | 137 | @property 138 | def data(self) -> bytes: 139 | ''' 140 | Returns the data after encoding, ready to be sent over the socket. 141 | ''' 142 | return self._data 143 | 144 | @property 145 | def command(self) -> Command: 146 | ''' 147 | Returns the command. 148 | ''' 149 | return self._command 150 | 151 | @property 152 | def id(self) -> int: 153 | ''' 154 | Returns the object ID. 155 | ''' 156 | return self._id 157 | 158 | @property 159 | def address(self) -> int: 160 | ''' 161 | Returns the address for plant communication. Note that this returns 0 unless plant-communication was requested. 162 | ''' 163 | return self._address 164 | 165 | @property 166 | def frame_type(self) -> FrameType: 167 | ''' 168 | Returns the type of communication frame. 169 | ''' 170 | return self._frame_type 171 | 172 | @property 173 | def payload(self) -> bytes: 174 | ''' 175 | Returns the payload (input data). To get the result to send to a device, use ``data``. Note that this returns 176 | an empty byte stream for ``READ`` commands, regardless of any input. 177 | ''' 178 | return self._payload 179 | 180 | 181 | class ReceiveFrame: 182 | ''' 183 | Structure to receive and decode data received from the RCT device. Each instance can only consume a single frame 184 | and a new one needs to be constructed to receive the next one. Frames are decoded incrementally, by feeding data 185 | to the ``consume()`` function as it arrives over the network. The function returns the amount of bytes it consumed, 186 | and it automatically stops consumption the frame has been received completely. 187 | 188 | Use ``complete()`` to determin if the frame has been received in its entirety. 189 | 190 | When the frame has been received completely (``complete()`` returns *True*), it extracts the CRC and compares it 191 | with its own calculated checksum. Unless ``ignore_crc_match`` is set, a 192 | :class:`~rctclient.exceptions.FrameCRCMismatch` is raised if the checksums do **not** match. 193 | 194 | If the command encoded in the frame is not supported or invalid, a :class:`~rctclient.exceptions.InvalidCommand` 195 | will be raised unconditionally during consumption. 196 | 197 | Both exceptions carry the amount of consumed bytes in a field as to allow users to remove the already consumed 198 | bytes from their input buffer in order to start consumption with a new ReceiveFrame instance for the next frame. 199 | 200 | Some of the fields (such as command, id, frame_type, ...) are populated if enough data has arrived, even before the 201 | checksum has been received and compared. Until ``complete()`` returns *True*, this data may not be valid, but it 202 | may hint towards invalid frames (invalid length being the most common problem). 203 | 204 | To decode the payload, use :func:`~rctclient.utils.decode_value`. 205 | 206 | .. note:: 207 | 208 | The parsing deviates from the protocol in the following ways: 209 | 210 | * arbitrary data (instead of just a single ``0x00``) before a start byte is ignored. 211 | * the distinction between normal and long commands is ignored. No error is reported if a frame that should be a 212 | ``LONG_RESPONSE`` is received with ``RESPONSE``, for example. 213 | 214 | .. versionchanged:: 0.0.3 215 | 216 | * The frame type is detected from the consumed data and the ``frame_type`` parameter was removed. 217 | * Debug logging is now handled using the Python logging framework. ``debug()`` was dropped as it served no use 218 | anymore. 219 | * The ``decode()`` method was removed as the functionality was integrated into ``consume()``. 220 | 221 | .. versionadded:: 0.0.3 222 | 223 | The ``ignore_crc_match`` parameter prevents raising an exception on CRC mismatch. Use the ``crc_ok`` property to 224 | figure out if the CRC matched when setting the parameter to ``True``. 225 | 226 | :param ignore_crc_mismatch: If not set, no exception is raised when the CRC checksums do **not** match. Use 227 | ``crc_ok`` to figure out if they matched. 228 | ''' 229 | # frame complete yet? 230 | _complete: bool 231 | # did the crc match? 232 | _crc_ok: bool 233 | # parser in escape mode? 234 | _escaping: bool 235 | # received crc16 checksum data 236 | _crc16: int 237 | # frame length 238 | _frame_length: int 239 | # frame type 240 | _frame_type: FrameType 241 | # data buffer 242 | _buffer: bytearray 243 | # command 244 | _command: Command 245 | # amount of bytes consumed 246 | _consumed_bytes: int 247 | 248 | _frame_header_length: int 249 | 250 | # ID, once decoded 251 | _id: int 252 | # raw received payload 253 | _data: bytes 254 | # address for plant frames 255 | _address: int 256 | 257 | _ignore_crc_mismatch: bool 258 | 259 | def __init__(self, ignore_crc_mismatch: bool = False) -> None: 260 | self._log = logging.getLogger(__name__ + '.ReceiveFrame') 261 | self._complete = False 262 | self._crc_ok = False 263 | self._escaping = False 264 | self._crc16 = 0 265 | self._frame_length = 0 266 | self._frame_type = FrameType._NONE 267 | self._command = Command._NONE 268 | self._buffer = bytearray() 269 | self._ignore_crc_mismatch = ignore_crc_mismatch 270 | self._consumed_bytes = 0 271 | 272 | # set initially to the minimum length a frame header (i.e. everything before the data) can be. 273 | # 1 byte start, 1 byte command, 1 byte length, no address, 4 byte ID 274 | self._frame_header_length = 1 + 1 + 1 + 0 + 4 275 | 276 | # output data 277 | self._id = 0 278 | self._data = b'' 279 | self._address = 0 280 | 281 | def __repr__(self) -> str: 282 | return f'' 284 | 285 | @property 286 | def address(self) -> int: 287 | ''' 288 | Returns the address if the frame is a plant frame (``FrameType.PLANT``) or 0. 289 | 290 | :raises FrameNotComplete: If the frame has not been fully received. 291 | ''' 292 | return self._address 293 | 294 | @property 295 | def command(self) -> Command: 296 | ''' 297 | Returns the command. 298 | ''' 299 | return self._command 300 | 301 | def complete(self) -> bool: 302 | ''' 303 | Returns whether the frame has been received completely. If this returns True, do **not** ``consume()`` any more 304 | data with this instance, but instead create a new instance of this class for further consumption of data. 305 | ''' 306 | return self._complete 307 | 308 | @property 309 | def consumed_bytes(self) -> int: 310 | ''' 311 | Returns how many bytes the frame has consumed over its lifetime. This includes data that was consumed before 312 | the start of a frame was found, so the amount reported here may be larger than the amount of data that makes up 313 | the frame. 314 | 315 | .. versionadded:: 0.0.3 316 | ''' 317 | return self._consumed_bytes 318 | 319 | @property 320 | def crc_ok(self) -> bool: 321 | ''' 322 | Returns whether the CRC is valid. The value is only valid after a complete frame has arrived. 323 | 324 | .. versionadded:: 0.0.3 325 | ''' 326 | return self._crc_ok 327 | 328 | @property 329 | def data(self) -> bytes: 330 | ''' 331 | Returns the received data payload. This is empty if there has been no data received or the CRC did not match. 332 | ''' 333 | return bytes(self._data) 334 | 335 | @property 336 | def frame_type(self) -> FrameType: 337 | ''' 338 | Returns the frame type if enough data has been received to decode it, and ``FrameType._NONE`` otherwise. 339 | 340 | .. versionadded:: 0.0.3 341 | ''' 342 | return self._frame_type 343 | 344 | @property 345 | def id(self) -> int: 346 | ''' 347 | Returns the ID. If the frame has been received but the checksum does not match up, 0 is returned. 348 | ''' 349 | return self._id 350 | 351 | @property 352 | def ignore_crc_mismatch(self) -> bool: 353 | ''' 354 | Returns whether CRC mismatches are ignored during decoding. 355 | 356 | .. versionadded:: 0.0.3 357 | ''' 358 | return self._ignore_crc_mismatch 359 | 360 | @ignore_crc_mismatch.setter 361 | def ignore_crc_mismatch(self, newval: bool) -> None: 362 | ''' 363 | Changes whether CRC mismatches are ignored during decoding. 364 | 365 | .. versionadded:: 0.0.3 366 | ''' 367 | self._ignore_crc_mismatch = newval 368 | 369 | @property 370 | def frame_length(self) -> int: 371 | ''' 372 | Returns the length of the frame. This is ``0`` until the header containing the length field has been received. 373 | Note that this is not the length field of the protocol but rather the length of the frame in its entirety, 374 | from start byte to the end of the CRC. 375 | 376 | .. versionadded:: 0.0.3 377 | ''' 378 | return self._frame_length 379 | 380 | def consume(self, data: Union[bytes, bytearray]) -> int: # pylint: disable=too-many-branches,too-many-statements 381 | ''' 382 | Consumes data until the frame is complete. Returns the number of consumed bytes. Exceptions raised also carry 383 | the amount of consumed bytes. 384 | 385 | :param data: Data to consume. 386 | :return: The amount of bytes consumed from the input data. 387 | :raises FrameCRCMismatch: If the checksum didn't match and ``ignore_crc_mismatch`` was not set. 388 | :raises FrameLengthExceeded: If the parser read past the frames advertised length. 389 | :raises InvalidCommand: If the command byte is invalid or can't be decoded (such as ``EXTENSION``). 390 | ''' 391 | 392 | i = 0 393 | for d_byte in data: # pylint: disable=too-many-nested-blocks # ugly parser is ugly 394 | self._consumed_bytes += 1 395 | i += 1 396 | c = bytes([d_byte]) # pylint: disable=invalid-name 397 | self._log.debug('read: 0x%s', c.hex()) 398 | 399 | # sync to start_token 400 | if len(self._buffer) == 0: 401 | self._log.debug(' buffer empty') 402 | if c == START_TOKEN: 403 | self._log.debug(' start token found') 404 | self._buffer += c 405 | continue 406 | 407 | if self._escaping: 408 | self._log.debug(' resetting escape') 409 | self._escaping = False 410 | else: 411 | if c == ESCAPE_TOKEN: 412 | self._log.debug(' setting escape') 413 | # set the escape mode and ignore the byte at hand. 414 | self._escaping = True 415 | continue 416 | 417 | self._buffer += c 418 | self._log.debug(' adding to buffer') 419 | 420 | blen = len(self._buffer) # cache 421 | 422 | if blen == BUFFER_LEN_COMMAND: 423 | cmd = struct.unpack('B', bytes([self._buffer[1]]))[0] 424 | try: 425 | self._command = Command(cmd) 426 | except ValueError as exc: 427 | raise InvalidCommand(str(exc), cmd, i) from exc 428 | 429 | if self._command == Command.EXTENSION: 430 | raise InvalidCommand('EXTENSION is not supported', cmd, i) 431 | 432 | self._log.debug(' have command: 0x%x, is_plant: %s', self._command, 433 | Command.is_plant(self._command)) 434 | if Command.is_plant(self._command): 435 | self._frame_header_length += 4 436 | self._log.debug(' plant frame, extending header length by 4 to %d', self._frame_header_length) 437 | if Command.is_long(self._command): 438 | self._frame_header_length += 1 439 | self._log.debug(' long cmd, extending header length by 1 to %d', self._frame_header_length) 440 | elif blen == self._frame_header_length: 441 | self._log.debug(' buffer length %d indicates that it contains entire header', blen) 442 | if Command.is_long(self._command): 443 | data_length = struct.unpack('>H', self._buffer[2:4])[0] 444 | address_idx = 4 445 | else: 446 | data_length = struct.unpack('>B', bytes([self._buffer[2]]))[0] 447 | address_idx = 3 448 | 449 | if Command.is_plant(self._command): 450 | # length field includes address and id length == 8 bytes 451 | self._frame_length = (self._frame_header_length - 8) + data_length + FRAME_LENGTH_CRC16 452 | self._address = struct.unpack('>I', self._buffer[address_idx:address_idx + 4])[0] 453 | oid_idx = address_idx + 4 454 | 455 | else: 456 | # length field includes id length == 4 bytes 457 | self._frame_length = (self._frame_header_length - 4) + data_length + FRAME_LENGTH_CRC16 458 | oid_idx = address_idx 459 | 460 | self._log.debug(' data_length: %d bytes, frame_length: %d', data_length, self._frame_length) 461 | 462 | self._id = struct.unpack('>I', self._buffer[oid_idx:oid_idx + 4])[0] 463 | self._log.debug(' oid index: %d, OID: 0x%X', oid_idx, self._id) 464 | 465 | elif self._frame_length > 0 and blen == self._frame_length: 466 | self._log.debug(' buffer contains full frame') 467 | self._log.debug(' buffer: %s', self._buffer.hex()) 468 | self._complete = True 469 | 470 | self._crc16 = struct.unpack('>H', self._buffer[-2:])[0] 471 | calc_crc16 = CRC16(self._buffer[len(START_TOKEN):-FRAME_LENGTH_CRC16]) 472 | self._crc_ok = self._crc16 == calc_crc16 473 | self._log.debug(' crc: %04x calculated: %04x match: %s', self._crc16, calc_crc16, self._crc_ok) 474 | 475 | self._data = self._buffer[self._frame_header_length:-FRAME_LENGTH_CRC16] 476 | 477 | if not self._crc_ok and not self._ignore_crc_mismatch: 478 | raise FrameCRCMismatch('CRC mismatch', self._crc16, calc_crc16, i) 479 | return i 480 | elif self._frame_length > 0 and blen > self._frame_length: 481 | raise FrameLengthExceeded('Frame consumed more than it should', i) 482 | return i 483 | -------------------------------------------------------------------------------- /src/rctclient/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/svalouch/python-rctclient/9300ea947e6f45a100e6b598b8146c2e59a9d646/src/rctclient/py.typed -------------------------------------------------------------------------------- /src/rctclient/simulator.py: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2020, Peter Oberhofer (pob90) 3 | # Copyright 2020, Stefan Valouch (svalouch) 4 | # SPDX-License-Identifier: GPL-3.0-only 5 | 6 | import logging 7 | import select 8 | import socket 9 | import threading 10 | 11 | from .exceptions import FrameCRCMismatch 12 | from .frame import ReceiveFrame, SendFrame 13 | from .registry import REGISTRY as R 14 | from .types import Command 15 | from .utils import decode_value, encode_value 16 | 17 | 18 | def run_simulator(host: str, port: int, verbose: bool) -> None: 19 | ''' 20 | Starts the simulator. The simulator will bind to `host:port` and allow up to 5 concurrent clients to connect. Each 21 | client connection is handled in a thread. The function is intended to be run from a terminal and stopped by sending 22 | a keyboard interrupt (Ctrl+c). It runs forever until interrupted. 23 | ''' 24 | 25 | log = logging.getLogger('rctclient.simulator') 26 | try: 27 | serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 28 | serversocket.bind((host, port)) 29 | serversocket.listen(5) 30 | except socket.error as e: 31 | log.error(f'Unable to bind to {host}:{port}: {str(e)}') 32 | serversocket.close() 33 | raise 34 | 35 | log.info('Waiting for client connections') 36 | try: 37 | while True: 38 | connection, address = serversocket.accept() 39 | msg = f'connection accepted: {connection} {address}' 40 | if verbose: 41 | log.info(msg) 42 | else: 43 | log.debug(msg) 44 | 45 | # Start a new thread and return its identifier 46 | threading.Thread(target=socket_thread, args=(connection, address), daemon=True).start() 47 | except KeyboardInterrupt: 48 | msg = 'Keyboard interrupt, shutting down' 49 | if verbose: 50 | log.info(msg) 51 | else: 52 | log.debug(msg) 53 | 54 | serversocket.close() 55 | 56 | 57 | def socket_thread(connection, address) -> None: 58 | log = logging.getLogger(f'rctclient.simulator.socket_thread.{address[1]}') 59 | frame = ReceiveFrame() 60 | while True: 61 | try: 62 | ready_read, _, _ = select.select([connection], [], [], 1000.0) 63 | except select.error as e: 64 | log.error(f'Error during select call: {str(e)}') 65 | break 66 | 67 | if ready_read: 68 | # read up to 4k bytes in one chunk 69 | buf = connection.recv(4096) 70 | if len(buf) > 0: 71 | log.debug(f'Read {len(buf)} bytes: {buf.hex()}') 72 | consumed = 0 73 | while consumed < len(buf): 74 | try: 75 | i = frame.consume(buf) 76 | except FrameCRCMismatch as exc: 77 | log.warning(f'Discard frame with wrong CRC checksum. Got 0x{exc.received_crc:x}, calculated ' 78 | f'0x{exc.calculated_crc:x}, consumed {exc.consumed_bytes} bytes') 79 | log.warning(f'Frame data: {frame.data.hex()}') 80 | consumed += exc.consumed_bytes 81 | frame = ReceiveFrame() 82 | else: 83 | log.debug(f'Frame consumed {i} bytes') 84 | consumed += i 85 | if frame.complete(): 86 | log.debug(f'Complete frame: {frame}') 87 | try: 88 | send_sim_response(connection, frame, log) 89 | except Exception as exc: 90 | log.warning(f'Caught {type(exc)} during send_sim_response: {str(exc)}') 91 | 92 | frame = ReceiveFrame() 93 | buf = buf[consumed:] 94 | else: 95 | break 96 | 97 | connection.close() 98 | log.debug(f'Closing connection {connection}') 99 | 100 | 101 | def send_sim_response(connection, frame: ReceiveFrame, log: logging.Logger) -> None: 102 | oinfo = R.get_by_id(frame.id) 103 | 104 | if frame.command == Command.READ: 105 | payload = encode_value(oinfo.response_data_type, oinfo.sim_data) 106 | sframe = SendFrame(command=Command.RESPONSE, id=frame.id, address=frame.address, payload=payload) 107 | log.info(f'Read : 0x{oinfo.object_id:08X} {oinfo.name:{R.name_max_length()}} -> {sframe.data.hex()}') 108 | log.debug(f'Sending frame {sframe} with {len(sframe.data)} bytes 0x{sframe.data.hex()}') 109 | connection.send(sframe.data) 110 | 111 | elif frame.command == Command.WRITE: 112 | value = decode_value(oinfo.request_data_type, frame.data) 113 | log.info(f'Write : #{oinfo.index:3} 0x{oinfo.object_id:08X} {oinfo.name:{R.name_max_length()}} ' 114 | f'-> {value}') 115 | 116 | # TODO send response 117 | 118 | elif frame.command == Command.LONG_WRITE: 119 | value = decode_value(oinfo.request_data_type, frame.data) 120 | log.info(f'Write L: #{oinfo.index:3} 0x{oinfo.object_id:08X} {oinfo.name:{R.name_max_length()}} ' 121 | f'-> {value}') 122 | 123 | # TODO send response 124 | -------------------------------------------------------------------------------- /src/rctclient/types.py: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2020, Peter Oberhofer (pob90) 3 | # Copyright 2020-2021, Stefan Valouch (svalouch) 4 | # SPDX-License-Identifier: GPL-3.0-only 5 | 6 | ''' 7 | Type declarations. 8 | ''' 9 | 10 | # pylint: disable=too-many-arguments,too-few-public-methods 11 | 12 | from datetime import datetime 13 | from enum import auto, IntEnum 14 | from typing import Optional 15 | 16 | 17 | class Command(IntEnum): 18 | ''' 19 | Commands that can be used with :class:`~rctclient.frame.ReceiveFrame`, :func:`~rctclient.frame.make_frame` as well 20 | as :class:`~rctclient.frame.SendFrame`. 21 | ''' 22 | #: Read command 23 | READ = 0x01 24 | #: Write command 25 | WRITE = 0x02 26 | #: Long write command (use for variables > 251 bytes) 27 | LONG_WRITE = 0x03 28 | # Reserved 29 | # RESERVED = 0x04 30 | #: Response to a read or write command 31 | RESPONSE = 0x05 32 | #: Long response (for variables > 251 bytes) 33 | LONG_RESPONSE = 0x06 34 | # Reserved 35 | # RESERVED = 0x07 36 | #: Periodic reading 37 | READ_PERIODICALLY = 0x08 38 | 39 | #: Plant: Read command 40 | PLANT_READ = READ | 0x40 41 | #: Plant: Write command 42 | PLANT_WRITE = WRITE | 0x40 43 | #: Plant: Long write 44 | PLANT_LONG_WRITE = LONG_WRITE | 0x40 45 | # Reserved 46 | # PLANT_RESERVED = 0x04 | 0x40 47 | #: Plant: Response to a read or write to master 48 | PLANT_RESPONSE = RESPONSE | 0x40 49 | #: Plant: Long response to a read or write to master 50 | PLANT_LONG_RESPONSE = LONG_RESPONSE | 0x40 51 | # Reserved 52 | # PLANT_RESERVED = 0x07 | 0x40 53 | #: Plant: Periodic reading 54 | PLANT_READ_PERIODICALLY = READ_PERIODICALLY | 0x40 55 | 56 | #: Extension, can't be parsed using :class:`~rctclient.frame.ReceiveFrame`. 57 | EXTENSION = 0x3c 58 | 59 | #: Internal sentinel value, do not use 60 | _NONE = 0xff 61 | 62 | @staticmethod 63 | def is_plant(command: 'Command') -> bool: 64 | ''' 65 | Returns whether a command is for plant communication by checking if bit 6 is set. 66 | ''' 67 | return bool(command & 0x40) 68 | 69 | @staticmethod 70 | def is_long(command: 'Command') -> bool: 71 | ''' 72 | Returns whether a command is a long command. 73 | ''' 74 | return command in (Command.LONG_WRITE, Command.LONG_RESPONSE, Command.PLANT_LONG_WRITE, 75 | Command.PLANT_LONG_RESPONSE) 76 | 77 | @staticmethod 78 | def is_write(command: 'Command') -> bool: 79 | ''' 80 | Returns whether a command is a write command. 81 | 82 | .. versionadded:: 0.0.4 83 | ''' 84 | return command in (Command.WRITE, Command.LONG_WRITE, Command.PLANT_WRITE, Command.PLANT_LONG_WRITE) 85 | 86 | @staticmethod 87 | def is_response(command: 'Command') -> bool: 88 | ''' 89 | Returns whether a command is a response. 90 | 91 | .. versionadded:: 0.0.4 92 | ''' 93 | return command in (Command.RESPONSE, Command.LONG_RESPONSE, Command.PLANT_RESPONSE, 94 | Command.PLANT_LONG_RESPONSE) 95 | 96 | 97 | class ObjectGroup(IntEnum): 98 | ''' 99 | Grouping information for object IDs. The information is not used by the protocol and only provided to aid the user 100 | in using the software. 101 | ''' 102 | RB485 = 0 103 | ENERGY = 1 104 | GRID_MON = 2 105 | TEMPERATURE = 3 106 | BATTERY = 4 107 | CS_NEG = 5 108 | HW_TEST = 6 109 | G_SYNC = 7 110 | LOGGER = 8 111 | WIFI = 9 112 | ADC = 10 113 | NET = 11 114 | ACC_CONV = 12 115 | DC_CONV = 13 116 | NSM = 14 117 | IO_BOARD = 15 118 | FLASH_RTC = 16 119 | POWER_MNG = 17 120 | BUF_V_CONTROL = 18 121 | DB = 19 122 | SWITCH_ON_COND = 20 123 | P_REC = 21 124 | MODBUS = 22 125 | BAT_MNG_STRUCT = 23 126 | ISO_STRUCT = 24 127 | GRID_LT = 25 128 | CAN_BUS = 26 129 | DISPLAY_STRUCT = 27 130 | FLASH_PARAM = 28 131 | FAULT = 29 132 | PRIM_SM = 30 133 | CS_MAP = 31 134 | LINE_MON = 32 135 | OTHERS = 33 136 | BATTERY_PLACEHOLDER = 34 137 | FRT = 35 138 | PARTITION = 36 139 | 140 | 141 | class FrameType(IntEnum): 142 | ''' 143 | Enumeration of supported frame types. 144 | ''' 145 | #: Standard frame with an ID 146 | STANDARD = 4 147 | #: Plant frame with ID and address 148 | PLANT = 8 149 | 150 | #: Sentinel value, denotes unknown type. 151 | _NONE = 0 152 | 153 | 154 | class DataType(IntEnum): 155 | ''' 156 | Enumeration of types, used to select the correct structure when encoding for sending or decoding received data. See 157 | :func:`~rctclient.utils.decode_value` and :func:`~rctclient.utils.encode_value`. 158 | ''' 159 | 160 | #: Unknown type, default. Do not use for encoding or decoding. 161 | UNKNOWN = 0 162 | #: Boolean data (true or false) 163 | BOOL = 1 164 | #: 8-bit unsigned integer 165 | UINT8 = 2 166 | #: 8-bit signed integer 167 | INT8 = 3 168 | #: 16-bit unsigned integer 169 | UINT16 = 4 170 | #: 16-bit signed integer 171 | INT16 = 5 172 | #: 32-bit unsigned integer 173 | UINT32 = 6 174 | #: 32-bit signed integer 175 | INT32 = 7 176 | #: Enum, will be handled like a 16-bit unsigned integer 177 | ENUM = 8 178 | #: Floating point number 179 | FLOAT = 9 180 | #: String (may contain `\0` padding). 181 | STRING = 10 182 | 183 | # user defined, usually composite datatypes follow: 184 | 185 | #: Non-native type: Timeseries data consisting of a tuple of a timestamp for the record (usually the day) and a 186 | #: dict mapping values to timestamps. Can not be used for encoding. 187 | TIMESERIES = 20 # timestamp, [(timestamp, value), ...] 188 | #: Non-native: Event table entries consisting of a tuple of a timestamp for the record (usually the day) and a dict 189 | #: mapping values to timestamps. Can not be used for encoding. 190 | EVENT_TABLE = 21 191 | 192 | 193 | class EventEntry: 194 | ''' 195 | A single entry in the event table. An entry consists of a type, which controls the meaning of the other fields. 196 | 197 | .. note:: 198 | 199 | Not a whole lot is known about the entries. Information (and this structure) may change in the future. The 200 | payload fields are stored as-is for now, as the information known to this date is too limited. Refer to the 201 | documentation for more information about the event table. 202 | 203 | Furthermore, the entry type is believed to be a single byte, so unless more information is known that changes 204 | this, the type is validated to be in the range 0-255. 205 | 206 | Each entry has a timestamp field that, depending on the type, either denotes the start time for a ranged event 207 | (such as the start of an error condition) or the precise time when the event occured (such as for a parameter 208 | change). 209 | 210 | The element-fields contain the raw value from the device. Use :func:`~rctclient.utils.decode_value` to decode them 211 | if the data type is known. 212 | 213 | :param entry_type: The type of the entry. 214 | :param timestamp: Timestamp of the entry (element1). 215 | :param element2: The second element of the entry. 216 | :param element3: The third element of the entry. 217 | :param element4: The fourth element of the entry. 218 | ''' 219 | entry_type: int 220 | timestamp: datetime 221 | element2: Optional[bytes] 222 | element3: Optional[bytes] 223 | element4: Optional[bytes] 224 | 225 | def __init__(self, entry_type: int, timestamp: datetime, element2: Optional[bytes] = None, 226 | element3: Optional[bytes] = None, element4: Optional[bytes] = None) -> None: 227 | if entry_type < 0 or entry_type > 0xff: 228 | raise ValueError(f'Entry type {entry_type} outside of range 0-255') 229 | self.entry_type = entry_type 230 | self.timestamp = timestamp 231 | self.element2 = element2 232 | self.element3 = element3 233 | self.element4 = element4 234 | 235 | def __repr__(self) -> str: 236 | return f'' 237 | -------------------------------------------------------------------------------- /src/rctclient/utils.py: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2020, Peter Oberhofer (pob90) 3 | # Copyright 2020-2021, Stefan Valouch (svalouch) 4 | # SPDX-License-Identifier: GPL-3.0-only 5 | 6 | import struct 7 | from datetime import datetime 8 | from typing import overload, Dict, Tuple, Union 9 | 10 | try: 11 | # Python 3.8+ 12 | from typing import Literal 13 | except ImportError: 14 | # Python < 3.8 15 | from typing_extensions import Literal 16 | 17 | from .types import DataType, EventEntry 18 | 19 | 20 | # pylint: disable=invalid-name 21 | def CRC16(data: Union[bytes, bytearray]) -> int: 22 | ''' 23 | Calculates the CRC16 checksum of data. Note that this automatically skips the first byte (start token) if the 24 | length is uneven. 25 | ''' 26 | crcsum = 0xFFFF 27 | polynom = 0x1021 # CCITT Polynom 28 | buffer = bytearray(data) 29 | 30 | # skip start token 31 | if len(data) & 0x01: 32 | buffer.append(0) 33 | 34 | for byte in buffer: 35 | crcsum ^= byte << 8 36 | for _bit in range(8): 37 | crcsum <<= 1 38 | if crcsum & 0x7FFF0000: 39 | # ~~ overflow in bit 16 40 | crcsum = (crcsum & 0x0000FFFF) ^ polynom 41 | return crcsum 42 | 43 | 44 | @overload 45 | def encode_value(data_type: Literal[DataType.BOOL], value: bool) -> bytes: 46 | ... 47 | 48 | 49 | @overload 50 | def encode_value(data_type: Union[Literal[DataType.INT8], Literal[DataType.UINT8], Literal[DataType.INT16], 51 | Literal[DataType.UINT16], Literal[DataType.INT32], Literal[DataType.UINT32], 52 | Literal[DataType.ENUM]], value: int) -> bytes: 53 | ... 54 | 55 | 56 | @overload 57 | def encode_value(data_type: Literal[DataType.FLOAT], value: float) -> bytes: 58 | ... 59 | 60 | 61 | @overload 62 | def encode_value(data_type: Literal[DataType.STRING], value: Union[str, bytes]) -> bytes: 63 | ... 64 | 65 | 66 | # pylint: disable=too-many-branches,too-many-return-statements 67 | def encode_value(data_type: DataType, value: Union[bool, bytes, float, int, str]) -> bytes: 68 | ''' 69 | Encodes a value suitable for transmitting as payload to the device. The actual encoding depends on the `data_type`. 70 | 71 | :param data_type: Data type of the `value` to be encoded. This selects the encoding mechanism. 72 | :param value: Data to be encoded according to the `data_type`. 73 | :return: The encoded value. 74 | :raises struct.error: If the packing failed, usually when the input value can't be encoded using the selected type. 75 | :raises ValueError: For string values, if the data type is not ``str`` or ``bytes``. 76 | ''' 77 | if data_type == DataType.BOOL: 78 | return struct.pack('>B', bool(value)) 79 | if data_type in (DataType.UINT8, DataType.ENUM): 80 | value = struct.unpack('B", value) 82 | if data_type == DataType.INT8: 83 | return struct.pack(">b", value) 84 | if data_type == DataType.UINT16: 85 | value = struct.unpack('H", value) 87 | if data_type == DataType.INT16: 88 | return struct.pack(">h", value) 89 | if data_type == DataType.UINT32: 90 | value = struct.unpack('I", value) 92 | if data_type == DataType.INT32: 93 | return struct.pack(">i", value) 94 | if data_type == DataType.FLOAT: 95 | return struct.pack(">f", value) 96 | if data_type == DataType.STRING: 97 | if isinstance(value, str): 98 | return value.encode('utf-8') 99 | if isinstance(value, bytes): 100 | return value 101 | raise ValueError(f'Invalid value of type {type(value)} for string type encoding') 102 | # return struct.pack("s", value) 103 | raise KeyError('Undefinded or unknown type') 104 | 105 | 106 | @overload 107 | def decode_value(data_type: Literal[DataType.BOOL], data: bytes) -> bool: 108 | ... 109 | 110 | 111 | @overload 112 | def decode_value(data_type: Union[Literal[DataType.INT8], Literal[DataType.UINT8], Literal[DataType.INT16], 113 | Literal[DataType.UINT16], Literal[DataType.INT32], Literal[DataType.UINT32], 114 | Literal[DataType.ENUM]], data: bytes) -> int: 115 | ... 116 | 117 | 118 | @overload 119 | def decode_value(data_type: Literal[DataType.FLOAT], data: bytes) -> float: 120 | ... 121 | 122 | 123 | @overload 124 | def decode_value(data_type: Literal[DataType.STRING], data: bytes) -> str: 125 | ... 126 | 127 | 128 | @overload 129 | def decode_value(data_type: Literal[DataType.TIMESERIES], data: bytes) -> Tuple[datetime, Dict[datetime, int]]: 130 | ... 131 | 132 | 133 | @overload 134 | def decode_value(data_type: Literal[DataType.EVENT_TABLE], data: bytes) -> Tuple[datetime, Dict[datetime, EventEntry]]: 135 | ... 136 | 137 | 138 | # pylint: disable=too-many-branches,too-many-return-statements 139 | def decode_value(data_type: DataType, data: bytes) -> Union[bool, bytes, float, int, str, 140 | Tuple[datetime, Dict[datetime, int]], 141 | Tuple[datetime, Dict[datetime, EventEntry]]]: 142 | ''' 143 | Decodes a value received from the device. 144 | 145 | .. note:: 146 | 147 | Values for a message id may be decoded using a different type than was used for encoding. For example, the 148 | logger history writes a unix timestamp and receives a timeseries data structure. 149 | 150 | :param data_type: Data type of the `value` to be decoded. This selects the decoding mechanism. 151 | :param value: The value to be decoded. 152 | :return: The decoded value, depending on the `data_type`. 153 | :raises struct.error: If decoding of native types failed. 154 | ''' 155 | if data_type == DataType.BOOL: 156 | value = struct.unpack(">B", data)[0] 157 | if value != 0: 158 | return True 159 | return False 160 | if data_type in (DataType.UINT8, DataType.ENUM): 161 | return struct.unpack(">B", data)[0] 162 | if data_type == DataType.INT8: 163 | return struct.unpack(">b", data)[0] 164 | if data_type == DataType.UINT16: 165 | return struct.unpack(">H", data)[0] 166 | if data_type == DataType.INT16: 167 | return struct.unpack(">h", data)[0] 168 | if data_type == DataType.UINT32: 169 | return struct.unpack(">I", data)[0] 170 | if data_type == DataType.INT32: 171 | return struct.unpack(">i", data)[0] 172 | if data_type == DataType.FLOAT: 173 | return struct.unpack(">f", data)[0] 174 | if data_type == DataType.STRING: 175 | pos = data.find(0x00) 176 | if pos == -1: 177 | return data.decode('ascii') 178 | return data[0:pos].decode('ascii') 179 | if data_type == DataType.TIMESERIES: 180 | return _decode_timeseries(data) 181 | if data_type == DataType.EVENT_TABLE: 182 | return _decode_event_table(data) 183 | raise KeyError(f'Undefined or unknown type {data_type}') 184 | 185 | 186 | def _decode_timeseries(data: bytes) -> Tuple[datetime, Dict[datetime, int]]: 187 | ''' 188 | Helper function to decode the timeseries type. 189 | ''' 190 | timestamp = datetime.fromtimestamp(struct.unpack('>I', data[0:4])[0]) 191 | tsval: Dict[datetime, int] = dict() 192 | assert len(data) % 4 == 0, 'Data should be divisible by 4' 193 | assert int(len(data) / 4 % 2) == 1, 'Data should be an even number of 4-byte pairs plus the starting timestamp' 194 | for pair in range(0, int(len(data) / 4 - 1), 2): 195 | pair_ts = datetime.fromtimestamp(struct.unpack('>I', data[4 + pair * 4:4 + pair * 4 + 4])[0]) 196 | pair_val = struct.unpack('>f', data[4 + pair * 4 + 4:4 + pair * 4 + 4 + 4])[0] 197 | tsval[pair_ts] = pair_val 198 | return timestamp, tsval 199 | 200 | 201 | def _decode_event_table(data: bytes) -> Tuple[datetime, Dict[datetime, EventEntry]]: 202 | ''' 203 | Helper function to decode the event table type. 204 | ''' 205 | timestamp = datetime.fromtimestamp(struct.unpack('>I', data[0:4])[0]) 206 | tabval: Dict[datetime, EventEntry] = dict() 207 | assert len(data) % 4 == 0 208 | assert (len(data) - 4) % 20 == 0 209 | for pair in range(0, int(len(data) / 4 - 1), 5): 210 | # this is most likely a single byte of information, but this is not sure yet 211 | # entry_type = bytes([struct.unpack('>I', data[4 + pair * 4:4 + pair * 4 + 4])[0]]).decode('ascii') 212 | entry_type = struct.unpack('>I', data[4 + pair * 4:4 + pair * 4 + 4])[0] 213 | timestamp = datetime.fromtimestamp(struct.unpack('>I', data[4 + pair * 4 + 4:4 + pair * 4 + 8])[0]) 214 | element2 = struct.unpack('>I', data[4 + pair * 4 + 8:4 + pair * 4 + 12])[0] 215 | element3 = struct.unpack('>I', data[4 + pair * 4 + 12:4 + pair * 4 + 16])[0] 216 | element4 = struct.unpack('>I', data[4 + pair * 4 + 16:4 + pair * 4 + 20])[0] 217 | tabval[timestamp] = EventEntry(entry_type=entry_type, timestamp=timestamp, element2=element2, element3=element3, 218 | element4=element4) 219 | # these two are known to contain object IDs 220 | # if entry_type in ['s', 'w']: 221 | # object_id = struct.unpack('>I', data[4 + pair * 4 + 8:4 + pair * 4 + 12])[0] 222 | # value_old = struct.unpack('>I', data[4 + pair * 4 + 12:4 + pair * 4 + 16])[0] 223 | # value_new = struct.unpack('>I', data[4 + pair * 4 + 16:4 + pair * 4 + 20])[0] 224 | # tabval[timestamp] = EventEntry(timestamp=timestamp, object_id=object_id, entry_type=entry_type, 225 | # value_old=value_old, value_new=value_new) 226 | # the rest is assumed to be range-based events 227 | # else: 228 | # timestamp_end = datetime.fromtimestamp( 229 | # struct.unpack('>I', data[4 + pair * 4 + 12:4 + pair * 4 + 16])[0]) 230 | # object_id = struct.unpack('>I', data[4 + pair * 4 + 16:4 + pair * 4 + 20])[0] 231 | # tabval[timestamp] = EventEntry(timestamp=timestamp, object_id=object_id, entry_type=entry_type, 232 | # timestamp_end=timestamp_end) 233 | return timestamp, tabval 234 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/svalouch/python-rctclient/9300ea947e6f45a100e6b598b8146c2e59a9d646/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_bytecode_generation.py: -------------------------------------------------------------------------------- 1 | 2 | ''' 3 | Tests the bytecode generator. 4 | Identical tests are done for the SendFrame class in test_sendframe.py. 5 | ''' 6 | 7 | import struct 8 | import pytest 9 | from rctclient.frame import make_frame 10 | from rctclient.types import Command 11 | 12 | # pylint: disable=no-self-use 13 | # TODO: escape sequences 14 | 15 | 16 | class TestBytecodeGenerator: 17 | ''' 18 | Tests for the bytecode generator. 19 | ''' 20 | 21 | @pytest.mark.parametrize('id_in,data_out', [(0x0, '2b0204000000000c56'), (0xc0de, '2b02040000c0de30b1'), 22 | (0xffffffff, '2b0204ffffffff9599')]) 23 | def test_write_standard_nopayload(self, id_in: int, data_out: str) -> None: 24 | ''' 25 | Tests the encoding of various write commands without payload. 26 | ''' 27 | assert make_frame(Command.WRITE, id_in) == bytearray.fromhex(data_out) 28 | 29 | @pytest.mark.parametrize('id_in,data_out', [(0x00, '2b010400000000c2b6'), (0xc0de, '2b01040000c0defe51'), 30 | (0xffffffff, '2b0104ffffffff5b79')]) 31 | def test_read_standard_nopayload(self, id_in: int, data_out: str) -> None: 32 | ''' 33 | Create a SendFrame and encode various read commands without payload. 34 | ''' 35 | assert make_frame(Command.READ, id_in) == bytearray.fromhex(data_out) 36 | 37 | @pytest.mark.parametrize('id_in,data_out', [(0x00, '2b06000400000000b754'), (0xc0de, '2b0600040000c0dea78b'), 38 | (0xffffffff, '2b060004ffffffff6ac4')]) 39 | def test_longresponse_standard_nopayload(self, id_in: int, data_out: str) -> None: 40 | ''' 41 | Create a SendFrame and encode various long response commands without payload. 42 | ''' 43 | assert make_frame(Command.LONG_RESPONSE, id_in) == bytearray.fromhex(data_out) 44 | 45 | @pytest.mark.parametrize('id_in,data_out', [(0x00, '2b050400000000c417'), (0xc0de, '2b05040000c0def8f0'), 46 | (0xffffffff, '2b0504ffffffff5dd8')]) 47 | def test_response_standard_nopayload(self, id_in: int, data_out: str) -> None: 48 | ''' 49 | Create a SendFrame and encode various response commands without payload. 50 | ''' 51 | assert make_frame(Command.RESPONSE, id_in) == bytearray.fromhex(data_out) 52 | 53 | def test_invalid_id_too_big(self): 54 | ''' 55 | Passing in an ID that is too big (more than 4 bytes) should result in an error. 56 | ''' 57 | with pytest.raises(struct.error): 58 | make_frame(Command.READ, 0xfffffffff) 59 | -------------------------------------------------------------------------------- /tests/test_decode_value.py: -------------------------------------------------------------------------------- 1 | 2 | ''' 3 | Tests for decode_value. 4 | ''' 5 | 6 | import pytest 7 | from rctclient.types import DataType 8 | from rctclient.utils import decode_value 9 | 10 | # pylint: disable=invalid-name,no-self-use 11 | 12 | 13 | class TestDecodeValue: 14 | ''' 15 | Tests for decode_value. 16 | ''' 17 | 18 | @pytest.mark.parametrize('data_in,data_out', [(b'\x00', False), (b'\x01', True), (b'\x02', True), 19 | (b'\xff', True)]) 20 | def test_BOOL_happy(self, data_in: bytes, data_out: bool) -> None: 21 | ''' 22 | Tests the boolean happy path. 23 | ''' 24 | assert decode_value(data_type=DataType.BOOL, data=data_in) == data_out 25 | 26 | @pytest.mark.parametrize('data_in,data_out', [(b'\x00', 0), (b'\x01', 1), (b'\x02', 2), (b'\xff', 255)]) 27 | def test_UINT8_happy(self, data_in: bytes, data_out: int) -> None: 28 | ''' 29 | Tests the uint8 happy path. 30 | ''' 31 | assert decode_value(data_type=DataType.UINT8, data=data_in) == data_out 32 | 33 | def test_STRING_happy_null(self) -> None: 34 | ''' 35 | Tests that a NULL terminated string can be decoded. 36 | ''' 37 | data = bytearray.fromhex('505320362e30204241334c00000000000000000000000000000000000000000000000000000000000000' 38 | '00000000000000000000000000000000000000000000') 39 | plain = 'PS 6.0 BA3L' 40 | result = decode_value(data_type=DataType.STRING, data=data) 41 | assert isinstance(result, str), 'The resulting type should be a string' 42 | assert result == plain 43 | 44 | def test_STRING_happy_nonull(self) -> None: 45 | ''' 46 | Tests that a not NULL terminated string can be decoded. 47 | ''' 48 | data = bytearray.fromhex('505320362e30204241334c') 49 | plain = 'PS 6.0 BA3L' 50 | result = decode_value(data_type=DataType.STRING, data=data) 51 | assert isinstance(result, str), 'The resulting type should be a string' 52 | assert result == plain 53 | -------------------------------------------------------------------------------- /tests/test_receiveframe.py: -------------------------------------------------------------------------------- 1 | 2 | ''' 3 | Tests the ReceiveFrame class. 4 | ''' 5 | 6 | import pytest 7 | from rctclient.frame import ReceiveFrame 8 | from rctclient.types import Command 9 | 10 | # pylint: disable=no-self-use 11 | # TODO: escape sequences 12 | 13 | 14 | class TestReceiveFrame: 15 | ''' 16 | Tests for ReceiveFrame. 17 | ''' 18 | 19 | @pytest.mark.parametrize('data_in,result', [('2b0204000000000c56', {'cmd': Command.WRITE, 'id': 0x0}), 20 | ('2b02040000c0de30b1', {'cmd': Command.WRITE, 'id': 0xc0de}), 21 | ('2b0204ffffffff9599', {'cmd': Command.WRITE, 'id': 0xffffffff}), 22 | ('2b010400000000c2b6', {'cmd': Command.READ, 'id': 0x0}), 23 | ('2b01040000c0defe51', {'cmd': Command.READ, 'id': 0xc0de}), 24 | ('2b0104ffffffff5b79', {'cmd': Command.READ, 'id': 0xffffffff}), 25 | ('2b06000400000000b754', {'cmd': Command.LONG_RESPONSE, 'id': 0x0}), 26 | ('2b0600040000c0dea78b', {'cmd': Command.LONG_RESPONSE, 'id': 0xc0de}), 27 | ('2b060004ffffffff6ac4', {'cmd': Command.LONG_RESPONSE, 28 | 'id': 0xffffffff}), 29 | ('2b050400000000c417', {'cmd': Command.RESPONSE, 'id': 0x0}), 30 | ('2b05040000c0def8f0', {'cmd': Command.RESPONSE, 'id': 0xc0de}), 31 | ('2b0504ffffffff5dd8', {'cmd': Command.RESPONSE, 'id': 0xffffffff}), 32 | ]) 33 | def test_read_standard_nopayload(self, data_in: str, result: dict) -> None: 34 | ''' 35 | Tests that payloadless data can be read. 36 | ''' 37 | data = bytearray.fromhex(data_in) 38 | frame = ReceiveFrame() 39 | assert frame.consume(data) == len(data), 'The frame should consume all the data' 40 | assert frame.complete(), 'The frame should be complete' 41 | assert frame.command == result['cmd'] 42 | assert frame.id == result['id'] 43 | assert frame.address == 0, 'Standard frames have no address' 44 | assert frame.data == b'', 'No data was attached, so the shouldn\'t be any' 45 | 46 | def test_read_standard_int(self) -> None: 47 | ''' 48 | Tests that a integer payload can be read. The data has a leading NULL byte, too. Response for 49 | `display_struct.brightness` from a real device. 50 | ''' 51 | data = bytearray.fromhex('002b050529bda75fffb8d2') 52 | frame = ReceiveFrame() 53 | assert frame.consume(data) == len(data), 'The frame should consume all the data' 54 | assert frame.complete(), 'The frame should be complete' 55 | assert frame.command == Command.RESPONSE 56 | assert frame.id == 0x29bda75f 57 | assert frame.address == 0, 'Standard frames have no address' 58 | assert frame.data == b'\xff' 59 | 60 | def test_read_standard_string(self) -> None: 61 | ''' 62 | Tests that a larger string can be read. Response for `android_name` from a real device. 63 | ''' 64 | data = bytearray.fromhex('002b0544ebc62737505320362e30204241334c0000000000000000000000000000000000000000000000' 65 | '000000000000000000000000000000000000000000000000000000000000476c') 66 | frame = ReceiveFrame() 67 | assert frame.consume(data) == len(data), 'The frame should consume all the data' 68 | assert frame.complete(), 'The frame should be complete' 69 | assert frame.command == Command.RESPONSE 70 | assert frame.id == 0xebc62737 71 | assert frame.address == 0, 'Standard frames have no address' 72 | assert frame.data == bytearray.fromhex('505320362e30204241334c000000000000000000000000000000000000000000000000' 73 | '0000000000000000000000000000000000000000000000000000000000') 74 | -------------------------------------------------------------------------------- /tests/test_sendframe.py: -------------------------------------------------------------------------------- 1 | 2 | ''' 3 | Tests the SendFrame class. 4 | Identical tests are done for the make_frame function in test_bytecode_generation.py. 5 | ''' 6 | 7 | import struct 8 | import pytest 9 | from rctclient.frame import SendFrame 10 | from rctclient.types import Command 11 | 12 | # pylint: disable=no-self-use 13 | # TODO: escape sequences 14 | 15 | 16 | class TestSendFrame: 17 | ''' 18 | Tests for SendFrame. 19 | ''' 20 | 21 | @pytest.mark.parametrize('id_in,data_out', [(0x0, '2b0204000000000c56'), (0xc0de, '2b02040000c0de30b1'), 22 | (0xffffffff, '2b0204ffffffff9599')]) 23 | def test_write_standard_nopayload(self, id_in: int, data_out: str) -> None: 24 | ''' 25 | Create a SendFrame and encode various write commands without payload. 26 | ''' 27 | assert SendFrame(Command.WRITE, id_in).data == bytearray.fromhex(data_out) 28 | 29 | @pytest.mark.parametrize('id_in,data_out', [(0x00, '2b010400000000c2b6'), (0xc0de, '2b01040000c0defe51'), 30 | (0xffffffff, '2b0104ffffffff5b79')]) 31 | def test_read_standard_nopayload(self, id_in: int, data_out: str) -> None: 32 | ''' 33 | Create a SendFrame and encode various read commands without payload. 34 | ''' 35 | assert SendFrame(Command.READ, id_in).data == bytearray.fromhex(data_out) 36 | 37 | @pytest.mark.parametrize('id_in,data_out', [(0x00, '2b06000400000000b754'), (0xc0de, '2b0600040000c0dea78b'), 38 | (0xffffffff, '2b060004ffffffff6ac4')]) 39 | def test_longresponse_standard_nopayload(self, id_in: int, data_out: str) -> None: 40 | ''' 41 | Create a SendFrame and encode various long response commands without payload. 42 | ''' 43 | assert SendFrame(Command.LONG_RESPONSE, id_in).data == bytearray.fromhex(data_out) 44 | 45 | @pytest.mark.parametrize('id_in,data_out', [(0x00, '2b050400000000c417'), (0xc0de, '2b05040000c0def8f0'), 46 | (0xffffffff, '2b0504ffffffff5dd8')]) 47 | def test_response_standard_nopayload(self, id_in: int, data_out: str) -> None: 48 | ''' 49 | Create a SendFrame and encode various response commands without payload. 50 | ''' 51 | assert SendFrame(Command.RESPONSE, id_in).data == bytearray.fromhex(data_out) 52 | 53 | def test_invalid_id_too_big(self): 54 | ''' 55 | Passing in an ID that is too big (more than 4 bytes) should result in an error. 56 | ''' 57 | with pytest.raises(struct.error): 58 | SendFrame(Command.READ, 0xfffffffff) 59 | -------------------------------------------------------------------------------- /tools/.gitignore: -------------------------------------------------------------------------------- 1 | *.csv 2 | *.yml 3 | -------------------------------------------------------------------------------- /tools/README.md: -------------------------------------------------------------------------------- 1 | TOOLS 2 | ===== 3 | 4 | This directory contains tools that fulfil special purposes or have dependencies that should not be pulled into the 5 | main module. Please head over to the [documentation](https://rctclient.readthedocs.io/en/latest/tools.html) for more 6 | information on the files in this folder. 7 | -------------------------------------------------------------------------------- /tools/csv2influxdb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ''' 4 | Imports CSV histogram data from file to influxdb. Intended to be used on the output of timeseries2csv.py. 5 | ''' 6 | 7 | # Copyright 2020-2021, Stefan Valouch (svalouch) 8 | # SPDX-License-Identifier: GPL-3.0-only 9 | 10 | import csv 11 | import sys 12 | from datetime import datetime, timedelta 13 | 14 | import click 15 | import requests 16 | from influxdb import InfluxDBClient # type: ignore 17 | 18 | # pylint: disable=too-many-arguments,too-many-locals 19 | 20 | 21 | def datetime_range(start: datetime, end: datetime, delta: timedelta): 22 | ''' 23 | Generator yielding datetime objects between `start` and `end` with `delta` increments. 24 | ''' 25 | current = start 26 | while current < end: 27 | yield current 28 | current += delta 29 | 30 | 31 | @click.command() 32 | @click.option('-i', '--input', type=click.Path(readable=True, dir_okay=False, allow_dash=True), required=True, 33 | help='Input CSV file (with headers). Supply "-" to read from standard input') 34 | @click.option('-n', '--device-name', type=str, default='rct1', help='Name of the device [rct1]') 35 | @click.option('-h', '--influx-host', type=str, default='localhost', help='InfluxDB hostname [localhost]') 36 | @click.option('-p', '--influx-port', type=int, default=8086, help='InfluxDB port [8086]') 37 | @click.option('-d', '--influx-db', type=str, default='rct', help='InfluxDB database name [rct]') 38 | @click.option('-u', '--influx-user', type=str, default='rct', help='InfluxDB user name [rct]') 39 | @click.option('-P', '--influx-pass', type=str, default='rct', help='InfluxDB password [rct]') 40 | @click.option('-r', '--resolution', type=click.Choice(['minutes', 'day', 'month', 'year']), default='day', 41 | help='Resolution of the input data') 42 | def csv2influxdb(input: str, device_name: str, influx_host: str, influx_port: int, influx_db: str, 43 | influx_user: str, influx_pass: str, resolution: str) -> None: 44 | 45 | ''' 46 | Reads a CSV file produced by `timeseries2csv.py` (requires headers) and pushes it to an InfluxDB v1.x database. 47 | This tool is intended to get you started and not a complete solution. It blindly trusts the timestamps and headers 48 | in the file. InfluxDB v2.x supports reading CSV natively using Flux and via the `influx write` command. 49 | 50 | The `--resolution` flag defines the name of the table/measurement into which the results are written. The schema is 51 | `history_${resolution}`. 52 | ''' 53 | if input == '-': 54 | fin = sys.stdin 55 | else: 56 | fin = open(input, 'rt') 57 | reader = csv.DictReader(fin) 58 | 59 | influx = InfluxDBClient(host=influx_host, port=influx_port, username=influx_user, password=influx_pass, 60 | database=influx_db) 61 | 62 | try: 63 | influx.ping() 64 | except requests.exceptions.ConnectionError as exc: 65 | click.echo(f'InfluxDB refused connection: {str(exc)}') 66 | sys.exit(2) 67 | 68 | points = [] 69 | for row in reader: 70 | points.append({ 71 | 'measurement': f'history_{resolution}', 72 | 'tags': { 73 | 'rct': device_name, 74 | }, 75 | 'time': row.pop('timestamp'), 76 | 'fields': {k: float(v) for k, v in row.items()}, 77 | }) 78 | 79 | if points: 80 | click.echo(f'Writing {len(points)} points') 81 | influx.write_points(time_precision='s', points=points) 82 | 83 | if input != '-': 84 | fin.close() 85 | 86 | 87 | if __name__ == '__main__': 88 | csv2influxdb() 89 | -------------------------------------------------------------------------------- /tools/read_pcap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ''' 4 | PCAP reader. This tool reads a pcap file (first and only argument) and parses the payload and presents the result. This 5 | is a debugging tool. As with the rest of the software, this tool comes with no warranty etc. whatsoever, use at your 6 | own risk. 7 | ''' 8 | 9 | # the ugly truth about debugging tools: 10 | # pylint: disable=too-many-statements,too-many-nested-blocks,too-many-branches,too-many-locals 11 | 12 | import struct 13 | import sys 14 | from datetime import datetime 15 | from scapy.utils import rdpcap # type: ignore 16 | from scapy.layers.inet import TCP # type: ignore 17 | from rctclient.exceptions import RctClientException, FrameCRCMismatch, InvalidCommand 18 | from rctclient.frame import ReceiveFrame 19 | from rctclient.registry import REGISTRY as R 20 | from rctclient.types import Command, DataType 21 | from rctclient.utils import decode_value 22 | 23 | 24 | def main(): 25 | ''' Main program ''' 26 | packets = rdpcap(sys.argv[1]) 27 | 28 | streams = dict() 29 | 30 | i = 0 31 | for name, stream in packets.sessions().items(): 32 | print(f'Stream {i:4} {name} {stream} ', end='') 33 | length = 0 34 | streams[i] = dict() 35 | for k in stream: 36 | if TCP in k: 37 | if len(k[TCP].payload) > 0: 38 | if k[TCP].sport == 8899 or k[TCP].dport == 8899: 39 | payload = bytes(k[TCP].payload) 40 | 41 | # skip AT+ keepalive and app serial "protocol switch" '2b3ce1' 42 | if payload in [b'AT+\r', b'+<\xe1']: 43 | continue 44 | ptime = float(k.time) 45 | if ptime not in streams[i]: 46 | streams[i][ptime] = b'' 47 | streams[i][ptime] += payload 48 | length += len(payload) 49 | print(f'{length} bytes') 50 | i += 1 51 | 52 | frame = None 53 | sid = 0 54 | for _, data in streams.items(): 55 | print(f'\nNEW STREAM #{sid}\n') 56 | 57 | for timestamp, data_item in data.items(): 58 | print(f'NEW INPUT: {datetime.fromtimestamp(timestamp):%Y-%m-%d %H:%M:%S.%f} | {data_item.hex()}') 59 | 60 | # frames should not cross segments (though it may be valid, but the devices haven't been observed doing 61 | # that). Sometimes, the device sends invalid data with a very high length field, causing the code to read 62 | # way byond the end of the actual data, causing it to miss frames until its length is satisfied. This way, 63 | # if the next segment starts with the typical 0x002b used by the devices, the current frame is dropped. 64 | # This way only on segment is lost. 65 | if frame and data_item[0:2] == b'\0+': 66 | print('Frame not complete at segment start, starting new frame.') 67 | print(f'command: {frame.command}, length: {frame.frame_length}, oid: 0x{frame.id:X}') 68 | frame = None 69 | 70 | while len(data_item) > 0: 71 | if not frame: 72 | frame = ReceiveFrame() 73 | try: 74 | i = frame.consume(data_item) 75 | except InvalidCommand as exc: 76 | if frame.command == Command.EXTENSION: 77 | print('Frame is an extension frame and we don\'t know how to parse it') 78 | else: 79 | print(f'Invalid command 0x{exc.command:x} received after consuming {exc.consumed_bytes} bytes') 80 | i = exc.consumed_bytes 81 | except FrameCRCMismatch as exc: 82 | print(f'CRC mismatch, got 0x{exc.received_crc:X} but calculated ' 83 | f'0x{exc.calculated_crc:X}. Buffer: {frame._buffer.hex()}') 84 | i = exc.consumed_bytes 85 | except struct.error as exc: 86 | print(f'skipping 2 bytes ahead as struct could not unpack: {str(exc)}') 87 | i = 2 88 | frame = ReceiveFrame() 89 | 90 | data_item = data_item[i:] 91 | print(f'frame consumed {i} bytes, {len(data_item)} remaining') 92 | if frame.complete(): 93 | if frame.id == 0: 94 | print(f'Frame complete: {frame} Buffer: {frame._buffer.hex()}') 95 | else: 96 | print(f'Frame complete: {frame}') 97 | try: 98 | rid = R.get_by_id(frame.id) 99 | except KeyError: 100 | print('Could not find ID in registry') 101 | else: 102 | if frame.command == Command.READ: 103 | print(f'Received read : {rid.name:40}') 104 | else: 105 | if frame.command in [Command.RESPONSE, Command.LONG_RESPONSE]: 106 | dtype = rid.response_data_type 107 | else: 108 | dtype = rid.request_data_type 109 | is_write = frame.command in [Command.WRITE, Command.LONG_WRITE] 110 | 111 | try: 112 | value = decode_value(dtype, frame.data) 113 | except (struct.error, UnicodeDecodeError) as exc: 114 | print(f'Could not decode value: {str(exc)}') 115 | if is_write: 116 | print(f'Received write : {rid.name:40} type: {dtype.name:17} value: UNKNOWN') 117 | else: 118 | print(f'Received reply : {rid.name:40} type: {dtype.name:17} value: UNKNOWN') 119 | except KeyError: 120 | print('Could not decode unknown type') 121 | if is_write: 122 | print(f'Received write : {rid.name:40} value: 0x{frame.data.hex()}') 123 | else: 124 | print(f'Received reply : {rid.name:40} value: 0x{frame.data.hex()}') 125 | else: 126 | if dtype == DataType.ENUM: 127 | try: 128 | value = rid.enum_str(value) 129 | except RctClientException as exc: 130 | print(f'ENUM mapping failed: {str(exc)}') 131 | except KeyError: 132 | print('ENUM value out of bounds') 133 | if is_write: 134 | print(f'Received write : {rid.name:40} type: {dtype.name:17} value: {value}') 135 | else: 136 | print(f'Received reply : {rid.name:40} type: {dtype.name:17} value: {value}') 137 | frame = None 138 | print() 139 | print('END OF INPUT-SEGMENT') 140 | sid += 1 141 | 142 | 143 | if __name__ == '__main__': 144 | main() 145 | -------------------------------------------------------------------------------- /tools/timeseries2csv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ''' 4 | Retrieves time series data from the device and converts it to CSV. 5 | ''' 6 | 7 | # Copyright 2020-2021, Stefan Valouch (svalouch) 8 | # SPDX-License-Identifier: GPL-3.0-only 9 | 10 | import csv 11 | import os 12 | import select 13 | import socket 14 | import struct 15 | import sys 16 | from datetime import datetime, timedelta 17 | from tempfile import mkstemp 18 | from typing import Dict, Optional 19 | 20 | import click 21 | import pytz 22 | from dateutil.relativedelta import relativedelta 23 | 24 | from rctclient.exceptions import FrameCRCMismatch 25 | from rctclient.frame import ReceiveFrame, make_frame 26 | from rctclient.registry import REGISTRY as R 27 | from rctclient.types import Command, DataType 28 | from rctclient.utils import decode_value, encode_value 29 | 30 | # pylint: disable=too-many-arguments,too-many-locals 31 | 32 | 33 | def datetime_range(start: datetime, end: datetime, delta: relativedelta): 34 | ''' 35 | Generator yielding datetime objects between `start` and `end` with `delta` increments. 36 | ''' 37 | current = start 38 | while current < end: 39 | yield current 40 | current += delta 41 | 42 | 43 | be_quiet: bool = False 44 | 45 | 46 | def cprint(text: str) -> None: 47 | ''' 48 | Custom print to output to stderr if quiet is not set. 49 | ''' 50 | if not be_quiet: 51 | click.echo(text, err=True) 52 | 53 | 54 | @click.command() 55 | @click.option('-h', '--host', type=str, required=True, help='Host to query') 56 | @click.option('-p', '--port', type=int, default=8899, help='Port on the host to query [8899]') 57 | @click.option('-o', '--output', type=click.Path(writable=True, dir_okay=False, allow_dash=True), required=False, 58 | help='Output file (use "-" for standard output), omit for "data__.csv"') 59 | @click.option('-H', '--header-format', type=click.Choice(['simple', 'influx2', 'none'], case_sensitive=False), 60 | default='simple', help='Header format [simple]') 61 | @click.option('--time-zone', type=str, default='Europe/Berlin', help='Timezone of the device (not the host running the' 62 | ' script) [Europe/Berlin].') 63 | @click.option('-q', '--quiet', type=bool, is_flag=True, default=False, help='Supress output.') 64 | @click.option('-r', '--resolution', type=click.Choice(['minutes', 'day', 'month', 'year'], case_sensitive=False), 65 | default='day', help='Resolution to query [minutes].') 66 | @click.option('-c', '--count', type=int, default=1, help='Amount of time to go back, depends on --resolution, see ' 67 | '--help.') 68 | @click.argument('DAY_BEFORE_TODAY', type=int) 69 | def timeseries2csv(host: str, port: int, output: Optional[str], header_format: bool, time_zone: str, quiet: bool, 70 | resolution: str, count: int, day_before_today: int) -> None: 71 | 72 | ''' 73 | Extract time series data from an RCT device. The tool works similar to the official App, but can be run 74 | independantly, it is designed to be run from a cronjob or as part of a script. 75 | 76 | The output format is CSV. If --output is not given, then a name is constructed from the resolution and the current 77 | date. Specify "-" to have the tool print the table to standard output, for use with other tools. 78 | 79 | Use --header-format to select the format of the first lines. If "none", no headers will be included. With "simple", 80 | a single line will be included naming all the columns. This is the preferred format for "csv2influx.py" as well as 81 | for importing into spreadsheet applications. Specify "influx2" to have a set of headers added that are meant for 82 | use with InfluxDB 2.x "influx write" command. See the documentation for details. 83 | 84 | Data is queried into the past, by specifying the latest point in time for which data should be queried. Thus, 85 | DAYS_BEFORE_TODAY selects the last second of the day that is the given amount in the past. 0 therefor is the 86 | incomplete current day, 1 is the end of yesterday etc. 87 | 88 | The device has multiple sampling memories at varying sampling intervals. The resolution can be selected using 89 | --resolution, which supports "minutes" (which is at 5 minute intervals), day, month and year. The amount of time 90 | to cover (back from the end of DAY_BEFORE_TODAY) can be selected using --count: 91 | 92 | * For --resolution=minute, if DAY_BEFORE_TODAY is 0 it selects the last --count hours up to the current time. 93 | 94 | * For --resolution=minute, if DAY_BEFORE_TODAY is greater than 0, it selects --count days back. 95 | 96 | * For all the other resolutions, --count selects the amount of days, months and years to go back, respectively. 97 | 98 | Note that the tool does not remove extra information: If the device sends more data than was requested, that extra 99 | data is included. 100 | 101 | Examples: 102 | 103 | * The previous 3 hours at finest resolution: --resolution=minutes --count=3 0 104 | 105 | * A whole day, 3 days ago, at finest resolution: --resolution=minutes --count=24 3 106 | 107 | * 4 Months back, at 1 month resolution: --resolution=month --count=4 0 108 | ''' 109 | global be_quiet 110 | be_quiet = quiet 111 | 112 | if count < 1: 113 | cprint('Error: --count must be a positive integer') 114 | sys.exit(1) 115 | 116 | timezone = pytz.timezone(time_zone) 117 | now = datetime.now() 118 | 119 | if resolution == 'minutes': 120 | oid_names = ['logger.minutes_ubat_log_ts', 'logger.minutes_ul3_log_ts', 'logger.minutes_ub_log_ts', 121 | 'logger.minutes_temp2_log_ts', 'logger.minutes_eb_log_ts', 'logger.minutes_eac1_log_ts', 122 | 'logger.minutes_eext_log_ts', 'logger.minutes_ul2_log_ts', 'logger.minutes_ea_log_ts', 123 | 'logger.minutes_soc_log_ts', 'logger.minutes_ul1_log_ts', 'logger.minutes_eac2_log_ts', 124 | 'logger.minutes_eac_log_ts', 'logger.minutes_ua_log_ts', 'logger.minutes_soc_targ_log_ts', 125 | 'logger.minutes_egrid_load_log_ts', 'logger.minutes_egrid_feed_log_ts', 126 | 'logger.minutes_eload_log_ts', 'logger.minutes_ebat_log_ts', 'logger.minutes_temp_bat_log_ts', 127 | 'logger.minutes_eac3_log_ts', 'logger.minutes_temp_log_ts'] 128 | # the prefix is cut from the front of individual oid_names to produce the name (the end is cut off, too) 129 | name_prefix = 'logger.minutes_' 130 | # one sample every 5 minutes 131 | timediff = relativedelta(minutes=5) 132 | # select whole days when not querying the current day 133 | if day_before_today > 0: 134 | # lowest timestamp that's of interest 135 | ts_start = (now - timedelta(days=day_before_today)).replace(hour=0, minute=0, second=0, microsecond=0) 136 | # highest timestamp, we stop when this is reached 137 | ts_end = ts_start.replace(hour=23, minute=59, second=59, microsecond=0) 138 | else: 139 | ts_start = ((now - (now - datetime.min) % timedelta(minutes=30)) - timedelta(hours=count)) \ 140 | .replace(second=0, microsecond=0) 141 | ts_end = now.replace(second=59, microsecond=0) 142 | 143 | elif resolution == 'day': 144 | oid_names = ['logger.day_ea_log_ts', 'logger.day_eac_log_ts', 'logger.day_eb_log_ts', 145 | 'logger.day_eext_log_ts', 'logger.day_egrid_feed_log_ts', 'logger.day_egrid_load_log_ts', 146 | 'logger.day_eload_log_ts'] 147 | name_prefix = 'logger.day_' 148 | # one sample every day 149 | timediff = relativedelta(days=1) 150 | # days 151 | ts_start = (now - timedelta(days=day_before_today + count)) \ 152 | .replace(hour=0, minute=59, second=59, microsecond=0) 153 | ts_end = (now - timedelta(days=day_before_today)).replace(hour=23, minute=59, second=59, microsecond=0) 154 | elif resolution == 'month': 155 | oid_names = ['logger.month_ea_log_ts', 'logger.month_eac_log_ts', 'logger.month_eb_log_ts', 156 | 'logger.month_eext_log_ts', 'logger.month_egrid_feed_log_ts', 'logger.month_egrid_load_log_ts', 157 | 'logger.month_eload_log_ts'] 158 | name_prefix = 'logger.month_' 159 | # one sample per month 160 | timediff = relativedelta(months=1) 161 | # months 162 | ts_start = (now - timedelta(days=day_before_today) - relativedelta(months=count)) \ 163 | .replace(day=2, hour=0, minute=59, second=59, microsecond=0) 164 | if ts_start.year < 2000: 165 | ts_start = ts_start.replace(year=2000) 166 | ts_end = (now - timedelta(days=day_before_today)).replace(day=2, hour=23, minute=59, second=59, microsecond=0) 167 | elif resolution == 'year': 168 | oid_names = ['logger.year_ea_log_ts', 'logger.year_eac_log_ts', 'logger.year_eb_log_ts', 169 | 'logger.year_eext_log_ts', 'logger.year_egrid_feed_log_ts', 'logger.year_egrid_load_log_ts', 170 | 'logger.year_eload_log_ts'] # , 'logger.year_log_ts'] 171 | name_prefix = 'logger.year_' 172 | # one sample per year 173 | timediff = relativedelta(years=1) 174 | # years 175 | ts_start = (now - timedelta(days=day_before_today) - relativedelta(years=count)) \ 176 | .replace(month=1, day=2, hour=0, minute=59, second=59, microsecond=0) 177 | ts_end = (now - timedelta(days=day_before_today)) \ 178 | .replace(month=1, day=2, hour=23, minute=59, second=59, microsecond=0) 179 | else: 180 | cprint('Unsupported resolution') 181 | sys.exit(1) 182 | 183 | if day_before_today < 0: 184 | cprint('DAYS_BEFORE_TODAY must be a positive number') 185 | sys.exit(1) 186 | if day_before_today > 365: 187 | cprint('DAYS_BEFORE_TODAY must be less than a year ago') 188 | sys.exit(1) 189 | 190 | oids = [x for x in R.all() if x.name in oid_names] 191 | 192 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 193 | try: 194 | sock.connect((host, port)) 195 | except ConnectionRefusedError: 196 | cprint('Device refused connection') 197 | sys.exit(2) 198 | 199 | datetable: Dict[datetime, Dict[str, int]] = {dt: dict() for dt in datetime_range(ts_start, ts_end, timediff)} 200 | 201 | for oid in oids: 202 | name = oid.name.replace(name_prefix, '').replace('_log_ts', '') 203 | cprint(f'Requesting {name}') 204 | 205 | # set to true if the current time series reached its end, e.g. year 2000 for "year" resolution 206 | iter_end = False 207 | highest_ts = ts_end 208 | 209 | while highest_ts > ts_start and not iter_end: 210 | cprint(f'\ttimestamp: {highest_ts}') 211 | sock.send(make_frame(command=Command.WRITE, id=oid.object_id, 212 | payload=encode_value(DataType.INT32, int(highest_ts.timestamp())))) 213 | 214 | rframe = ReceiveFrame() 215 | while True: 216 | try: 217 | rread, _, _ = select.select([sock], [], [], 2) 218 | except select.error as exc: 219 | cprint(f'Select error: {str(exc)}') 220 | raise 221 | 222 | if rread: 223 | buf = sock.recv(1024) 224 | if len(buf) > 0: 225 | try: 226 | rframe.consume(buf) 227 | except FrameCRCMismatch: 228 | cprint('\tCRC error') 229 | break 230 | if rframe.complete(): 231 | break 232 | else: 233 | cprint('Device closed connection') 234 | sys.exit(2) 235 | else: 236 | cprint('\tTimeout, retrying') 237 | break 238 | 239 | if not rframe.complete(): 240 | cprint('\tIncomplete frame, retrying') 241 | continue 242 | 243 | # in case something (such as a "net.package") slips in, make sure to ignore all irelevant responses 244 | if rframe.id != oid.object_id: 245 | cprint(f'\tGot unexpected frame oid 0x{rframe.id:08X}') 246 | continue 247 | 248 | try: 249 | _, table = decode_value(DataType.TIMESERIES, rframe.data) 250 | except (AssertionError, struct.error): 251 | # the device sent invalid data with the correct CRC 252 | cprint('\tInvalid data received, retrying') 253 | continue 254 | 255 | # work with the data 256 | for t_ts, t_val in table.items(): 257 | 258 | # set the "highest" point in time to know what to request next when the day is not complete 259 | if t_ts < highest_ts: 260 | highest_ts = t_ts 261 | 262 | # break if we reached the end of the day 263 | if t_ts < ts_start: 264 | cprint('\tReached limit') 265 | break 266 | 267 | # Check if the timestamp fits the raster, adjust depending on the resolution 268 | if t_ts not in datetable: 269 | if resolution == 'minutes': 270 | # correct up to one full minute 271 | nt_ts = t_ts.replace(second=0) 272 | if nt_ts not in datetable: 273 | nt_ts = t_ts.replace(second=0, minute=t_ts.minute + 1) 274 | if nt_ts not in datetable: 275 | cprint(f'\t{t_ts} does not fit raster, skipped') 276 | continue 277 | t_ts = nt_ts 278 | elif resolution in ['day', 'month']: 279 | # correct up to one hour 280 | nt_ts = t_ts.replace(hour=0) 281 | if nt_ts not in datetable: 282 | nt_ts = t_ts.replace(hour=t_ts.hour + 1) 283 | if nt_ts not in datetable: 284 | cprint(f'\t{t_ts} does not fit raster, skipped') 285 | continue 286 | t_ts = nt_ts 287 | datetable[t_ts][name] = t_val 288 | 289 | # year statistics stop at 2000-01-02 00:59:59, so if the year hits 2000 we know we're done 290 | if resolution == 'year' and t_ts.year == 2000: 291 | iter_end = True 292 | 293 | if output is None: 294 | output = f'data_{resolution}_{ts_start.isoformat("T")}.csv' 295 | 296 | if output == '-': 297 | fd = sys.stdout 298 | else: 299 | filedes, filepath = mkstemp(dir=os.path.dirname(output), text=True) 300 | fd = open(filedes, 'wt') 301 | 302 | if header_format == 'influx2': 303 | fd.write(f'#constant measurement,{resolution}\n') 304 | fd.write('#datatype dateTime,field,field,field,field,field,field,field,field,field,field,field,field,field,' 305 | 'field,field,field,field,field,field,field,field,field\n') 306 | 307 | writer = csv.writer(fd) 308 | 309 | names = [oid.name.replace(name_prefix, '').replace('_log_ts', '') for oid in oids] 310 | 311 | if header_format == 'simple': 312 | writer.writerow(['timestamp'] + names) 313 | 314 | for bts, btval in datetable.items(): 315 | if btval: # there may be holes in the data 316 | writer.writerow([timezone.localize(bts).isoformat('T')] + [str(btval[name]) for name in names]) 317 | 318 | if output != '-': 319 | fd.flush() 320 | os.fsync(fd.fileno()) 321 | try: 322 | os.rename(filepath, output) 323 | except OSError as exc: 324 | cprint(f'Could not move destination file: {str(exc)}') 325 | try: 326 | os.unlink(filepath) 327 | except Exception: 328 | cprint(f'Could not remove temporary file {filepath}') 329 | sys.exit(1) 330 | 331 | 332 | if __name__ == '__main__': 333 | timeseries2csv() 334 | --------------------------------------------------------------------------------