├── requirements.txt ├── flipper_raw_rfid ├── __init__.py ├── stuff.py ├── bits.py ├── cli.py ├── rifl.py └── utils.py ├── setup.py ├── MANIFEST.in ├── docs └── signal-plot.png ├── tests ├── assets │ ├── Red354.ask.raw │ ├── Red354.psk.raw │ ├── Nothing.ask.raw │ ├── Nothing.psk.raw │ ├── Red354b.ask.raw │ └── Red354b.psk.raw ├── test_rifl_file.py └── test_utils.py ├── requirements-dev.txt ├── pyproject.toml ├── .gitignore ├── LICENSE ├── Makefile ├── setup.cfg └── README.md /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scipy 3 | docopt -------------------------------------------------------------------------------- /flipper_raw_rfid/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.2' 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | include requirements.txt 4 | include *.py -------------------------------------------------------------------------------- /docs/signal-plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hnesk/flipper-raw-rfid/HEAD/docs/signal-plot.png -------------------------------------------------------------------------------- /tests/assets/Red354.ask.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hnesk/flipper-raw-rfid/HEAD/tests/assets/Red354.ask.raw -------------------------------------------------------------------------------- /tests/assets/Red354.psk.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hnesk/flipper-raw-rfid/HEAD/tests/assets/Red354.psk.raw -------------------------------------------------------------------------------- /tests/assets/Nothing.ask.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hnesk/flipper-raw-rfid/HEAD/tests/assets/Nothing.ask.raw -------------------------------------------------------------------------------- /tests/assets/Nothing.psk.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hnesk/flipper-raw-rfid/HEAD/tests/assets/Nothing.psk.raw -------------------------------------------------------------------------------- /tests/assets/Red354b.ask.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hnesk/flipper-raw-rfid/HEAD/tests/assets/Red354b.ask.raw -------------------------------------------------------------------------------- /tests/assets/Red354b.psk.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hnesk/flipper-raw-rfid/HEAD/tests/assets/Red354b.psk.raw -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | wheel 2 | setuptools 3 | build 4 | unittest-xml-reporting 5 | html-testRunner 6 | flake8 7 | mypy 8 | codespell 9 | matplotlib -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | # Minimum requirements for the build system to execute. 3 | requires = ["setuptools>=42", "wheel"] # PEP 508 specifications. 4 | build-backend = "setuptools.build_meta" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # Distribution / packaging 6 | build/ 7 | dist/ 8 | .eggs/ 9 | sdist/ 10 | *.egg-info/ 11 | MANIFEST 12 | 13 | # Unit test / coverage reports 14 | htmlcov/ 15 | .tox/ 16 | .nox/ 17 | build_artifacts/ 18 | 19 | # Translations 20 | *.mo 21 | *.pot 22 | 23 | # Environments 24 | .env 25 | .venv 26 | env/ 27 | venv*/ 28 | ENV/ 29 | env.bak/ 30 | venv*.bak/ 31 | 32 | # mypy 33 | .mypy_cache/ 34 | .dmypy.json 35 | dmypy.json 36 | 37 | # Pyre type checker 38 | .pyre/ 39 | 40 | # Autosaved glade files 41 | *.ui# 42 | *.ui~ 43 | 44 | /.coverage 45 | .ipynb_checkpoints/ 46 | /unittest.xml 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © 2020 Johannes Künsebeck 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the "Software"), 5 | to deal in the Software without restriction, including without limitation 6 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 7 | and/or sell copies of the Software, and to permit persons to whom the 8 | Software is furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included 11 | in all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 14 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 15 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 16 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 18 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE 19 | OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export 2 | 3 | SHELL = /bin/bash 4 | PYTHON = python3 5 | PIP = pip3 6 | LOG_LEVEL = INFO 7 | PYTHONIOENCODING=utf8 8 | SHARE_DIR=~/.local/share 9 | 10 | deps-dev: 11 | $(PIP) install -r requirements-dev.txt 12 | 13 | deps: 14 | $(PIP) install -r requirements.txt 15 | 16 | install: 17 | $(PIP) install -e . 18 | 19 | clean-build: pyclean 20 | rm -Rf build dist *.egg-info 21 | 22 | pyclean: 23 | rm -f **/*.pyc 24 | rm -rf .pytest_cache 25 | rm -rf .mypy_cache/ 26 | 27 | build: clean-build 28 | $(PYTHON) -m build 29 | 30 | testpypi: clean-build build 31 | twine upload --repository testpypi ./dist/flipper[_-]raw[_-]rfid*.{tar.gz,whl} 32 | 33 | pypi: clean-build build 34 | twine upload ./dist/flipper[_-]raw[_-]rfid*.{tar.gz,whl} 35 | 36 | flake8: deps-dev 37 | $(PYTHON) -m flake8 flipper_raw_rfid tests 38 | 39 | mypy: deps-dev 40 | $(PYTHON) -m mypy --show-error-codes -p flipper_raw_rfid 41 | 42 | codespell: deps-dev 43 | codespell 44 | 45 | test: deps-dev 46 | $(PYTHON) -m xmlrunner discover -v -s tests --output-file $(CURDIR)/unittest.xml 47 | 48 | ci: flake8 mypy test codespell 49 | 50 | 51 | .PHONY: assets-clean 52 | # Remove symlinks in test/assets 53 | assets-clean: 54 | rm -rf test/assets 55 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = flipper-raw-rfid 3 | version = attr: flipper_raw_rfid.__version__ 4 | description = A library for reading and analyzing Flipper Zero raw RFID files 5 | author = Johannes Künsebeck 6 | author_email = kuensebeck@googlemail.com 7 | url = https://github.com/hnesk/flipper-raw-rfid 8 | long_description = file: README.md 9 | long_description_content_type = text/markdown 10 | keywords = 11 | flipper-zero 12 | flipper 13 | RFID 14 | license = MIT License 15 | license_files = LICENSE 16 | classifiers = 17 | Programming Language :: Python :: 3.8 18 | Programming Language :: Python :: 3.9 19 | Programming Language :: Python :: 3.10 20 | License :: OSI Approved :: MIT License 21 | Development Status :: 5 - Production/Stable 22 | Intended Audience :: Developers 23 | Intended Audience :: Science/Research 24 | 25 | [options] 26 | python_requires = >=3.8 27 | install_requires = file:requirements.txt 28 | setup_requires = 29 | wheel 30 | packages = find: 31 | 32 | include_package_data = True 33 | 34 | [options.packages.find] 35 | include = 36 | flipper_raw_rfid* 37 | 38 | 39 | [options.entry_points] 40 | console_scripts = 41 | flipper-raw-rfid = flipper_raw_rfid.cli:main 42 | 43 | 44 | [flake8] 45 | ignore=E501 46 | 47 | [mypy] 48 | warn_return_any = True 49 | warn_unused_configs = True 50 | warn_unreachable = True 51 | warn_redundant_casts = True 52 | warn_unused_ignores = True 53 | implicit_reexport = False 54 | disallow_any_generics = True 55 | strict_optional = False 56 | disallow_untyped_defs = True 57 | plugins = numpy.typing.mypy_plugin 58 | 59 | [mypy-scipy.*] 60 | ignore_missing_imports = True 61 | 62 | 63 | [codespell] 64 | skip = ./venv*,docs* 65 | count = 66 | quiet-level = 2 -------------------------------------------------------------------------------- /flipper_raw_rfid/stuff.py: -------------------------------------------------------------------------------- 1 | """ 2 | Some useful functions not used in the main code yet 3 | """ 4 | from typing import Any, Generator 5 | 6 | import numpy 7 | import numpy.typing as npt 8 | 9 | from flipper_raw_rfid.utils import find_peaks 10 | 11 | 12 | def correlation_offset(d1: npt.NDArray[numpy.int32], d2: npt.NDArray[numpy.int32]) -> int: 13 | """ 14 | Find the offset where two signals would correlate best 15 | :param d1: 16 | :param d2: 17 | :return: 18 | """ 19 | middle = len(d1) // 2 20 | cor = numpy.correlate(d1, d2, mode='same') 21 | return find_peaks(cor)[0].center - middle 22 | 23 | 24 | def roll(a: npt.NDArray[Any], shift: int) -> npt.NDArray[Any]: 25 | """ 26 | Like numpy roll but with padding instead of wrapping around 27 | :param a: array 28 | :param shift: amount to shift 29 | :return: shifted array 30 | """ 31 | if shift > 0: 32 | return numpy.pad(a[:-shift], (shift, 0)) 33 | else: 34 | return numpy.pad(a[-shift:], (0, -shift)) 35 | 36 | 37 | def rationalizations(x: float, rtol: float = 1e-05, atol: float = 1e-08) -> Generator[tuple[int, int], None, None]: 38 | """ 39 | Find short rational approximations of a float 40 | 41 | :param x: the float to approximate 42 | :param rtol: Relative tolerance, see numpy.isclose 43 | :param atol: Absolute tolerance, see numpy.isclose 44 | :return: numerator and denominator of the approximation 45 | """ 46 | assert 0 <= x 47 | ix = int(x) 48 | yield ix, 1 49 | if numpy.isclose(x, ix, rtol, atol): 50 | return 51 | for numer, denom in rationalizations(1.0 / (x - ix), rtol, atol): 52 | yield denom + ix * numer, numer 53 | 54 | 55 | def find_fundamental(fs: list[float], rtol: float = 1e-3) -> tuple[float, npt.NDArray[numpy.int32]]: 56 | """ 57 | Find the fundamental frequency of a list of frequencies 58 | 59 | :param fs: frequencies 60 | :param rtol: Relative tolerance, see numpy.isclose 61 | :return: fundamental frequency and multiplication factors for each frequency in fs 62 | """ 63 | f0 = fs[0] 64 | res = numpy.zeros((len(fs), 2), dtype=numpy.int32) 65 | for i, f in enumerate(fs): 66 | res[i] = list(rationalizations(f0 / f, rtol=rtol))[-1] 67 | 68 | lcm = numpy.lcm.reduce(res[:, 0]) 69 | common = (lcm // res[:, 0] * res.T)[1] 70 | return numpy.mean(fs / common), common 71 | -------------------------------------------------------------------------------- /tests/test_rifl_file.py: -------------------------------------------------------------------------------- 1 | from io import BytesIO 2 | from pathlib import Path 3 | from unittest import TestCase 4 | 5 | import numpy 6 | from numpy.testing import assert_array_equal 7 | 8 | from flipper_raw_rfid.rifl import Rifl, RiflHeader 9 | 10 | TEST_BASE_PATH = Path(__file__).parent.absolute() 11 | 12 | 13 | class RiflFileTest(TestCase): 14 | 15 | example_bytes = bytes.fromhex('f101a903ae028506a604fb05bb028706ad04b90404c403') 16 | example_ints = [241, 425, 302, 773, 550, 763, 315, 775, 557, 569, 4, 452] 17 | 18 | def test_header_to_bytes_and_back(self): 19 | header = RiflHeader(1, 125_000, 0.5, 2048) 20 | self.assertEqual(header, RiflHeader.from_bytes(header.to_bytes())) 21 | 22 | def test_header_checks_magic(self): 23 | RiflHeader.from_bytes(bytes.fromhex('5249464C 01000000 0024F447 0000003F 00080000')) 24 | 25 | with self.assertRaisesRegex(ValueError, 'Not a RIFL file'): 26 | RiflHeader.from_bytes(bytes.fromhex('C0FFEEAA 01000000 0024F447 0000003F 00080000')) 27 | 28 | def test_header_checks_version(self): 29 | header = RiflHeader(2, 125_000, 0.5, 2048) 30 | with self.assertRaisesRegex(ValueError, 'Unsupported RIFL Version 2'): 31 | RiflHeader.from_bytes(header.to_bytes()) 32 | 33 | def test_read_varint(self): 34 | buffer = BytesIO(self.example_bytes) 35 | self.assertEqual(self.example_ints, list(Rifl.read_varint(buffer))) 36 | 37 | def test_write_varint(self): 38 | buffer = BytesIO() 39 | for v in self.example_ints: 40 | Rifl.write_varint(buffer, v) 41 | self.assertEqual(self.example_bytes, buffer.getvalue()) 42 | 43 | def test_to_and_from_io(self): 44 | # Load file 45 | rifl = Rifl.load(TEST_BASE_PATH / 'assets' / 'Red354.ask.raw') 46 | buffer = BytesIO() 47 | rifl.to_io(buffer) 48 | 49 | # Load from buffer, should be the same 50 | rifl2 = Rifl.from_io(BytesIO(buffer.getvalue())) 51 | buffer2 = BytesIO() 52 | rifl2.to_io(buffer2) 53 | 54 | self.assertEqual(rifl.header, rifl2.header) 55 | assert_array_equal(rifl.pulse_and_durations, rifl2.pulse_and_durations) 56 | self.assertEqual(buffer.getvalue(), buffer2.getvalue()) 57 | 58 | def test_save_and_load(self): 59 | dummy_file = TEST_BASE_PATH / 'assets' / 'dummy.ask.raw' 60 | 61 | header = RiflHeader(1, 125000.0, 0.5, 2048) 62 | pads = numpy.reshape(self.example_ints, (-1, 2)) 63 | dummy1 = Rifl(header, pads) 64 | dummy1.save(dummy_file) 65 | 66 | dummy2 = Rifl.load(dummy_file) 67 | self.assertEqual(dummy1.header, dummy2.header) 68 | assert_array_equal(dummy1.pulse_and_durations, dummy2.pulse_and_durations) 69 | 70 | dummy_file.unlink() 71 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from itertools import zip_longest 2 | from pathlib import Path 3 | from unittest import TestCase 4 | 5 | import numpy 6 | from numpy.testing import assert_array_equal 7 | 8 | from flipper_raw_rfid.rifl import Rifl 9 | from flipper_raw_rfid.utils import pad_to_signal, signal_to_pad 10 | 11 | TEST_BASE_PATH = (Path(__file__).parent).absolute() 12 | 13 | 14 | class UtilsTest(TestCase): 15 | 16 | @property 17 | def a_signal(self): 18 | signal = numpy.zeros(10000, dtype=numpy.int8) 19 | start = 0 20 | signal[start:start + 310] = 1 21 | start += 515 22 | signal[start:start + 274] = 1 23 | start += 527 24 | signal[start:start + 252] = 1 25 | start += 743 26 | signal[start:start + 291] = 1 27 | start += 534 28 | signal[start:start + 515] = 1 29 | start += 1016 30 | signal[start:start + 266] = 1 31 | start += 515 32 | 33 | return signal[:start] 34 | 35 | @property 36 | def a_pad(self): 37 | return numpy.array([ 38 | [310, 515], 39 | [274, 527], 40 | [252, 743], 41 | [291, 534], 42 | [515, 1016], 43 | [266, 515] 44 | ]) 45 | 46 | def load(self, file: str | Path) -> Rifl: 47 | return Rifl.load(TEST_BASE_PATH / 'assets' / file) 48 | 49 | def test_pad_to_signal(self): 50 | 51 | signal = pad_to_signal(self.a_pad) 52 | 53 | # First row, a pulse from 0-309 then a zero from 310 - 514 54 | self.assertEqual(signal[0], 1) 55 | self.assertEqual(signal[309], 1) 56 | self.assertEqual(signal[310], 0) 57 | self.assertEqual(signal[514], 0) 58 | 59 | # Second row, a pulse for 274 samples starting at 515 60 | self.assertEqual(signal[515], 1) 61 | self.assertEqual(signal[515 + 273], 1) 62 | self.assertEqual(signal[515 + 274], 0) 63 | self.assertEqual(signal[515 + 527 - 1], 0) 64 | 65 | self.assertEqual(signal[515 + 527], 1) 66 | # and so on 67 | 68 | assert_array_equal(signal, self.a_signal) 69 | 70 | def test_signal_to_pad(self): 71 | pad = signal_to_pad(self.a_signal) 72 | assert_array_equal(pad, self.a_pad) 73 | 74 | def test_signal_to_pad_and_back(self): 75 | 76 | def test(signal): 77 | rec_signal = pad_to_signal(signal_to_pad(signal)) 78 | assert_array_equal(signal, rec_signal) 79 | 80 | test(self.a_signal) 81 | test(pad_to_signal(self.load('Red354b.ask.raw').pulse_and_durations)) 82 | 83 | def test_pad_to_signal_and_back(self): 84 | 85 | def test(pad): 86 | rec_pad = signal_to_pad(pad_to_signal(pad)) 87 | 88 | pos = 0 89 | for i, (pd, rpd) in enumerate(zip_longest(pad, rec_pad)): 90 | if not numpy.array_equal(pd, rpd): 91 | assert_array_equal(pd, rpd, err_msg=f'Difference in reconstructed pad in row {i}, sample {pos}') 92 | pos += pd[1] 93 | 94 | test(self.a_pad) 95 | test(self.load('Red354b.ask.raw').pulse_and_durations) 96 | # This fails because of 0 pulse width 97 | # test(self.load_pad('Red354.ask.raw')) 98 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Flipper Zero Raw RFID Tools 2 | 3 | A python library for reading and analyzing Flipper Zero raw RFID files (`tag.[ap]sk.raw`) 4 | * [Installation](#installation) 5 | * [Via pip](#via-pip) 6 | * [From source](#from-source) 7 | * [Usage](#usage) 8 | * [As a library](#as-a-library) 9 | * [From commandline](#from-commandline) 10 | * [Tutorial](#tutorial) 11 | 12 | ## Installation 13 | 14 | ### Via pip 15 | 16 | ```bash 17 | pip install flipper-raw-rfid 18 | ``` 19 | 20 | ### From source 21 | ```bash 22 | git clone https://github.com/hnesk/flipper-raw-rfid.git 23 | cd flipper-raw-rfid 24 | make install 25 | ``` 26 | 27 | 28 | ## Usage 29 | 30 | ### As a library 31 | 32 | ``` python 33 | 34 | from flipper_raw_rfid.rifl import Rifl 35 | from flipper_raw_rfid.utils import pad_to_signal 36 | import matplotlib.pyplot as plt 37 | 38 | rifl = Rifl.load('test/assets/Red354.ask.raw') 39 | # Read the the raw pulse and duration values 40 | pad = rifl.pulse_and_durations 41 | # reconstructed binary signal 42 | signal = pad_to_signal(pad) 43 | 44 | plt.plot(signal[0:20000]) 45 | 46 | ``` 47 | 48 | results in: 49 | 50 | ![Plot of the RFID signal with matplotlib](docs/signal-plot.png) 51 | 52 | 53 | There is also a short [tutorial notebook](docs/rifl-tutorial-1.ipynb) 54 | 55 | ### From commandline 56 | 57 | ``` bash 58 | # Plot a file (requires matplotlib) 59 | $ flipper-raw-rfid plot tests/assets/Red354.ask.raw 60 | # Dump the contents in pad format (see below) 61 | $ flipper-raw-rfid convert --format=pad tests/assets/Red354.ask.raw Red354.pad.csv 62 | # Dump the contents in signal format 63 | $ flipper-raw-rfid convert --format=signal tests/assets/Red354.ask.raw Red354.signal.csv 64 | ``` 65 | 66 | #### Commandline help 67 | ```bash 68 | flipper-raw-rfid --help 69 | ``` 70 | ``` 71 | flipper-raw-rfid 72 | 73 | Description: 74 | Reads a raw rfid file from flipper zero and plots or converts the signal 75 | 76 | Usage: 77 | flipper-raw-rfid convert [-f ] RAW_FILE [OUTPUT_FILE] 78 | flipper-raw-rfid plot RAW_FILE 79 | flipper-raw-rfid check RAW_FILE 80 | flipper-raw-rfid (-h | --help) 81 | flipper-raw-rfid --version 82 | 83 | Arguments: 84 | RAW_FILE The raw rfid file from flipper (xyz.ask.raw or xyz.psk.raw) 85 | OUTPUT_FILE The converted file as csv (default: stdout) 86 | 87 | Options: 88 | -h --help Show this screen. 89 | --version Show version. 90 | -f --format=(pad|signal) Output format: "pad" (=Pulse And Duration) is the internal format of the Flipper Zero, 91 | each line represents a pulse and a duration value measured in samples, see 92 | "Pulse and duration format" below. 93 | In "signal" format the pulses are written out as a reconstructed signal with a "1" marking a 94 | sample with high value and "0" marking a sample with low value [default: pad] 95 | 96 | Pulse and duration (pad) format: 97 | 98 | column 0: pulse - (number of samples while output high) and 99 | column 1: duration - (number of samples till next signal) 100 | 101 | Diagram: 102 | 103 | ______________ __________ 104 | ______ __________ ....... 105 | 106 | ^ - pulse0 - ^ ^-pulse1-^ ....... 107 | ^ - duration0 -^^ - duration1 -^ ....... 108 | 109 | The csv file has the following format: 110 | 111 | pulse0, duration0 112 | pulse1, duration1 113 | .... 114 | ``` 115 | 116 | # Tutorial 117 | 118 | There is a short [RFID tutorial notebook](docs/rifl-tutorial-1.ipynb) to see if there is data in the recording and what to do with it. -------------------------------------------------------------------------------- /flipper_raw_rfid/bits.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utilities for working with bitstreams 3 | """ 4 | import re 5 | import numpy 6 | from flipper_raw_rfid.utils import batched, Peak 7 | import numpy.typing as npt 8 | 9 | 10 | def decode_lengths(pads: npt.NDArray[numpy.int64], peaks: list[Peak]) -> tuple[npt.NDArray[numpy.int8], int]: 11 | """ 12 | Loops through pulses and durations and matches them to peaks 13 | Checks for the length of the peak as a multiple of the first peak and adds as many 1/0 to the result 14 | 15 | :param pads: Pulse and duration values 16 | :param peaks: A list of peaks from find_peaks, the center frequencies should be more or less multiples of the first peak 17 | :return: The decoded bitstream 18 | """ 19 | result: list[int] = [] 20 | position = 0 21 | result_position = None 22 | first_length = peaks[0].center 23 | for high, duration in pads: 24 | low = duration - high 25 | 26 | high_peak = None 27 | low_peak = None 28 | 29 | for p in peaks: 30 | if high in p: 31 | high_peak = p 32 | if low in p: 33 | low_peak = p 34 | if high_peak and low_peak: 35 | break 36 | 37 | if not (high_peak and low_peak): 38 | if not high_peak: 39 | print(f'Found nothing for high {high}, restarting') 40 | if not low_peak: 41 | print(f'Found nothing for low {low}, restarting') 42 | result = [] 43 | result_position = position 44 | continue 45 | 46 | result.extend([1] * int(round(high_peak.center / first_length))) 47 | result.extend([0] * int(round(low_peak.center / first_length))) 48 | position += duration 49 | 50 | return numpy.array(result, dtype=numpy.int8), result_position 51 | 52 | 53 | def decode_manchester(manchester: npt.NDArray[numpy.int8], biphase: bool = True) -> npt.NDArray[numpy.int8]: 54 | """ 55 | Decode manchester encoded bitstream 56 | :param manchester: manchester encoded bitstream 57 | :param biphase: True for biphase, False for diphase 58 | :return: decoded bitstream 59 | """ 60 | if manchester[0] == manchester[1]: 61 | manchester = manchester[1:] 62 | 63 | result = [] 64 | for pair in batched(manchester, 2): 65 | if len(pair) < 2: 66 | break 67 | assert pair[0] != pair[1] 68 | result.append(pair[0 if biphase else 1]) 69 | 70 | return numpy.array(result, dtype=numpy.int8) 71 | 72 | 73 | def to_str(bits: npt.NDArray[numpy.int8]) -> str: 74 | """ 75 | Convert a bitstream to a string 76 | :param bits: 77 | :return: 78 | """ 79 | return ''.join(str(b) for b in bits) 80 | 81 | 82 | def find_pattern(bits: npt.NDArray[numpy.int8], pattern: str | re.Pattern[str]) -> npt.NDArray[numpy.int8] | None: 83 | bitstring = ''.join(str(b) for b in bits) 84 | m = re.search(pattern, bitstring) 85 | if not m: 86 | return None 87 | 88 | return bits[m.start(0):m.end(0)] 89 | 90 | 91 | def decode_em_4100(bits: npt.NDArray[numpy.int8]) -> npt.NDArray[numpy.int8]: 92 | """ 93 | Decode bitstream as EM 4100 94 | 95 | :param bits: bitstream 96 | :return: decoded nibbles 97 | 98 | 99 | EM 4100 100 | has a header of 9 '1's 101 | 111111111 102 | 103 | followed by 10 nibbles (4 bits each) of data plus one parity bit 104 | 1010 0 105 | 1000 1 106 | ... 107 | 108 | followed by 4 column parity bits and a final 0 109 | 110 | 0010 0 111 | 112 | """ 113 | 114 | em4100_bits = find_pattern(bits, r'1{9}.{54}0') 115 | 116 | datagrid = em4100_bits[9:].reshape((11, 5)) 117 | 118 | column_parity = datagrid[:, :4].sum(axis=0) 119 | assert numpy.all(column_parity % 2 == 0) 120 | row_parity = datagrid[:10].sum(axis=1) 121 | assert numpy.all(row_parity % 2 == 0) 122 | nibbles = (datagrid[:10, :4] * [8, 4, 2, 1]).sum(axis=1) 123 | 124 | return numpy.array(nibbles, dtype=numpy.int8) 125 | 126 | 127 | def longest_run(bits: npt.NDArray[numpy.int8], value: int = 1) -> int: 128 | """ 129 | Find the longest run of a value (1/0) in a bitstream 130 | 131 | :param bits: the bitstream 132 | :param value: the value to look for 133 | :return: index of the first bit of the longest run 134 | """ 135 | longest = 0 136 | current = 0 137 | longest_i = None 138 | 139 | for i, b in enumerate(bits): 140 | if b != value: 141 | current = 0 142 | continue 143 | current += 1 144 | if current > longest: 145 | longest = current 146 | longest_i = i 147 | 148 | return longest_i - longest + 1 149 | -------------------------------------------------------------------------------- /flipper_raw_rfid/cli.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3 2 | """flipper-raw-rfid 3 | 4 | Description: 5 | Reads a raw rfid file from flipper zero and plots or converts the signal 6 | 7 | Usage: 8 | flipper-raw-rfid convert [-f ] RAW_FILE [OUTPUT_FILE] 9 | flipper-raw-rfid plot RAW_FILE 10 | flipper-raw-rfid check RAW_FILE 11 | flipper-raw-rfid (-h | --help) 12 | flipper-raw-rfid --version 13 | 14 | Arguments: 15 | RAW_FILE The raw rfid file from flipper (xyz.ask.raw or xyz.psk.raw) 16 | OUTPUT_FILE The converted file as csv (default: stdout) 17 | 18 | Options: 19 | -h --help Show this screen. 20 | --version Show version. 21 | -f --format=(pad|signal) Output format: "pad" (=Pulse And Duration) is the internal format of the Flipper Zero, 22 | each line represents a pulse and a duration value measured in µs, see 23 | "Pulse and duration format" below. 24 | In "signal" format the pulses are written out as a reconstructed signal with a "1" marking a 25 | sample with high value and "0" marking a sample with low value [default: pad] 26 | 27 | Pulse and duration format: 28 | 29 | column 0: pulse - (µs while output high) and 30 | column 1: duration - (µs till next signal) 31 | 32 | Diagram: 33 | 34 | ______________ __________ 35 | ______ __________ ....... 36 | 37 | ^ - pulse0 - ^ ^-pulse1-^ ....... 38 | ^ - duration0 -^^ - duration1 -^ ....... 39 | 40 | The csv file has the following format: 41 | 42 | pulse0, duration0 43 | pulse1, duration1 44 | .... 45 | 46 | """ 47 | import contextlib 48 | import csv 49 | import os 50 | import sys 51 | from typing import Any, Generator, Iterable, IO 52 | 53 | from docopt import docopt, printable_usage 54 | from flipper_raw_rfid import __version__ 55 | from flipper_raw_rfid.rifl import Rifl, RiflError 56 | from flipper_raw_rfid.utils import pad_to_signal, autocorrelate 57 | from scipy import signal as scipy_signal 58 | 59 | 60 | @contextlib.contextmanager 61 | def smart_open(filename: str, mode: str = 'r', *args: Any, **kwargs: Any) -> Generator[IO[Any], None, None]: 62 | """ 63 | Open files and i/o streams transparently. 64 | """ 65 | fh: IO[Any] 66 | if filename == '-' or filename is None: 67 | if 'r' in mode: 68 | stream = sys.stdin 69 | else: 70 | stream = sys.stdout 71 | if 'b' in mode: 72 | fh = stream.buffer 73 | else: 74 | fh = stream 75 | close = False 76 | else: 77 | fh = open(filename, mode, *args, **kwargs) 78 | close = True 79 | 80 | try: 81 | yield fh 82 | fh.flush() 83 | except BrokenPipeError: 84 | # Python flushes standard streams on exit; redirect remaining output 85 | # to devnull to avoid another BrokenPipeError at shutdown 86 | devnull = os.open(os.devnull, os.O_WRONLY) 87 | os.dup2(devnull, fh.fileno()) 88 | finally: 89 | if close: 90 | try: 91 | fh.close() 92 | except AttributeError: 93 | pass 94 | 95 | 96 | def assert_in(value: Any, vset: Iterable[Any], name: str = 'Value') -> None: 97 | if value not in vset: 98 | raise ValueError(f'{name} must be one of: {"/".join(vset)} but was "{value}"') 99 | 100 | 101 | def convert(raw: str, output: str, format: str = 'pad') -> int: 102 | assert_in(format, {'signal', 'pad'}, 'Format') 103 | with smart_open(output, 'w', newline='') as csvfile: 104 | csvwriter = csv.writer(csvfile) 105 | rifl = Rifl.load(raw) 106 | pads = rifl.pulse_and_durations 107 | if format == 'pad': 108 | for pulse_and_duration in pads: 109 | csvwriter.writerow(pulse_and_duration) 110 | elif format == 'signal': 111 | signal = pad_to_signal(pads) 112 | csvwriter.writerows(signal.reshape((-1, 1))) 113 | 114 | return 0 115 | 116 | 117 | def plot(raw: str) -> int: 118 | try: 119 | import matplotlib.pyplot as plt 120 | except ModuleNotFoundError: 121 | raise RuntimeError('For the plot command you need matplotlib, install it with: pip install matplotlib') 122 | rifl = Rifl.load(raw) 123 | pads = rifl.pulse_and_durations 124 | signal = pad_to_signal(pads) 125 | fig, ax = plt.subplots() 126 | ax.set(xlabel='Samples', ylabel='Signal', title='Reconstructed Signal') 127 | ax.set_xlim(0, 20000) 128 | ax.plot(signal) 129 | plt.show() 130 | 131 | return 0 132 | 133 | 134 | def check(raw: str) -> int: 135 | rifl = Rifl.load(raw) 136 | pads = rifl.pulse_and_durations 137 | signal = pad_to_signal(pads) 138 | signal_autocorrelation = autocorrelate(signal) 139 | peaks, _ = scipy_signal.find_peaks(signal_autocorrelation, height=0.8, prominence=1) 140 | print(peaks) 141 | return 0 142 | 143 | 144 | def process(args: dict[str, Any]) -> int: 145 | try: 146 | if args['convert']: 147 | return convert(args['RAW_FILE'], args['OUTPUT_FILE'], args['--format']) 148 | elif args['plot']: 149 | return plot(args['RAW_FILE']) 150 | elif args['check']: 151 | return check(args['RAW_FILE']) 152 | else: 153 | print(printable_usage(__doc__)) 154 | return 1 155 | except RiflError as e: 156 | print(printable_usage(__doc__)) 157 | print(f'Error: {e}: {e.file.name if e.file.name else ""}') 158 | return 1 159 | except (ValueError, FileNotFoundError, IsADirectoryError, RuntimeError) as e: 160 | print(printable_usage(__doc__)) 161 | print(f'Error: {e}') 162 | return 1 163 | 164 | 165 | def main() -> int: 166 | args = docopt(__doc__, version=f'flipper-raw-rfid {__version__}') 167 | return process(args) 168 | -------------------------------------------------------------------------------- /flipper_raw_rfid/rifl.py: -------------------------------------------------------------------------------- 1 | """ 2 | Classes to load/save a raw rfid file from flipper (xyz.ask.raw or xyz.psk.raw) 3 | 4 | Usage: 5 | 6 | rifl = Rifl.load('path/to/raw.ask.raw') 7 | # get header values 8 | frequency = rifl.header.frequency 9 | duty_cycle = rifl.header.duty_cycle 10 | # get pulse and duration values 11 | pd = rifl.pulse_and_durations 12 | 13 | """ 14 | from __future__ import annotations 15 | 16 | from io import BytesIO 17 | from typing import BinaryIO, Generator, Any 18 | from struct import unpack, pack, error as struct_error 19 | from pathlib import Path 20 | from dataclasses import dataclass 21 | 22 | import numpy 23 | import numpy.typing as npt 24 | 25 | from flipper_raw_rfid.utils import batched 26 | 27 | LFRFID_RAW_FILE_MAGIC = 0x4C464952 # binary string "RIFL" 28 | LFRFID_RAW_FILE_VERSION = 1 29 | 30 | 31 | class RiflError(ValueError): 32 | def __init__(self, message: Any, file: BinaryIO = None): 33 | super().__init__(message) 34 | self.file = file 35 | 36 | 37 | @dataclass 38 | class RiflHeader: 39 | """ 40 | Rifl Header data structure 41 | """ 42 | version: int 43 | """ Version of the rifl file format: 1 supported """ 44 | frequency: float 45 | """ Frequency of the signal in Hz """ 46 | duty_cycle: float 47 | """ Duty cycle of the signal""" 48 | max_buffer_size: int 49 | """ Maximum buffer size in bytes""" 50 | 51 | @staticmethod 52 | def from_io(io: BinaryIO) -> RiflHeader: 53 | try: 54 | return RiflHeader.from_bytes(io.read(20)) 55 | except RiflError as e: 56 | e.file = io 57 | raise e 58 | 59 | @staticmethod 60 | def from_bytes(f: bytes) -> RiflHeader: 61 | try: 62 | magic, version, frequency, duty_cycle, max_buffer_size = unpack('IIffI', f) 63 | except struct_error: 64 | raise RiflError('Not a RIFL file') 65 | if magic != LFRFID_RAW_FILE_MAGIC: 66 | raise RiflError('Not a RIFL file') 67 | if version != LFRFID_RAW_FILE_VERSION: 68 | raise RiflError(f'Unsupported RIFL Version {version}') 69 | 70 | return RiflHeader(version, frequency, duty_cycle, max_buffer_size) 71 | 72 | def to_bytes(self) -> bytes: 73 | return pack('IIffI', LFRFID_RAW_FILE_MAGIC, self.version, self.frequency, self.duty_cycle, self.max_buffer_size) 74 | 75 | 76 | @dataclass 77 | class Rifl: 78 | """ 79 | A raw rfid file from flipper (xyz.ask.raw or xyz.psk.raw) 80 | 81 | """ 82 | header: RiflHeader 83 | """ The header of the file """ 84 | 85 | pulse_and_durations: npt.NDArray[numpy.int64] = None 86 | """ 87 | a nx2 numpy array with: 88 | column 0: pulse - (number of µs while output high) and 89 | column 1: duration - (number of µs till next signal) 90 | 91 | Diagram: 92 | 93 | _____________ _____ 94 | ______ _______________ ....... 95 | 96 | ^ - pulse - ^ 97 | 98 | ^ - duration -^ 99 | 100 | 101 | """ 102 | 103 | @staticmethod 104 | def load(path: Path | str) -> Rifl: 105 | path = Path(path) 106 | with path.open('rb') as f: 107 | return Rifl.from_io(f) 108 | 109 | @staticmethod 110 | def from_io(io: BinaryIO) -> Rifl: 111 | header = RiflHeader.from_io(io) 112 | pads = numpy.array(list(Rifl._pulse_and_durations(io, header.max_buffer_size)), dtype=numpy.int64) 113 | return Rifl(header, pads) 114 | 115 | def save(self, path: Path | str) -> None: 116 | path = Path(path) 117 | with path.open('wb') as f: 118 | self.to_io(f) 119 | 120 | def to_io(self, io: BinaryIO) -> None: 121 | 122 | def write(b: BytesIO) -> None: 123 | io.write(pack('I', b.getbuffer().nbytes)) 124 | io.write(b.getvalue()) 125 | 126 | def write_pair(b: BytesIO, pair: BytesIO) -> BytesIO: 127 | if b.getbuffer().nbytes + pair.getbuffer().nbytes > self.header.max_buffer_size: 128 | write(b) 129 | b = BytesIO() 130 | b.write(pair.getvalue()) 131 | return b 132 | 133 | io.write(self.header.to_bytes()) 134 | 135 | buffer = BytesIO() 136 | for pulse, duration in self.pulse_and_durations: 137 | pair_buffer = BytesIO() 138 | Rifl.write_varint(pair_buffer, pulse) 139 | Rifl.write_varint(pair_buffer, duration) 140 | buffer = write_pair(buffer, pair_buffer) 141 | 142 | write(buffer) 143 | 144 | @staticmethod 145 | def _buffers(io: BinaryIO, max_buffer_size: int) -> Generator[BinaryIO, None, None]: 146 | """ 147 | Read raw binary buffers and loop through them 148 | 149 | Each buffer holds varint (https://github.com/flipperdevices/flipperzero-firmware/blob/dev/lib/toolbox/varint.c#L13) encoded pairs 150 | """ 151 | while True: 152 | try: 153 | buffer_size, = unpack('I', io.read(4)) 154 | except struct_error: 155 | # No more bytes left, EOF 156 | break 157 | if buffer_size > max_buffer_size: 158 | raise RiflError(f'read pair: buffer size is too big {buffer_size} > {max_buffer_size}', io) 159 | buffer = io.read(buffer_size) 160 | if len(buffer) != buffer_size: 161 | raise RiflError(f'Tried to read {buffer_size} bytes got only {len(buffer)}', io) 162 | yield BytesIO(buffer) 163 | 164 | @staticmethod 165 | def _pulse_and_durations(io: BinaryIO, max_buffer_size: int) -> Generator[tuple[int, int], None, None]: 166 | """ 167 | loop through buffers and yield a pulse and duration tuple 168 | """ 169 | for buffer in Rifl._buffers(io, max_buffer_size): 170 | for pulse, duration in batched(Rifl.read_varint(buffer), 2): 171 | yield pulse, duration 172 | 173 | @staticmethod 174 | def read_varint(buffer: BinaryIO) -> Generator[int, None, None]: 175 | """ 176 | Read one varint from buffer 177 | 178 | Python implementation of https://github.com/flipperdevices/flipperzero-firmware/blob/dev/lib/toolbox/varint.c#L13 179 | 180 | """ 181 | res = 0 182 | i = 1 183 | while (vs := buffer.read(1)) != b'': 184 | v = vs[0] 185 | # the low 7 bits are the value 186 | res = res | (v & 0x7F) * i 187 | i = i << 7 188 | # yield when continue bit (bit 8) is not set 189 | if v & 0x80 == 0: 190 | yield res 191 | res = 0 192 | i = 1 193 | 194 | @staticmethod 195 | def write_varint(buffer: BinaryIO, value: int) -> int: 196 | """ 197 | Write one varint to buffer 198 | """ 199 | i = 1 200 | while value > 0x80: 201 | buffer.write(bytes([value & 0x7F | 0x80])) 202 | value >>= 7 203 | i += 1 204 | 205 | buffer.write(bytes([value & 0x7F])) 206 | return i 207 | 208 | 209 | __all__ = ['Rifl', 'RiflError'] 210 | -------------------------------------------------------------------------------- /flipper_raw_rfid/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass, field 4 | from itertools import islice, pairwise 5 | from typing import Iterable, Any, Generator, cast 6 | 7 | import numpy 8 | import numpy.typing as npt 9 | from scipy.ndimage import gaussian_filter1d 10 | from scipy.signal import find_peaks as scipy_signal_find_peaks 11 | from scipy.optimize import minimize_scalar 12 | from skimage.filters import threshold_otsu 13 | 14 | try: 15 | # python 3.12? 16 | from itertools import batched # type: ignore[attr-defined] 17 | except ImportError: 18 | def batched(iterable: Iterable[Any], n: int) -> Iterable[tuple[Any, ...]]: 19 | # batched('ABCDEFG', 3) --> ABC DEF G 20 | if n < 1: 21 | raise ValueError('n must be at least one') 22 | it = iter(iterable) 23 | while batch := tuple(islice(it, n)): 24 | yield batch 25 | 26 | 27 | def pad_to_signal(pulse_and_durations: npt.NDArray[numpy.int64]) -> npt.NDArray[numpy.int8]: 28 | """ 29 | Convert pulse and duration values from flipper to a binary (0/1) signal 30 | 31 | :param pulse_and_durations: The pulse and duration values 32 | :return: reconstructed signal 33 | """ 34 | length = pulse_and_durations[:, 1].sum() 35 | signal = numpy.zeros(length, dtype=numpy.int8) 36 | 37 | position = 0 38 | for pulse, duration in pulse_and_durations: 39 | # Fill signal with ones for pulse 40 | signal[position:position + pulse] = 1 41 | # Fill signal with zeros for the rest of the duration (not needed, because signal default to zero, just for clarity) 42 | # signal[position+pulse:position+duration] = 0 43 | position += duration 44 | 45 | return signal 46 | 47 | 48 | def signal_to_pad(signal: npt.NDArray[numpy.int8]) -> npt.NDArray[numpy.int64]: 49 | """ 50 | Convert a binary (0/1) signal to pulse and duration values like used in flipper 51 | 52 | :param signal: The signal 53 | :return: Pulse and duration values 54 | """ 55 | def it(s: npt.NDArray[numpy.int8]) -> Generator[tuple[int, int], None, None]: 56 | position = -1 57 | changes = numpy.where(s[:-1] != s[1:])[0] 58 | for p in batched(changes, 2): 59 | if len(p) == 2: 60 | yield p[0] - position, p[1] - position 61 | position = p[1] 62 | 63 | if len(p) == 1: 64 | yield p[0] - position, len(s) - position - 1 65 | 66 | return numpy.array(list(it(signal))) 67 | 68 | 69 | def find_first_transition_index(signal: npt.NDArray[numpy.int8], to: int = 1) -> int: 70 | """ 71 | Find the first index in the signal where it transitions to `to 72 | 73 | :param signal: the signal 74 | :param to: the value to transition to 75 | :return: index of the first transition 76 | """ 77 | # Array of the first 2 indices where there is a transition in any direction 78 | changes = (signal[:-1] != signal[1:]).nonzero()[0][:2] + 1 79 | # One of them is the change to the requested `to` value 80 | needed_index = (signal[changes] == to).nonzero()[0][0] 81 | 82 | change_index = changes[needed_index] 83 | # Some clarity after too much numpy magic ;) 84 | assert signal[change_index] == to 85 | assert signal[change_index - 1] != to 86 | 87 | return cast(int, change_index) 88 | 89 | 90 | @dataclass 91 | class Peak: 92 | """ 93 | A peak in a distribution described by left, center and right index 94 | """ 95 | left: int = field(compare=False) 96 | center: int = field(compare=False) 97 | right: int = field(compare=False) 98 | height: float = field(default=0.0, repr=False) 99 | 100 | def merge(self, other: Peak) -> Peak: 101 | """ 102 | Merge this peak with another peak 103 | :param other: Peak to merge with 104 | :return: merged peak 105 | """ 106 | return Peak( 107 | min(self.left, other.left), 108 | (self.center + other.center) // 2, 109 | max(self.right, other.right), 110 | max(self.height, other.height) 111 | ) 112 | 113 | def slice(self, distribution: npt.NDArray[Any]) -> npt.NDArray[Any]: 114 | """ 115 | Slice the distribution with the peak 116 | 117 | :param distribution: 118 | :return: 119 | """ 120 | return distribution[self.left:self.right] 121 | 122 | def fit(self, distribution: npt.NDArray[Any], quantile: float = 1.0) -> Peak: 123 | """ 124 | Fit the distribution to the peak 125 | :param distribution: 126 | :param quantile: 127 | :return: 128 | """ 129 | my_excerpt = distribution[self.left:self.right] 130 | if quantile < 1.0: 131 | to_capture = numpy.sum(my_excerpt) * quantile 132 | 133 | def objective(thr: float) -> float: 134 | # 1.0 for capturing enough and a little nudge to find bigger thresholds 135 | return cast(float, 1.0 * (to_capture > numpy.sum(my_excerpt[my_excerpt > thr])) - thr * 0.0001) 136 | 137 | res = minimize_scalar(objective, (0, my_excerpt.max())) 138 | threshold = int(res.x) 139 | else: 140 | threshold = 0 141 | 142 | first, *_, last = (my_excerpt > threshold).nonzero()[0] 143 | 144 | return Peak( 145 | self.left + first - 1, 146 | self.left + (first + last) // 2, 147 | self.left + last + 1, 148 | my_excerpt[first:last].max() 149 | ) 150 | 151 | def __contains__(self, v: float | int) -> bool: 152 | """ 153 | Check if a value is inside the peak 154 | :param v: value to check 155 | :return: 156 | """ 157 | return self.left <= v <= self.right 158 | 159 | 160 | def histogram(values: npt.NDArray[Any], min_length: int = None) -> npt.NDArray[numpy.int32]: 161 | """ 162 | Calculate a stupid histogram of a distribution, each value is it's own "bin" 163 | 164 | :param values: The values to count 165 | :param min_length: Optional minimum length of histogram 166 | :return: histogram 167 | """ 168 | length = max(values.max() + 20, min_length or 0) 169 | hist = numpy.zeros(length, dtype=numpy.int32) 170 | for v in values: 171 | hist[v] += 1 172 | return hist 173 | 174 | 175 | def find_peaks(distribution: npt.NDArray[Any], min_height: float = None, separate_peaks: bool = True) -> list[Peak]: 176 | """ 177 | Simple wrapper around scipy.signal.find_peaks auto-tuned for histograms and sorted Peak[] as return value 178 | 179 | :param distribution: The signal to analyze 180 | :param min_height: optional minimum height for peaks, if None, mean of distribution is used 181 | :param separate_peaks: if True, separate peaks that have overlap via otsu thresholding 182 | :return: array of Peak 183 | """ 184 | if min_height is None: 185 | min_height = numpy.mean(distribution) 186 | peaks_center, pd = scipy_signal_find_peaks(distribution, height=min_height, prominence=min_height) 187 | peaks = [Peak(l, c, r, h) for c, l, r, h in zip(peaks_center, pd['left_bases'], pd['right_bases'], pd['peak_heights'])] 188 | 189 | # Separate peaks that have overlap 190 | if separate_peaks: 191 | for left, right in pairwise(sorted(peaks, key=lambda p: p.center)): 192 | if left.right > right.left: 193 | left.right = right.left = left.left + threshold_otsu(hist=distribution[left.left:right.right]) 194 | 195 | return sorted(peaks, key=lambda p: p.height, reverse=True) 196 | 197 | 198 | def autocorrelate(x: npt.NDArray[Any]) -> npt.NDArray[numpy.float32]: 199 | """ 200 | Calculate fast statistical autocorrelation 201 | 202 | :param x: signal 203 | :return: autocorrelation of signal 204 | 205 | taken from: 206 | https://stackoverflow.com/a/51168178 207 | autocorr3 / fft, pad 0s, non partial 208 | """ 209 | n = len(x) 210 | # pad 0s to 2n-1 211 | ext_size = 2 * n - 1 212 | # nearest power of 2 213 | fsize = 2 ** numpy.ceil(numpy.log2(ext_size)).astype('int') 214 | 215 | xp = x - numpy.mean(x) 216 | var = numpy.var(x) 217 | 218 | # do fft and ifft 219 | cf = numpy.fft.fft(xp, fsize) 220 | sf = cf.conjugate() * cf 221 | corr = numpy.fft.ifft(sf).real 222 | corr = corr / var / n 223 | 224 | return (corr[:len(corr) // 2]).astype(numpy.float32) 225 | 226 | 227 | def smooth(signal: npt.NDArray[numpy.int8], sigma: float = 10) -> npt.NDArray[numpy.float32]: 228 | """ 229 | Apply gaussian filtering to signal 230 | 231 | :param signal: The input signal 232 | :param sigma: sigma for gaussian filter, how much smoothing 233 | :return: smoothed signal 234 | """ 235 | return cast(npt.NDArray[numpy.float32], gaussian_filter1d(numpy.float32(signal), sigma=sigma, mode='nearest')) 236 | 237 | 238 | def binarize(signal: npt.NDArray[numpy.float32], threshold: float = 0.5) -> npt.NDArray[numpy.int8]: 239 | """ 240 | Binarize (0/1) signal with threshold 241 | 242 | :param signal: The input signal 243 | :param threshold: threshold for binarization 244 | :return: binarized signal 245 | """ 246 | return (signal > threshold).astype(numpy.int8) 247 | 248 | 249 | __all__ = ['signal_to_pad', 'pad_to_signal', 'smooth', 'binarize', 'autocorrelate', 'batched', 'Peak', 'find_first_transition_index', 'find_peaks', 'histogram'] 250 | --------------------------------------------------------------------------------