├── tests ├── __init__.py ├── files │ ├── empty │ │ ├── new │ │ ├── old │ │ ├── patch │ │ ├── crle.patch │ │ ├── none.patch │ │ ├── nonempty.bin │ │ ├── heatshrink.patch │ │ ├── in-place.patch │ │ └── nonempty.patch │ ├── foo │ │ ├── empty.patch │ │ ├── one-byte.patch │ │ ├── missing-in-place-memory-size.patch │ │ ├── new │ │ ├── old │ │ ├── patch │ │ ├── crle.patch │ │ ├── lz4.patch │ │ ├── none.patch │ │ ├── zstd.patch │ │ ├── bsdiff.patch │ │ ├── short.patch │ │ ├── backwards.patch │ │ ├── no-delta.patch │ │ ├── hdiffpatch.patch │ │ ├── heatshrink.patch │ │ ├── short-none.patch │ │ ├── arm-cortex-m4.patch │ │ ├── bad-lzma-end.patch │ │ ├── bad-patch-type.patch │ │ ├── short-to-size.patch │ │ ├── bad-compression.patch │ │ ├── hdiffpatch-none.patch │ │ ├── heatshrink-10-5.patch │ │ ├── in-place-3000-500.mem │ │ ├── in-place-3k-1.5k.patch │ │ ├── diff-data-too-long.patch │ │ ├── extra-data-too-long.patch │ │ ├── in-place-3000-1500.patch │ │ ├── in-place-3000-500.patch │ │ ├── in-place-minimum-size.patch │ │ ├── hdiffpatch-match-score-0.patch │ │ ├── in-place-3000-1500-1500.patch │ │ ├── in-place-3000-500-crle.patch │ │ ├── in-place-6000-1000-crle.patch │ │ ├── in-place-many-segments.patch │ │ ├── match-blocks-sequential.patch │ │ ├── missing-in-place-from-size.patch │ │ ├── match-blocks-sequential-none.patch │ │ ├── missing-in-place-segment-size.patch │ │ ├── missing-in-place-shift-size.patch │ │ └── hdiffpatch-match-block-size-64.patch │ ├── fuzzer_1.old │ ├── fuzzer_1.new │ ├── fuzzer_1.patch │ ├── shell │ │ ├── new │ │ ├── old │ │ ├── patch │ │ ├── bz2.patch │ │ ├── crle.patch │ │ ├── lz4.patch │ │ ├── zstd.patch │ │ ├── arm-cortex-m4.patch │ │ ├── arm-cortex-m4-bz2.patch │ │ └── arm-cortex-m4-crle.patch │ ├── random │ │ ├── to.bin │ │ ├── from.bin │ │ ├── patch-bsdiff.bin │ │ ├── match-blocks-hdiffpatch.patch │ │ └── match-blocks-sequential-none.patch │ ├── shell-pi-3 │ │ ├── 1.bin │ │ ├── 2.bin │ │ ├── 1--2.patch │ │ ├── 1--2-aarch64.patch │ │ └── 1--2-aarch64-data-sections.patch │ ├── python3 │ │ └── aarch64 │ │ │ ├── README.rst │ │ │ ├── 3.6.6-1--3.7.2-3.patch │ │ │ ├── 3.7.2-3--3.7.3-1.patch │ │ │ ├── 3.6.6-1 │ │ │ └── libpython3.6m.so.1.0 │ │ │ ├── 3.7.2-3 │ │ │ └── libpython3.7m.so.1.0 │ │ │ ├── 3.7.3-1 │ │ │ └── libpython3.7m.so.1.0 │ │ │ ├── 3.6.6-1--3.7.2-3-aarch64.patch │ │ │ └── 3.7.2-3--3.7.3-1-aarch64.patch │ ├── sais-READ-ME.patch │ ├── synthesizer │ │ ├── 1.bin │ │ ├── 2.bin │ │ ├── 3.bin │ │ ├── 1--2.patch │ │ ├── 1--3.patch │ │ ├── 1--2-arm-cortex-m4.patch │ │ └── 1--3-arm-cortex-m4.patch │ ├── bsdiff-READ-ME.patch │ ├── errors.cpython-36.bin │ ├── programmer │ │ ├── 0.8.0.bin │ │ ├── 0.9.0.bin │ │ ├── 0.8.0--0.9.0.patch │ │ └── 0.8.0--0.9.0-arm-cortex-m4.patch │ ├── pybv11 │ │ ├── v1.10 │ │ │ ├── firmware.elf │ │ │ ├── firmware0.bin │ │ │ └── firmware1.bin │ │ ├── 1f5d945af │ │ │ ├── firmware.elf │ │ │ ├── firmware0.bin │ │ │ └── firmware1.bin │ │ ├── 1f5d945af-dirty │ │ │ ├── firmware.elf │ │ │ ├── firmware0.bin │ │ │ └── firmware1.bin │ │ ├── v1.10--1f5d945af-dirty.patch │ │ ├── 1f5d945af--1f5d945af-dirty.patch │ │ ├── v1.10--1f5d945af-dirty-arm-cortex-m4.patch │ │ ├── 1f5d945af--1f5d945af-dirty-arm-cortex-m4.patch │ │ ├── 1f5d945af--1f5d945af-dirty-arm-cortex-m4-data-sections.patch │ │ └── 1f5d945af--1f5d945af-dirty-arm-cortex-m4-elf-data-sections.patch │ ├── micropython │ │ ├── esp8266-20180511-v1.9.4.bin │ │ ├── esp8266-20180511-v1.9.4.elf │ │ ├── esp8266-20190125-v1.10.bin │ │ ├── esp8266-20190125-v1.10.elf │ │ ├── esp8266-20180511-v1.9.4--20190125-v1.10.patch │ │ ├── esp8266-20180511-v1.9.4--20190125-v1.10-lz4.patch │ │ ├── esp8266-20180511-v1.9.4--20190125-v1.10-bsdiff.patch │ │ ├── esp8266-20180511-v1.9.4--20190125-v1.10-crle.patch │ │ ├── esp8266-20180511-v1.9.4--20190125-v1.10-none.patch │ │ ├── esp8266-20180511-v1.9.4--20190125-v1.10-zstd.patch │ │ ├── esp8266-20180511-v1.9.4--20190125-v1.10-in-place.patch │ │ ├── esp8266-20180511-v1.9.4--20190125-v1.10-heatshrink.patch │ │ ├── esp8266-20180511-v1.9.4--20190125-v1.10-xtensa-lx106.patch │ │ └── esp8266-20180511-v1.9.4--20190125-v1.10-xtensa-lx106-data-sections.patch │ ├── 3f5531ba56182a807a5c358f04678b3b026d3a.bin │ ├── b2db59ab76ca36f67e61f720857021df8a660b.bin │ ├── d027a1e1f752f15b6a13d9f9d775f3914c83f7.bin │ ├── eb9ed88e9975028c4694e070cfaece2498e92d.bin │ ├── 3f5531ba56182a807a5c358f04678b3b026d3a-READ-ME.patch │ ├── b2db59ab76ca36f67e61f720857021df8a660b-READ-ME.patch │ ├── d027a1e1f752f15b6a13d9f9d775f3914c83f7-READ-ME.patch │ ├── eb9ed88e9975028c4694e070cfaece2498e92d-READ-ME.patch │ ├── READ-ME.rst │ └── bsdiff.py ├── test_data_format.py ├── test_suffix_array.py ├── fuzzer.c ├── test_bsdiff.py ├── test_none.py ├── benchmark.sh └── test_crle.py ├── detools ├── compression │ ├── __init__.py │ ├── lz4.py │ ├── none.py │ ├── zstd.py │ ├── heatshrink.py │ └── crle.py ├── version.py ├── errors.py ├── __main__.py ├── libdivsufsort │ ├── divsufsort.c │ ├── divsufsort64.c │ ├── config.h │ ├── divsufsort_private.h │ ├── divsufsort.h │ └── divsufsort64.h ├── sais │ └── sais.h ├── data_format │ ├── __init__.py │ └── elf.py ├── suffix_array.c ├── common.py └── info.py ├── .github ├── FUNDING.yml └── workflows │ └── test.yml ├── c ├── examples │ ├── in_place │ │ ├── .gitignore │ │ ├── Makefile │ │ └── main.c │ └── dump_restore │ │ ├── .gitignore │ │ ├── Makefile │ │ └── README.rst ├── heatshrink │ ├── README.rst │ ├── heatshrink_common.h │ ├── heatshrink_config.h │ └── heatshrink_decoder.h ├── Makefile ├── tst │ ├── Makefile │ ├── utils.h │ ├── fuzzer_corrupt_patch.c │ ├── test_command_line.c │ ├── test.mk │ ├── utils.c │ ├── test_fuzzer.c │ └── test_dump_restore.c ├── main.c └── README.rst ├── .gitattributes ├── .gitmodules ├── requirements.txt ├── .readthedocs.yaml ├── MANIFEST.in ├── LICENSE ├── .gitignore ├── setup.py ├── Makefile └── docs ├── Makefile ├── make.bat └── index.rst /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/files/empty/new: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/files/empty/old: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/files/empty/patch: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /tests/files/foo/empty.patch: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/files/fuzzer_1.old: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /detools/compression/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/files/empty/crle.patch: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /tests/files/empty/none.patch: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/files/empty/nonempty.bin: -------------------------------------------------------------------------------- 1 | 1 -------------------------------------------------------------------------------- /tests/files/foo/one-byte.patch: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /tests/files/empty/heatshrink.patch: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /tests/files/fuzzer_1.new: -------------------------------------------------------------------------------- 1 | andle_segv -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: eerimoq 2 | -------------------------------------------------------------------------------- /c/examples/in_place/.gitignore: -------------------------------------------------------------------------------- 1 | in-place* -------------------------------------------------------------------------------- /detools/version.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.53.0' 2 | -------------------------------------------------------------------------------- /tests/files/foo/missing-in-place-memory-size.patch: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /tests/files/fuzzer_1.patch: -------------------------------------------------------------------------------- 1 |  2 | 3 | andle_segv -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | tests/files/*.[hc] linguist-vendored 2 | -------------------------------------------------------------------------------- /detools/errors.py: -------------------------------------------------------------------------------- 1 | class Error(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /detools/__main__.py: -------------------------------------------------------------------------------- 1 | from . import _main as main 2 | 3 | main() 4 | -------------------------------------------------------------------------------- /c/examples/dump_restore/.gitignore: -------------------------------------------------------------------------------- 1 | dump-restore 2 | foo.new 3 | state.bin -------------------------------------------------------------------------------- /tests/files/foo/new: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/new -------------------------------------------------------------------------------- /tests/files/foo/old: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/old -------------------------------------------------------------------------------- /tests/files/foo/patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/patch -------------------------------------------------------------------------------- /tests/files/shell/new: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell/new -------------------------------------------------------------------------------- /tests/files/shell/old: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell/old -------------------------------------------------------------------------------- /tests/files/shell/patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell/patch -------------------------------------------------------------------------------- /tests/files/foo/crle.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/crle.patch -------------------------------------------------------------------------------- /tests/files/foo/lz4.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/lz4.patch -------------------------------------------------------------------------------- /tests/files/foo/none.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/none.patch -------------------------------------------------------------------------------- /tests/files/foo/zstd.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/zstd.patch -------------------------------------------------------------------------------- /tests/files/random/to.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/random/to.bin -------------------------------------------------------------------------------- /tests/files/foo/bsdiff.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/bsdiff.patch -------------------------------------------------------------------------------- /tests/files/foo/short.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/short.patch -------------------------------------------------------------------------------- /tests/files/random/from.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/random/from.bin -------------------------------------------------------------------------------- /tests/files/shell-pi-3/1.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell-pi-3/1.bin -------------------------------------------------------------------------------- /tests/files/shell-pi-3/2.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell-pi-3/2.bin -------------------------------------------------------------------------------- /tests/files/shell/bz2.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell/bz2.patch -------------------------------------------------------------------------------- /tests/files/shell/crle.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell/crle.patch -------------------------------------------------------------------------------- /tests/files/shell/lz4.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell/lz4.patch -------------------------------------------------------------------------------- /tests/files/shell/zstd.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell/zstd.patch -------------------------------------------------------------------------------- /tests/files/foo/backwards.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/backwards.patch -------------------------------------------------------------------------------- /tests/files/foo/no-delta.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/no-delta.patch -------------------------------------------------------------------------------- /tests/files/python3/aarch64/README.rst: -------------------------------------------------------------------------------- 1 | Downloaded from: http://tardis.tiny-vps.com/aarm/packages/p/python/ 2 | -------------------------------------------------------------------------------- /tests/files/sais-READ-ME.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/sais-READ-ME.patch -------------------------------------------------------------------------------- /tests/files/synthesizer/1.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/synthesizer/1.bin -------------------------------------------------------------------------------- /tests/files/synthesizer/2.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/synthesizer/2.bin -------------------------------------------------------------------------------- /tests/files/synthesizer/3.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/synthesizer/3.bin -------------------------------------------------------------------------------- /tests/files/bsdiff-READ-ME.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/bsdiff-READ-ME.patch -------------------------------------------------------------------------------- /tests/files/empty/in-place.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/empty/in-place.patch -------------------------------------------------------------------------------- /tests/files/empty/nonempty.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/empty/nonempty.patch -------------------------------------------------------------------------------- /tests/files/errors.cpython-36.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/errors.cpython-36.bin -------------------------------------------------------------------------------- /tests/files/foo/hdiffpatch.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/hdiffpatch.patch -------------------------------------------------------------------------------- /tests/files/foo/heatshrink.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/heatshrink.patch -------------------------------------------------------------------------------- /tests/files/foo/short-none.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/short-none.patch -------------------------------------------------------------------------------- /tests/files/programmer/0.8.0.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/programmer/0.8.0.bin -------------------------------------------------------------------------------- /tests/files/programmer/0.9.0.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/programmer/0.9.0.bin -------------------------------------------------------------------------------- /tests/files/shell-pi-3/1--2.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell-pi-3/1--2.patch -------------------------------------------------------------------------------- /tests/files/foo/arm-cortex-m4.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/arm-cortex-m4.patch -------------------------------------------------------------------------------- /tests/files/foo/bad-lzma-end.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/bad-lzma-end.patch -------------------------------------------------------------------------------- /tests/files/foo/bad-patch-type.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/bad-patch-type.patch -------------------------------------------------------------------------------- /tests/files/foo/short-to-size.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/short-to-size.patch -------------------------------------------------------------------------------- /tests/files/random/patch-bsdiff.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/random/patch-bsdiff.bin -------------------------------------------------------------------------------- /tests/files/synthesizer/1--2.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/synthesizer/1--2.patch -------------------------------------------------------------------------------- /tests/files/synthesizer/1--3.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/synthesizer/1--3.patch -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "detools/HDiffPatch"] 2 | path = detools/HDiffPatch 3 | url = https://github.com/eerimoq/HDiffPatch 4 | -------------------------------------------------------------------------------- /tests/files/foo/bad-compression.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/bad-compression.patch -------------------------------------------------------------------------------- /tests/files/foo/hdiffpatch-none.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/hdiffpatch-none.patch -------------------------------------------------------------------------------- /tests/files/foo/heatshrink-10-5.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/heatshrink-10-5.patch -------------------------------------------------------------------------------- /tests/files/foo/in-place-3000-500.mem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/in-place-3000-500.mem -------------------------------------------------------------------------------- /tests/files/foo/in-place-3k-1.5k.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/in-place-3k-1.5k.patch -------------------------------------------------------------------------------- /tests/files/pybv11/v1.10/firmware.elf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/v1.10/firmware.elf -------------------------------------------------------------------------------- /tests/files/pybv11/v1.10/firmware0.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/v1.10/firmware0.bin -------------------------------------------------------------------------------- /tests/files/pybv11/v1.10/firmware1.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/v1.10/firmware1.bin -------------------------------------------------------------------------------- /tests/files/shell/arm-cortex-m4.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell/arm-cortex-m4.patch -------------------------------------------------------------------------------- /tests/files/foo/diff-data-too-long.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/diff-data-too-long.patch -------------------------------------------------------------------------------- /tests/files/foo/extra-data-too-long.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/extra-data-too-long.patch -------------------------------------------------------------------------------- /tests/files/foo/in-place-3000-1500.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/in-place-3000-1500.patch -------------------------------------------------------------------------------- /tests/files/foo/in-place-3000-500.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/in-place-3000-500.patch -------------------------------------------------------------------------------- /tests/files/programmer/0.8.0--0.9.0.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/programmer/0.8.0--0.9.0.patch -------------------------------------------------------------------------------- /tests/files/pybv11/1f5d945af/firmware.elf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/1f5d945af/firmware.elf -------------------------------------------------------------------------------- /tests/files/shell-pi-3/1--2-aarch64.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell-pi-3/1--2-aarch64.patch -------------------------------------------------------------------------------- /tests/files/shell/arm-cortex-m4-bz2.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell/arm-cortex-m4-bz2.patch -------------------------------------------------------------------------------- /tests/files/foo/in-place-minimum-size.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/in-place-minimum-size.patch -------------------------------------------------------------------------------- /tests/files/pybv11/1f5d945af/firmware0.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/1f5d945af/firmware0.bin -------------------------------------------------------------------------------- /tests/files/pybv11/1f5d945af/firmware1.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/1f5d945af/firmware1.bin -------------------------------------------------------------------------------- /tests/files/shell/arm-cortex-m4-crle.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell/arm-cortex-m4-crle.patch -------------------------------------------------------------------------------- /tests/files/foo/hdiffpatch-match-score-0.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/hdiffpatch-match-score-0.patch -------------------------------------------------------------------------------- /tests/files/foo/in-place-3000-1500-1500.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/in-place-3000-1500-1500.patch -------------------------------------------------------------------------------- /tests/files/foo/in-place-3000-500-crle.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/in-place-3000-500-crle.patch -------------------------------------------------------------------------------- /tests/files/foo/in-place-6000-1000-crle.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/in-place-6000-1000-crle.patch -------------------------------------------------------------------------------- /tests/files/foo/in-place-many-segments.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/in-place-many-segments.patch -------------------------------------------------------------------------------- /tests/files/foo/match-blocks-sequential.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/match-blocks-sequential.patch -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | humanfriendly 2 | bitstruct 3 | pyelftools 4 | zstandard 5 | lz4 6 | heatshrink2 7 | nala 8 | setuptools 9 | pytest 10 | -------------------------------------------------------------------------------- /tests/files/foo/missing-in-place-from-size.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/missing-in-place-from-size.patch -------------------------------------------------------------------------------- /tests/files/pybv11/1f5d945af-dirty/firmware.elf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/1f5d945af-dirty/firmware.elf -------------------------------------------------------------------------------- /tests/files/pybv11/1f5d945af-dirty/firmware0.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/1f5d945af-dirty/firmware0.bin -------------------------------------------------------------------------------- /tests/files/pybv11/1f5d945af-dirty/firmware1.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/1f5d945af-dirty/firmware1.bin -------------------------------------------------------------------------------- /tests/files/pybv11/v1.10--1f5d945af-dirty.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/v1.10--1f5d945af-dirty.patch -------------------------------------------------------------------------------- /tests/files/random/match-blocks-hdiffpatch.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/random/match-blocks-hdiffpatch.patch -------------------------------------------------------------------------------- /tests/files/synthesizer/1--2-arm-cortex-m4.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/synthesizer/1--2-arm-cortex-m4.patch -------------------------------------------------------------------------------- /tests/files/synthesizer/1--3-arm-cortex-m4.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/synthesizer/1--3-arm-cortex-m4.patch -------------------------------------------------------------------------------- /tests/files/foo/match-blocks-sequential-none.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/match-blocks-sequential-none.patch -------------------------------------------------------------------------------- /tests/files/foo/missing-in-place-segment-size.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/missing-in-place-segment-size.patch -------------------------------------------------------------------------------- /tests/files/foo/missing-in-place-shift-size.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/missing-in-place-shift-size.patch -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4.bin -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4.elf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4.elf -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20190125-v1.10.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20190125-v1.10.bin -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20190125-v1.10.elf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20190125-v1.10.elf -------------------------------------------------------------------------------- /tests/files/pybv11/1f5d945af--1f5d945af-dirty.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/1f5d945af--1f5d945af-dirty.patch -------------------------------------------------------------------------------- /tests/files/python3/aarch64/3.6.6-1--3.7.2-3.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/python3/aarch64/3.6.6-1--3.7.2-3.patch -------------------------------------------------------------------------------- /tests/files/python3/aarch64/3.7.2-3--3.7.3-1.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/python3/aarch64/3.7.2-3--3.7.3-1.patch -------------------------------------------------------------------------------- /tests/files/foo/hdiffpatch-match-block-size-64.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/foo/hdiffpatch-match-block-size-64.patch -------------------------------------------------------------------------------- /tests/files/random/match-blocks-sequential-none.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/random/match-blocks-sequential-none.patch -------------------------------------------------------------------------------- /tests/files/3f5531ba56182a807a5c358f04678b3b026d3a.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/3f5531ba56182a807a5c358f04678b3b026d3a.bin -------------------------------------------------------------------------------- /tests/files/b2db59ab76ca36f67e61f720857021df8a660b.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/b2db59ab76ca36f67e61f720857021df8a660b.bin -------------------------------------------------------------------------------- /tests/files/d027a1e1f752f15b6a13d9f9d775f3914c83f7.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/d027a1e1f752f15b6a13d9f9d775f3914c83f7.bin -------------------------------------------------------------------------------- /tests/files/eb9ed88e9975028c4694e070cfaece2498e92d.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/eb9ed88e9975028c4694e070cfaece2498e92d.bin -------------------------------------------------------------------------------- /tests/files/programmer/0.8.0--0.9.0-arm-cortex-m4.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/programmer/0.8.0--0.9.0-arm-cortex-m4.patch -------------------------------------------------------------------------------- /tests/files/python3/aarch64/3.6.6-1/libpython3.6m.so.1.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/python3/aarch64/3.6.6-1/libpython3.6m.so.1.0 -------------------------------------------------------------------------------- /tests/files/python3/aarch64/3.7.2-3/libpython3.7m.so.1.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/python3/aarch64/3.7.2-3/libpython3.7m.so.1.0 -------------------------------------------------------------------------------- /tests/files/python3/aarch64/3.7.3-1/libpython3.7m.so.1.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/python3/aarch64/3.7.3-1/libpython3.7m.so.1.0 -------------------------------------------------------------------------------- /tests/files/shell-pi-3/1--2-aarch64-data-sections.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/shell-pi-3/1--2-aarch64-data-sections.patch -------------------------------------------------------------------------------- /tests/files/python3/aarch64/3.6.6-1--3.7.2-3-aarch64.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/python3/aarch64/3.6.6-1--3.7.2-3-aarch64.patch -------------------------------------------------------------------------------- /tests/files/python3/aarch64/3.7.2-3--3.7.3-1-aarch64.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/python3/aarch64/3.7.2-3--3.7.3-1-aarch64.patch -------------------------------------------------------------------------------- /tests/files/pybv11/v1.10--1f5d945af-dirty-arm-cortex-m4.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/v1.10--1f5d945af-dirty-arm-cortex-m4.patch -------------------------------------------------------------------------------- /c/heatshrink/README.rst: -------------------------------------------------------------------------------- 1 | The files in this folder are copied from the heatshrink project, 2 | https://github.com/atomicobject/heatshrink. 3 | 4 | Licensed under the ISC License. 5 | -------------------------------------------------------------------------------- /tests/files/3f5531ba56182a807a5c358f04678b3b026d3a-READ-ME.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/3f5531ba56182a807a5c358f04678b3b026d3a-READ-ME.patch -------------------------------------------------------------------------------- /tests/files/b2db59ab76ca36f67e61f720857021df8a660b-READ-ME.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/b2db59ab76ca36f67e61f720857021df8a660b-READ-ME.patch -------------------------------------------------------------------------------- /tests/files/d027a1e1f752f15b6a13d9f9d775f3914c83f7-READ-ME.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/d027a1e1f752f15b6a13d9f9d775f3914c83f7-READ-ME.patch -------------------------------------------------------------------------------- /tests/files/eb9ed88e9975028c4694e070cfaece2498e92d-READ-ME.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/eb9ed88e9975028c4694e070cfaece2498e92d-READ-ME.patch -------------------------------------------------------------------------------- /tests/files/pybv11/1f5d945af--1f5d945af-dirty-arm-cortex-m4.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/1f5d945af--1f5d945af-dirty-arm-cortex-m4.patch -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10.patch -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-lz4.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-lz4.patch -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-bsdiff.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-bsdiff.patch -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-crle.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-crle.patch -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-none.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-none.patch -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-zstd.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-zstd.patch -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-in-place.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-in-place.patch -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-heatshrink.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-heatshrink.patch -------------------------------------------------------------------------------- /tests/files/pybv11/1f5d945af--1f5d945af-dirty-arm-cortex-m4-data-sections.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/1f5d945af--1f5d945af-dirty-arm-cortex-m4-data-sections.patch -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-xtensa-lx106.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-xtensa-lx106.patch -------------------------------------------------------------------------------- /tests/files/pybv11/1f5d945af--1f5d945af-dirty-arm-cortex-m4-elf-data-sections.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/pybv11/1f5d945af--1f5d945af-dirty-arm-cortex-m4-elf-data-sections.patch -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-22.04 5 | tools: 6 | python: "3.11" 7 | 8 | sphinx: 9 | configuration: docs/conf.py 10 | 11 | python: 12 | install: 13 | - requirements: requirements.txt 14 | -------------------------------------------------------------------------------- /tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-xtensa-lx106-data-sections.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eerimoq/detools/HEAD/tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-xtensa-lx106-data-sections.patch -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include Makefile 3 | recursive-include detools/sais *.h *.c 4 | recursive-include detools/libdivsufsort *.h *.c 5 | recursive-include detools/HDiffPatch *.h *.c *.cpp 6 | recursive-include tests *.py *.old *.new *.patch *.bin *.rst *.c *.1.0 old new patch *.elf 7 | -------------------------------------------------------------------------------- /detools/libdivsufsort/divsufsort.c: -------------------------------------------------------------------------------- 1 | #ifdef BUILD_DIVSUFSORT64 2 | # undef BUILD_DIVSUFSORT64 3 | #endif 4 | #define HAVE_CONFIG_H 1 5 | #include 6 | #include "divsufsort_private.h" 7 | 8 | #include "divsufsort.c.inc.h" 9 | #include "trsort.c.inc.h" 10 | #define lg_table sssort_lg_table 11 | #include "sssort.c.inc.h" 12 | #include "utils.c.inc.h" 13 | 14 | -------------------------------------------------------------------------------- /detools/libdivsufsort/divsufsort64.c: -------------------------------------------------------------------------------- 1 | #ifdef BUILD_DIVSUFSORT64 2 | # undef BUILD_DIVSUFSORT64 3 | #endif 4 | #define BUILD_DIVSUFSORT64 1 5 | #define HAVE_CONFIG_H 1 6 | #include 7 | #include "divsufsort_private.h" 8 | 9 | #include "divsufsort.c.inc.h" 10 | #include "trsort.c.inc.h" 11 | #define lg_table sssort_lg_table 12 | #include "sssort.c.inc.h" 13 | #include "utils.c.inc.h" 14 | -------------------------------------------------------------------------------- /c/heatshrink/heatshrink_common.h: -------------------------------------------------------------------------------- 1 | #ifndef HEATSHRINK_H 2 | #define HEATSHRINK_H 3 | 4 | #define HEATSHRINK_AUTHOR "Scott Vokes " 5 | #define HEATSHRINK_URL "https://github.com/atomicobject/heatshrink" 6 | 7 | /* Version 0.4.1 */ 8 | #define HEATSHRINK_VERSION_MAJOR 0 9 | #define HEATSHRINK_VERSION_MINOR 4 10 | #define HEATSHRINK_VERSION_PATCH 1 11 | 12 | #define HEATSHRINK_MIN_WINDOW_BITS 4 13 | #define HEATSHRINK_MAX_WINDOW_BITS 15 14 | 15 | #define HEATSHRINK_MIN_LOOKAHEAD_BITS 3 16 | 17 | #define HEATSHRINK_LITERAL_MARKER 0x01 18 | #define HEATSHRINK_BACKREF_MARKER 0x00 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /detools/compression/lz4.py: -------------------------------------------------------------------------------- 1 | """LZ4 wrapper. 2 | 3 | """ 4 | 5 | import os 6 | from io import BytesIO 7 | import lz4.frame 8 | from ..errors import Error 9 | 10 | 11 | class Lz4Compressor(object): 12 | 13 | def __init__(self): 14 | self._compressor = lz4.frame.LZ4FrameCompressor( 15 | compression_level=lz4.frame.COMPRESSIONLEVEL_MAX) 16 | self._header = self._compressor.begin() 17 | 18 | def compress(self, data): 19 | compressed = self._compressor.compress(data) 20 | 21 | if self._header is not None: 22 | compressed = (self._header + compressed) 23 | self._header = None 24 | 25 | return compressed 26 | 27 | def flush(self): 28 | return self._compressor.flush() 29 | 30 | 31 | class Lz4Decompressor(lz4.frame.LZ4FrameDecompressor): 32 | 33 | pass 34 | -------------------------------------------------------------------------------- /c/examples/dump_restore/Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS += -std=c99 2 | CFLAGS += -Wall 3 | CFLAGS += -Wextra 4 | CFLAGS += -Werror 5 | CFLAGS += -g 6 | CFLAGS += -I../../heatshrink 7 | 8 | SRC += ../../detools.c 9 | SRC += main.c 10 | SRC += ../../heatshrink/heatshrink_decoder.c 11 | 12 | all: 13 | gcc $(CFLAGS) $(SRC) -llzma -o dump-restore 14 | rm -f state.bin 15 | @echo 16 | ./dump-restore \ 17 | ../../../tests/files/foo/old \ 18 | ../../../tests/files/foo/heatshrink.patch \ 19 | foo.new \ 20 | 10 25 21 | @echo 22 | ./dump-restore \ 23 | ../../../tests/files/foo/old \ 24 | ../../../tests/files/foo/heatshrink.patch \ 25 | foo.new \ 26 | 90 20 27 | @echo 28 | ./dump-restore \ 29 | ../../../tests/files/foo/old \ 30 | ../../../tests/files/foo/heatshrink.patch \ 31 | foo.new \ 32 | 25 0 33 | @echo 34 | cmp foo.new ../../../tests/files/foo/new 35 | -------------------------------------------------------------------------------- /c/heatshrink/heatshrink_config.h: -------------------------------------------------------------------------------- 1 | #ifndef HEATSHRINK_CONFIG_H 2 | #define HEATSHRINK_CONFIG_H 3 | 4 | /* Should functionality assuming dynamic allocation be used? */ 5 | #ifndef HEATSHRINK_DYNAMIC_ALLOC 6 | #define HEATSHRINK_DYNAMIC_ALLOC 0 7 | #endif 8 | 9 | #if HEATSHRINK_DYNAMIC_ALLOC 10 | /* Optional replacement of malloc/free */ 11 | #define HEATSHRINK_MALLOC(SZ) malloc(SZ) 12 | #define HEATSHRINK_FREE(P, SZ) free(P) 13 | #else 14 | /* Required parameters for static configuration */ 15 | #define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 256 16 | #define HEATSHRINK_STATIC_WINDOW_BITS 8 17 | #define HEATSHRINK_STATIC_LOOKAHEAD_BITS 7 18 | #endif 19 | 20 | /* Turn on logging for debugging. */ 21 | #define HEATSHRINK_DEBUGGING_LOGS 0 22 | 23 | /* Use indexing for faster compression. (This requires additional space.) */ 24 | #define HEATSHRINK_USE_INDEX 0 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /detools/compression/none.py: -------------------------------------------------------------------------------- 1 | """None encoding leaves the data as is. No compression is performed. 2 | 3 | """ 4 | 5 | from ..errors import Error 6 | 7 | 8 | class NoneCompressor(object): 9 | 10 | def compress(self, data): 11 | return data 12 | 13 | def flush(self): 14 | return b'' 15 | 16 | 17 | class NoneDecompressor(object): 18 | 19 | def __init__(self, number_of_bytes): 20 | self._number_of_bytes_left = number_of_bytes 21 | self._data = b'' 22 | 23 | def decompress(self, data, size): 24 | if self.eof: 25 | raise Error('Already at end of stream.') 26 | 27 | self._data += data 28 | decompressed = self._data[:size] 29 | self._data = self._data[size:] 30 | self._number_of_bytes_left -= len(decompressed) 31 | 32 | return decompressed 33 | 34 | @property 35 | def needs_input(self): 36 | return self._data == b'' and not self.eof 37 | 38 | @property 39 | def eof(self): 40 | return self._number_of_bytes_left == 0 41 | -------------------------------------------------------------------------------- /c/examples/in_place/Makefile: -------------------------------------------------------------------------------- 1 | FLAGS := \ 2 | -Os \ 3 | -ffunction-sections \ 4 | -fdata-sections \ 5 | -Wl,--gc-sections \ 6 | -std=c99 \ 7 | -I../../heatshrink 8 | 9 | SRC := \ 10 | ../../detools.c \ 11 | main.c 12 | 13 | all: 14 | gcc $(FLAGS) $(SRC) \ 15 | ../../heatshrink/heatshrink_decoder.c \ 16 | -llzma \ 17 | -o in-place 18 | size in-place 19 | 20 | heatshrink: 21 | gcc $(FLAGS) $(SRC) \ 22 | ../../heatshrink/heatshrink_decoder.c \ 23 | -DDETOOLS_CONFIG_FILE_IO=0 \ 24 | -DDETOOLS_CONFIG_COMPRESSION_NONE=0 \ 25 | -DDETOOLS_CONFIG_COMPRESSION_LZMA=0 \ 26 | -DDETOOLS_CONFIG_COMPRESSION_CRLE=0 \ 27 | -DDETOOLS_CONFIG_COMPRESSION_HEATSHRINK=1 \ 28 | -o in-place-heatshrink 29 | size in-place-heatshrink 30 | 31 | crle: 32 | gcc $(FLAGS) $(SRC) \ 33 | -DDETOOLS_CONFIG_FILE_IO=0 \ 34 | -DDETOOLS_CONFIG_COMPRESSION_NONE=0 \ 35 | -DDETOOLS_CONFIG_COMPRESSION_LZMA=0 \ 36 | -DDETOOLS_CONFIG_COMPRESSION_CRLE=1 \ 37 | -DDETOOLS_CONFIG_COMPRESSION_HEATSHRINK=0 \ 38 | -o in-place-crle 39 | size in-place-crle 40 | -------------------------------------------------------------------------------- /c/Makefile: -------------------------------------------------------------------------------- 1 | CC = $(CROSS_COMPILE)gcc 2 | AR = $(CROSS_COMPILE)ar 3 | 4 | # Install prefix. 5 | PREFIX ?= /usr/local 6 | 7 | OPT ?= -O2 8 | CFLAGS += -g 9 | CFLAGS += -Wall 10 | CFLAGS += -Wextra 11 | CFLAGS += -Wdouble-promotion 12 | CFLAGS += -Wfloat-equal 13 | CFLAGS += -Wformat=2 14 | CFLAGS += -Wshadow 15 | CFLAGS += -Werror 16 | CFLAGS += -std=c99 17 | CFLAGS += $(OPT) 18 | CFLAGS += -Iheatshrink 19 | CFLAGS += $(CFLAGS_EXTRA) 20 | 21 | SRC += heatshrink/heatshrink_decoder.c 22 | SRC += detools.c 23 | SRC += main.c 24 | 25 | all: 26 | $(CC) $(CFLAGS) $(SRC) -llzma -o detools 27 | 28 | clean: 29 | rm -f detools detools.o libdetools.a esp8266-20190125-v1.10.bin 30 | 31 | test: all 32 | ./detools apply_patch \ 33 | ../tests/files/micropython/esp8266-20180511-v1.9.4.bin \ 34 | ../tests/files/micropython/esp8266-20180511-v1.9.4--20190125-v1.10-heatshrink.patch \ 35 | esp8266-20190125-v1.10.bin 36 | cmp \ 37 | esp8266-20190125-v1.10.bin \ 38 | ../tests/files/micropython/esp8266-20190125-v1.10.bin 39 | 40 | library: 41 | $(CC) $(CFLAGS) detools.c -c -o detools.o 42 | $(AR) cr libdetools.a detools.o 43 | 44 | install: 45 | mkdir -p $(PREFIX)/include 46 | install -c -m 644 detools.h $(PREFIX)/include 47 | mkdir -p $(PREFIX)/lib 48 | install -c -m 644 libdetools.a $(PREFIX)/lib 49 | -------------------------------------------------------------------------------- /tests/test_data_format.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from elftools.elf.elffile import ELFFile 3 | 4 | from detools.data_format.utils import Blocks 5 | from detools.data_format import elf 6 | 7 | 8 | class DetoolsDataFormatTest(unittest.TestCase): 9 | 10 | def test_blocks(self): 11 | blocks = Blocks() 12 | 13 | self.assertEqual( 14 | repr(blocks), 15 | 'Blocks(number_of_blocks=0, blocks=[])') 16 | 17 | blocks.append(0, 1, [2, 3, 4]) 18 | 19 | self.assertEqual( 20 | repr(blocks), 21 | 'Blocks(number_of_blocks=1, blocks=[Block(from_offset=0, ' 22 | 'to_address=1, number_of_values=3)])') 23 | 24 | self.assertEqual(blocks.to_bytes(), 25 | (b'\x01\x00\x01\x03', b'\x02\x03\x04')) 26 | 27 | def test_from_elf_file(self): 28 | filename = 'tests/files/micropython/esp8266-20180511-v1.9.4.elf' 29 | 30 | with open(filename, 'rb') as fin: 31 | elffile = ELFFile(fin) 32 | (code_range, data_range) = elf.from_file(elffile) 33 | 34 | self.assertEqual(code_range.begin, 0x40209040) 35 | self.assertEqual(code_range.end, 0x4027b365) 36 | self.assertEqual(data_range.begin, 0x4027b368) 37 | self.assertEqual(data_range.end, 0x40293ab8) 38 | 39 | 40 | if __name__ == '__main__': 41 | unittest.main() 42 | -------------------------------------------------------------------------------- /detools/sais/sais.h: -------------------------------------------------------------------------------- 1 | /* 2 | * sais.c for sais-lite 3 | * Copyright (c) 2008-2010 Yuta Mori All Rights Reserved. 4 | * Copyright (c) 2019, Erik Moqvist (Python wrapper). 5 | * 6 | * Permission is hereby granted, free of charge, to any person 7 | * obtaining a copy of this software and associated documentation 8 | * files (the "Software"), to deal in the Software without 9 | * restriction, including without limitation the rights to use, 10 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | * copies of the Software, and to permit persons to whom the 12 | * Software is furnished to do so, subject to the following 13 | * conditions: 14 | * 15 | * The above copyright notice and this permission notice shall be 16 | * included in all copies or substantial portions of the Software. 17 | * 18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 20 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 21 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 22 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 23 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 25 | * OTHER DEALINGS IN THE SOFTWARE. 26 | */ 27 | 28 | #include 29 | 30 | int32_t sais(const uint8_t *t_p, int32_t *sa_p, int32_t n); 31 | -------------------------------------------------------------------------------- /c/tst/Makefile: -------------------------------------------------------------------------------- 1 | TESTS += test_detools.c 2 | TESTS += test_dump_restore.c 3 | TESTS += test_command_line.c 4 | TESTS += test_fuzzer.c 5 | INC += ../heatshrink 6 | SRC += ../detools.c 7 | SRC += ../heatshrink/heatshrink_decoder.c 8 | SRC += utils.c 9 | LIBS += lzma 10 | 11 | default: 12 | $(MAKE) -C .. OPT=-O0 CFLAGS_EXTRA="-coverage" 13 | $(MAKE) all 14 | 15 | FUZZER_CFLAGS += -fprofile-instr-generate 16 | FUZZER_CFLAGS += -g 17 | FUZZER_CFLAGS += -fcoverage-mapping 18 | FUZZER_CFLAGS += -fsanitize=address,fuzzer 19 | FUZZER_CFLAGS += -fsanitize=signed-integer-overflow 20 | FUZZER_CFLAGS += -fno-sanitize-recover=all 21 | FUZZER_CFLAGS += $(INC:%=-I%) 22 | FUZZER_CFLAGS += -I.. 23 | FUZZER_CFLAGS += -Wall 24 | FUZZER_EXECUTION_TIME ?= 30 25 | FUZZER_EXE = fuzzer-corrupt-patch 26 | 27 | fuzz-corpus-patch: 28 | mkdir -p corpus 29 | clang $(FUZZER_CFLAGS) \ 30 | ../detools.c \ 31 | ../heatshrink/heatshrink_decoder.c \ 32 | fuzzer_corrupt_patch.c \ 33 | -l lzma -o $(FUZZER_EXE) 34 | rm -f $(FUZZER_EXE).profraw 35 | LLVM_PROFILE_FILE="$(FUZZER_EXE).profraw" \ 36 | ./$(FUZZER_EXE) \ 37 | -max_total_time=$(FUZZER_EXECUTION_TIME) \ 38 | -print_final_stats \ 39 | -rss_limit_mb=0 \ 40 | corpus 41 | llvm-profdata merge \ 42 | -sparse $(FUZZER_EXE).profraw \ 43 | -o $(FUZZER_EXE).profdata 44 | llvm-cov show ./$(FUZZER_EXE) \ 45 | -instr-profile=$(FUZZER_EXE).profdata 46 | 47 | include test.mk 48 | -------------------------------------------------------------------------------- /c/tst/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_H 2 | #define UTILS_H 3 | 4 | #include 5 | 6 | struct files_t { 7 | struct { 8 | FILE *file_p; 9 | } from; 10 | struct { 11 | FILE *file_p; 12 | char *buf_p; 13 | size_t size; 14 | int offset; 15 | int saved_offset; 16 | } to; 17 | struct { 18 | FILE *file_p; 19 | char *buf_p; 20 | size_t size; 21 | } state; 22 | struct { 23 | const uint8_t *buf_p; 24 | size_t size; 25 | } patch; 26 | struct { 27 | const uint8_t *buf_p; 28 | size_t size; 29 | } expected_new; 30 | }; 31 | 32 | extern struct files_t utils_files; 33 | 34 | int utils_from_read(void *arg_p, uint8_t *buf_p, size_t size); 35 | 36 | int utils_from_seek(void *arg_p, int offset); 37 | 38 | int utils_to_write(void *arg_p, const uint8_t *buf_p, size_t size); 39 | 40 | int utils_state_write(void *arg_p, const void *buf_p, size_t size); 41 | 42 | int utils_state_read(void *arg_p, void *buf_p, size_t size); 43 | 44 | const uint8_t *utils_read_file(const char *filename_p, size_t *size_p); 45 | 46 | void utils_files_init(const char *from_filename_p, 47 | const char *patch_filename_p, 48 | const char *expected_new_filename_p); 49 | 50 | void utils_files_destroy(void); 51 | 52 | void utils_files_assert_and_destroy(void); 53 | 54 | void utils_files_reopen_from(void); 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /c/examples/dump_restore/README.rst: -------------------------------------------------------------------------------- 1 | Dump and restore patcher state 2 | ============================== 3 | 4 | Command line tool to test the dump and restore feature. 5 | 6 | Build and run 7 | ------------- 8 | 9 | .. code-block:: text 10 | 11 | $ make 12 | gcc -std=c99 -Wall -Wextra -Werror -I../../heatshrink ../../detools.c main.c ../../heatshrink/heatshrink_decoder.c -llzma -o dump-restore 13 | 14 | ./dump-restore \ 15 | ../../../../tests/files/foo/old \ 16 | ../../../../tests/files/foo/heatshrink.patch \ 17 | foo.new \ 18 | 10 25 19 | No state to restore. 20 | Processing 10 byte(s) patch data starting at offset 0. 21 | Storing state in 'state.bin'. 22 | Processing 25 byte(s) patch data starting at offset 10. 23 | 24 | ./dump-restore \ 25 | ../../../../tests/files/foo/old \ 26 | ../../../../tests/files/foo/heatshrink.patch \ 27 | foo.new \ 28 | 90 20 29 | Restoring state from 'state.bin'. 30 | Processing 90 byte(s) patch data starting at offset 10. 31 | Storing state in 'state.bin'. 32 | Processing 20 byte(s) patch data starting at offset 100. 33 | 34 | ./dump-restore \ 35 | ../../../../tests/files/foo/old \ 36 | ../../../../tests/files/foo/heatshrink.patch \ 37 | foo.new \ 38 | 25 0 39 | Restoring state from 'state.bin'. 40 | Processing 25 byte(s) patch data starting at offset 100. 41 | Removing state 'state.bin'. 42 | Patch successfully applied. To-file is 2780 bytes. 43 | 44 | cmp foo.new ../../../../tests/files/foo/new 45 | -------------------------------------------------------------------------------- /detools/compression/zstd.py: -------------------------------------------------------------------------------- 1 | """Zstandard wrapper. 2 | 3 | """ 4 | 5 | import os 6 | from io import BytesIO 7 | import zstandard 8 | from ..errors import Error 9 | 10 | 11 | class ZstdCompressor(object): 12 | 13 | def __init__(self): 14 | self._data = [] 15 | 16 | def compress(self, data): 17 | self._data.append(data) 18 | 19 | return b'' 20 | 21 | def flush(self): 22 | return zstandard.ZstdCompressor(level=22).compress(b''.join(self._data)) 23 | 24 | 25 | class ZstdDecompressor(object): 26 | 27 | def __init__(self, number_of_bytes): 28 | self._number_of_bytes_left = number_of_bytes 29 | self._output_offset = 0 30 | self._fout = BytesIO() 31 | decompressor = zstandard.ZstdDecompressor() 32 | self._decompressor = decompressor.stream_writer(self._fout) 33 | 34 | def decompress(self, data, size): 35 | if self.eof: 36 | raise Error('Already at end of stream.') 37 | 38 | self._number_of_bytes_left -= len(data) 39 | self._decompressor.write(data) 40 | 41 | self._fout.seek(self._output_offset, os.SEEK_SET) 42 | data = self._fout.read(size) 43 | self._output_offset += len(data) 44 | self._fout.seek(0, os.SEEK_END) 45 | 46 | return data 47 | 48 | @property 49 | def needs_input(self): 50 | return (self._output_offset == self._fout.tell()) and not self.eof 51 | 52 | @property 53 | def eof(self): 54 | return (self._number_of_bytes_left == 0 55 | and self._output_offset == self._fout.tell()) 56 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | bsdiff/sais.c based on sais-lite 2.4.1 : 2 | Copyright (c) 2008-2010 Yuta Mori. (MIT License) 3 | 4 | Other files: 5 | 6 | BSD 2-Clause License 7 | 8 | Copyright 2003-2005, Colin Percival (Original C implementation) 9 | Copyright (c) 2019, Erik Moqvist 10 | All rights reserved. 11 | 12 | Redistribution and use in source and binary forms, with or without 13 | modification, are permitted provided that the following conditions are met: 14 | 15 | * Redistributions of source code must retain the above copyright notice, this 16 | list of conditions and the following disclaimer. 17 | 18 | * Redistributions in binary form must reproduce the above copyright notice, 19 | this list of conditions and the following disclaimer in the documentation 20 | and/or other materials provided with the distribution. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 26 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 28 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | -------------------------------------------------------------------------------- /tests/test_suffix_array.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import struct 3 | 4 | import detools.sais 5 | 6 | 7 | def read_file(filename): 8 | with open(filename, 'rb') as fin: 9 | return fin.read() 10 | 11 | 12 | def suffix_array_list_to_bytearray(suffix_array): 13 | return bytearray().join([ 14 | struct.pack('=i', value) for value in suffix_array 15 | ]) 16 | 17 | 18 | class DetoolsSuffixArrayTest(unittest.TestCase): 19 | 20 | def test_suffix_array(self): 21 | datas = [ 22 | ( 23 | b'', 24 | [0] 25 | ), 26 | ( 27 | b'1', 28 | [1, 0] 29 | ), 30 | ( 31 | b'1234', 32 | [4, 0, 1, 2, 3] 33 | ), 34 | ( 35 | b'55555555', 36 | [8, 7, 6, 5, 4, 3, 2, 1, 0] 37 | ), 38 | ( 39 | b'adska9kkkoaofeopkjvuuuuewflk-0920314923fg', 40 | [ 41 | 41, 28, 32, 29, 34, 31, 37, 33, 38, 35, 42 | 30, 36, 5, 4, 0, 10, 1, 13, 23, 12, 43 | 39, 25, 40, 17, 27, 3, 16, 6, 7, 8, 44 | 26, 9, 11, 14, 15, 2, 22, 21, 20, 19, 45 | 18, 24 46 | ] 47 | ) 48 | ] 49 | 50 | for data, expected in datas: 51 | expected = suffix_array_list_to_bytearray(expected) 52 | suffix_array = bytearray(len(expected)) 53 | 54 | detools.suffix_array.sais(data, suffix_array) 55 | self.assertEqual(suffix_array, expected) 56 | 57 | detools.suffix_array.divsufsort(data, suffix_array) 58 | self.assertEqual(suffix_array, expected) 59 | 60 | 61 | if __name__ == '__main__': 62 | unittest.main() 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | *.o 8 | a.out 9 | 10 | # Distribution / packaging 11 | .Python 12 | env/ 13 | venv/ 14 | .venv/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *,cover 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | 57 | # Sphinx documentation 58 | docs/_build/ 59 | 60 | # PyBuilder 61 | target/ 62 | 63 | # Vim IDE 64 | *~ 65 | *.swp 66 | *.swo 67 | 68 | # IntelliJ IDEA 69 | .idea/ 70 | 71 | /foo* 72 | *.gcov 73 | *.gcda 74 | *.gcno 75 | /main 76 | c/detools 77 | c/foo.new 78 | /fuzzer* 79 | data-format-from.bin 80 | data-format-to.bin 81 | data-format-from-apply.bin 82 | pybv11-data-format-with-data-sections.patch 83 | pybv11-aarch64.patch 84 | pybv11-data-format-with-elf-data-sections.patch 85 | pybv11-elf-data-sections.new 86 | pybv11-data-format-with-elf-data-sections-offsets.patch 87 | libdetools.a 88 | Python-3.7.3.tar 89 | Python-3.8.1.tar 90 | Python.tar 91 | /*.patch 92 | to.tar 93 | c/tst/assert-apply-patch-in-place.mem 94 | c/tst/assert-apply-patch.new 95 | c/tst/corpus 96 | c/tst/fuzzer-corrupt-patch* 97 | c/tst/slow-unit-* 98 | c/tst/from 99 | c/tst/patch 100 | c/tst/to -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | test-python: 8 | 9 | runs-on: ubuntu-latest 10 | strategy: 11 | max-parallel: 4 12 | matrix: 13 | python-version: ['3.10', '3.13'] 14 | 15 | steps: 16 | - uses: actions/checkout@v1 17 | with: 18 | submodules: recursive 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v4 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install -r requirements.txt 27 | - name: Test 28 | run: | 29 | python setup.py build_ext -b . 30 | pytest 31 | 32 | test-c: 33 | 34 | runs-on: ubuntu-20.04 35 | 36 | steps: 37 | - uses: actions/checkout@v1 38 | - name: Set up Python 3.13 39 | uses: actions/setup-python@v4 40 | with: 41 | python-version: 3.13 42 | - name: Install dependencies 43 | run: | 44 | python -m pip install --upgrade pip 45 | pip install -r requirements.txt 46 | - name: Test 47 | run: | 48 | make test-c 49 | 50 | release: 51 | needs: [test-python, test-c] 52 | runs-on: ubuntu-20.04 53 | if: startsWith(github.ref, 'refs/tags') 54 | 55 | steps: 56 | - uses: actions/checkout@v1 57 | with: 58 | submodules: recursive 59 | - name: Set up Python 3.13 60 | uses: actions/setup-python@v4 61 | with: 62 | python-version: 3.13 63 | - name: Install pypa/build 64 | run: | 65 | python -m pip install build --user 66 | - name: Build a binary wheel and a source tarball 67 | run: | 68 | git clean -dfx 69 | python -m build --sdist --outdir dist/ . 70 | - name: Publish distribution 📦 to PyPI 71 | uses: pypa/gh-action-pypi-publish@release/v1 72 | with: 73 | skip_existing: true 74 | password: ${{ secrets.pypi_password }} 75 | -------------------------------------------------------------------------------- /c/tst/fuzzer_corrupt_patch.c: -------------------------------------------------------------------------------- 1 | /* #include */ 2 | #include 3 | #include "detools.h" 4 | 5 | static const uint8_t from_buf[256] = { 0, }; 6 | static int from_offset; 7 | 8 | static int from_read(void *arg_p, uint8_t *buf_p, size_t size) 9 | { 10 | if (((size_t)from_offset + size) > 256) { 11 | return (-1); 12 | } 13 | 14 | memcpy(buf_p, &from_buf[from_offset], size); 15 | from_offset += size; 16 | 17 | return (0); 18 | } 19 | 20 | static int from_seek(void *arg_p, int offset) 21 | { 22 | if ((offset < 0) || (offset > 256)) { 23 | return (-1); 24 | } 25 | 26 | if ((from_offset + offset) < 0) { 27 | return (-1); 28 | } 29 | 30 | if ((from_offset + offset) > 256) { 31 | return (-1); 32 | } 33 | 34 | from_offset += offset; 35 | 36 | return (0); 37 | } 38 | 39 | static int to_write(void *arg_p, const uint8_t *buf_p, size_t size) 40 | { 41 | return (0); 42 | } 43 | 44 | int LLVMFuzzerTestOneInput(const uint8_t *data_p, size_t size) 45 | { 46 | struct detools_apply_patch_t apply_patch; 47 | const uint8_t *patch_buf_p; 48 | size_t patch_size; 49 | int res; 50 | 51 | if (size < 1) { 52 | return (0); 53 | } 54 | 55 | patch_size = data_p[0]; 56 | size -= 1; 57 | 58 | if (size < patch_size) { 59 | return (0); 60 | } 61 | 62 | patch_buf_p = &data_p[1]; 63 | from_offset = 0; 64 | /* dbgh(patch_buf_p, patch_size); */ 65 | 66 | res = detools_apply_patch_init(&apply_patch, 67 | from_read, 68 | from_seek, 69 | patch_size, 70 | to_write, 71 | NULL); 72 | 73 | if (res != 0) { 74 | printf("detools_apply_patch_init() failed.\n"); 75 | exit(1); 76 | } 77 | 78 | detools_apply_patch_process(&apply_patch, patch_buf_p, patch_size); 79 | detools_apply_patch_finalize(&apply_patch); 80 | 81 | return (0); 82 | } 83 | -------------------------------------------------------------------------------- /tests/fuzzer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "../c/detools.h" 6 | 7 | #define membersof(v) (sizeof(v) / sizeof((v)[0])) 8 | 9 | static void create_file(const char *name_p, const uint8_t *buf_p, size_t size) 10 | { 11 | FILE *f_p; 12 | 13 | f_p = fopen(name_p, "wb"); 14 | assert(f_p != NULL); 15 | 16 | if (size > 0) { 17 | assert(fwrite(buf_p, size, 1, f_p) == 1); 18 | } 19 | 20 | assert(fclose(f_p) == 0); 21 | } 22 | 23 | static void create_patch(const char *from_p, 24 | const char *to_p, 25 | const char *patch_p, 26 | int compression) 27 | { 28 | char command[128]; 29 | static const char *compressions[] = { 30 | "none", 31 | "lzma" 32 | }; 33 | 34 | snprintf(&command[0], 35 | sizeof(command), 36 | "python3 -m detools create_patch -c %s %s %s %s", 37 | compressions[compression], 38 | from_p, 39 | to_p, 40 | patch_p); 41 | command[membersof(command) - 1] = '\0'; 42 | assert(system(command) == 0); 43 | } 44 | 45 | int LLVMFuzzerTestOneInput(const uint8_t *data_p, size_t size) 46 | { 47 | const uint8_t *from_p; 48 | const uint8_t *to_p; 49 | size_t from_size; 50 | size_t to_size; 51 | 52 | if (size < 2) { 53 | return (0); 54 | } 55 | 56 | size -= 2; 57 | from_size = ((data_p[0] * size) / 255); 58 | to_size = (size - from_size); 59 | from_p = &data_p[2]; 60 | to_p = &data_p[2 + from_size]; 61 | 62 | create_file("fuzzer.old", from_p, from_size); 63 | create_file("fuzzer.new", to_p, to_size); 64 | 65 | create_patch("fuzzer.old", "fuzzer.new", "fuzzer.patch", data_p[1] % 2); 66 | assert(detools_apply_patch_filenames("fuzzer.old", 67 | "fuzzer.patch", 68 | "fuzzer-patched.new") == to_size); 69 | assert(system("cmp fuzzer.new fuzzer-patched.new") == 0); 70 | 71 | return (0); 72 | } 73 | -------------------------------------------------------------------------------- /detools/data_format/__init__.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from operator import itemgetter 3 | 4 | from elftools.elf.elffile import ELFFile 5 | from elftools.elf.sections import SymbolTableSection 6 | 7 | from ..errors import Error 8 | from ..common import DATA_FORMAT_AARCH64 9 | from ..common import DATA_FORMAT_ARM_CORTEX_M4 10 | from ..common import DATA_FORMAT_XTENSA_LX106 11 | from ..common import format_bad_data_format 12 | from ..common import format_bad_data_format_number 13 | from . import aarch64 14 | from . import arm_cortex_m4 15 | from . import xtensa_lx106 16 | 17 | 18 | def encode(ffrom, fto, data_format, data_segment): 19 | """Returns the new from-data and to-data, along with a patch that can 20 | be used to convert the new from-data to the original to-data later 21 | (by the diff and from readers). 22 | 23 | """ 24 | 25 | if data_format == 'aarch64': 26 | return aarch64.encode(ffrom, fto, data_segment) 27 | elif data_format == 'arm-cortex-m4': 28 | return arm_cortex_m4.encode(ffrom, fto, data_segment) 29 | elif data_format == 'xtensa-lx106': 30 | return xtensa_lx106.encode(ffrom, fto, data_segment) 31 | else: 32 | raise Error(format_bad_data_format(data_format)) 33 | 34 | 35 | def create_readers(data_format, ffrom, patch, to_size): 36 | """Returns diff and from readers, used when applying a patch. 37 | 38 | """ 39 | 40 | if data_format == DATA_FORMAT_AARCH64: 41 | return aarch64.create_readers(ffrom, patch, to_size) 42 | elif data_format == DATA_FORMAT_ARM_CORTEX_M4: 43 | return arm_cortex_m4.create_readers(ffrom, patch, to_size) 44 | elif data_format == DATA_FORMAT_XTENSA_LX106: 45 | return xtensa_lx106.create_readers(ffrom, patch, to_size) 46 | else: 47 | raise Error(format_bad_data_format_number(data_format)) 48 | 49 | 50 | def info(data_format, patch, fsize): 51 | """Returns an info string. 52 | 53 | """ 54 | 55 | if data_format == DATA_FORMAT_AARCH64: 56 | return aarch64.info(patch, fsize) 57 | elif data_format == DATA_FORMAT_ARM_CORTEX_M4: 58 | return arm_cortex_m4.info(patch, fsize) 59 | elif data_format == DATA_FORMAT_XTENSA_LX106: 60 | return xtensa_lx106.info(patch, fsize) 61 | else: 62 | raise Error(format_bad_data_format_number(data_format)) 63 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import re 4 | from setuptools import setup 5 | from setuptools import find_packages 6 | from setuptools import Extension 7 | 8 | 9 | def find_version(): 10 | return re.search(r"^__version__ = '(.*)'$", 11 | open('detools/version.py', 'r').read(), 12 | re.MULTILINE).group(1) 13 | 14 | HDIFFPATCH_SOURCES = [ 15 | "HDiff/diff.cpp", 16 | "HDiff/private_diff/compress_detect.cpp", 17 | "HDiff/private_diff/suffix_string.cpp", 18 | "HDiff/private_diff/libdivsufsort/divsufsort.c", 19 | "HDiff/private_diff/libdivsufsort/divsufsort64.c", 20 | "HDiff/private_diff/limit_mem_diff/stream_serialize.cpp", 21 | "HDiff/private_diff/limit_mem_diff/digest_matcher.cpp", 22 | "HDiff/private_diff/limit_mem_diff/adler_roll.c", 23 | "HDiff/private_diff/bytes_rle.cpp", 24 | "HPatch/patch.c", 25 | ] 26 | 27 | HDIFFPATCH_SOURCES = [ 28 | "detools/HDiffPatch/libHDiffPatch/" + source 29 | for source in HDIFFPATCH_SOURCES 30 | ] 31 | HDIFFPATCH_SOURCES += ["detools/hdiffpatch.cpp"] 32 | HDIFFPATCH_SOURCES += ["detools/HDiffPatch/file_for_patch.c"] 33 | 34 | setup(name='detools', 35 | version=find_version(), 36 | description='Binary delta encoding tools.', 37 | long_description=open('README.rst', 'r').read(), 38 | author='Erik Moqvist', 39 | author_email='erik.moqvist@gmail.com', 40 | license='BSD', 41 | classifiers=[ 42 | 'License :: OSI Approved :: BSD License', 43 | 'Programming Language :: Python :: 3', 44 | ], 45 | url='https://github.com/eerimoq/detools', 46 | packages=find_packages(exclude=['tests']), 47 | install_requires=[ 48 | 'humanfriendly', 49 | 'bitstruct', 50 | 'pyelftools', 51 | 'zstandard', 52 | 'lz4', 53 | 'heatshrink2' 54 | ], 55 | ext_modules=[ 56 | Extension(name="detools.suffix_array", 57 | sources=[ 58 | "detools/suffix_array.c", 59 | "detools/sais/sais.c", 60 | "detools/libdivsufsort/divsufsort.c" 61 | ]), 62 | Extension(name="detools.bsdiff", sources=["detools/bsdiff.c"]), 63 | Extension(name="detools.hdiffpatch", sources=HDIFFPATCH_SOURCES) 64 | ], 65 | test_suite="tests", 66 | entry_points={ 67 | 'console_scripts': ['detools=detools.__init__:_main'] 68 | }) 69 | -------------------------------------------------------------------------------- /detools/compression/heatshrink.py: -------------------------------------------------------------------------------- 1 | """Heatshrink wrapper. 2 | 3 | """ 4 | 5 | import bitstruct 6 | 7 | from heatshrink2.core import Writer 8 | from heatshrink2.core import Reader 9 | from heatshrink2.core import Encoder 10 | 11 | 12 | def pack_header(window_sz2, lookahead_sz2): 13 | return bitstruct.pack('u4u4', window_sz2 - 4, lookahead_sz2 - 3) 14 | 15 | 16 | def unpack_header(data): 17 | window_sz2, lookahead_sz2 = bitstruct.unpack('u4u4', data) 18 | 19 | return window_sz2 + 4, lookahead_sz2 + 3 20 | 21 | 22 | class HeatshrinkCompressor(object): 23 | 24 | def __init__(self, window_sz2, lookahead_sz2): 25 | self._data = pack_header(window_sz2, lookahead_sz2) 26 | self._encoder = Encoder(Writer(window_sz2=window_sz2, 27 | lookahead_sz2=lookahead_sz2)) 28 | 29 | def compress(self, data): 30 | compressed = self._encoder.fill(data) 31 | 32 | if self._data: 33 | compressed = self._data + compressed 34 | self._data = b'' 35 | 36 | return compressed 37 | 38 | def flush(self): 39 | return self._data + self._encoder.finish() 40 | 41 | 42 | class HeatshrinkDecompressor(object): 43 | 44 | def __init__(self, number_of_bytes): 45 | self._number_of_bytes_left = number_of_bytes 46 | self._data = b'' 47 | self._encoder = None 48 | self.window_sz2 = None 49 | self.lookahead_sz2 = None 50 | 51 | def decompress(self, data, size): 52 | if self._encoder is None: 53 | if not data: 54 | return b'' 55 | 56 | self.window_sz2, self.lookahead_sz2 = unpack_header(data[:1]) 57 | self._encoder = Encoder(Reader(window_sz2=self.window_sz2, 58 | lookahead_sz2=self.lookahead_sz2)) 59 | data = data[1:] 60 | self._number_of_bytes_left -= 1 61 | 62 | if self._number_of_bytes_left > 0: 63 | self._data += self._encoder.fill(data) 64 | self._number_of_bytes_left -= len(data) 65 | 66 | if self._number_of_bytes_left == 0: 67 | self._data += self._encoder.finish() 68 | self._number_of_bytes_left = -1 69 | 70 | decompressed = self._data[:size] 71 | self._data = self._data[size:] 72 | 73 | return decompressed 74 | 75 | @property 76 | def needs_input(self): 77 | return self._data == b'' and not self.eof 78 | 79 | @property 80 | def eof(self): 81 | return self._number_of_bytes_left == -1 and self._data == b'' 82 | -------------------------------------------------------------------------------- /tests/test_bsdiff.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import struct 3 | 4 | import detools.bsdiff 5 | 6 | 7 | def read_file(filename): 8 | with open(filename, 'rb') as fin: 9 | return fin.read() 10 | 11 | 12 | def suffix_array_list_to_bytearray(suffix_array): 13 | return bytearray().join([ 14 | struct.pack('=i', value) for value in suffix_array 15 | ]) 16 | 17 | 18 | class DetoolsBsdiffTest(unittest.TestCase): 19 | 20 | def test_bsdiff(self): 21 | datas = [ 22 | ( 23 | [0], 24 | b'', 25 | b'', 26 | [] 27 | ), 28 | ( 29 | [1, 0], 30 | b'1', 31 | b'12', 32 | [ 33 | b'\x01', b'\x00', b'\x01', b'2', b'\x41' 34 | ] 35 | ), 36 | ( 37 | [4, 0, 1, 2, 3], 38 | b'1234', 39 | b'29990812398409812', 40 | [ 41 | b'\x00', b'', b'\x11', b'29990812398409812', b'\x01' 42 | ] 43 | ), 44 | ( 45 | [ 46 | 41, 28, 32, 29, 34, 31, 37, 33, 38, 35, 47 | 30, 36, 5, 4, 0, 10, 1, 13, 23, 12, 48 | 39, 25, 40, 17, 27, 3, 16, 6, 7, 8, 49 | 26, 9, 11, 14, 15, 2, 22, 21, 20, 19, 50 | 18, 24 51 | ], 52 | b'adska9kkkoaofeopkjvuuuuewflk-0920314923fg', 53 | b'adska9kkkoaofeopkjvuuuuewflk-0920314923fg1', 54 | [ 55 | b'\x29', 56 | b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' 57 | b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' 58 | b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 59 | b'\x01', 60 | b'1', 61 | b'\x47' 62 | ] 63 | ) 64 | ] 65 | 66 | for suffix_array, from_data, to_data, chunks in datas: 67 | suffix_array = suffix_array_list_to_bytearray(suffix_array) 68 | self.assertEqual( 69 | detools.bsdiff.create_patch(suffix_array, 70 | from_data, 71 | to_data, 72 | bytearray(4 * (len(from_data) + 1))), 73 | chunks) 74 | 75 | 76 | if __name__ == '__main__': 77 | unittest.main() 78 | -------------------------------------------------------------------------------- /detools/libdivsufsort/config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * config.h for libdivsufsort 3 | * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person 6 | * obtaining a copy of this software and associated documentation 7 | * files (the "Software"), to deal in the Software without 8 | * restriction, including without limitation the rights to use, 9 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the 11 | * Software is furnished to do so, subject to the following 12 | * conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be 15 | * included in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 19 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 21 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | * OTHER DEALINGS IN THE SOFTWARE. 25 | */ 26 | 27 | #ifndef _CONFIG_H 28 | #define _CONFIG_H 1 29 | 30 | #ifdef __cplusplus 31 | extern "C" { 32 | #endif /* __cplusplus */ 33 | 34 | /** Define to the version of this package. **/ 35 | #define PROJECT_VERSION_FULL "2.0.1-14-g5f60d6f" 36 | 37 | /** Define to 1 if you have the header files. **/ 38 | #define HAVE_INTTYPES_H 0 39 | #define HAVE_STDDEF_H 1 40 | #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 41 | # define HAVE_STDINT_H 1 42 | #else 43 | # define HAVE_STDINT_H 0 44 | #endif 45 | #define HAVE_STDLIB_H 1 46 | #define HAVE_STRING_H 1 47 | #define HAVE_STRINGS_H 0 48 | #define HAVE_MEMORY_H 1 49 | #define HAVE_SYS_TYPES_H 0 50 | 51 | /** for WinIO **/ 52 | /* #undef HAVE_IO_H */ 53 | /* #undef HAVE_FCNTL_H */ 54 | /* #undef HAVE__SETMODE */ 55 | /* #undef HAVE_SETMODE */ 56 | /* #undef HAVE__FILENO */ 57 | /* #undef HAVE_FOPEN_S */ 58 | /* #undef HAVE__O_BINARY */ 59 | #ifndef HAVE__SETMODE 60 | # if HAVE_SETMODE 61 | # define _setmode setmode 62 | # define HAVE__SETMODE 1 63 | # endif 64 | # if HAVE__SETMODE && !HAVE__O_BINARY 65 | # define _O_BINARY 0 66 | # define HAVE__O_BINARY 1 67 | # endif 68 | #endif 69 | 70 | /** for inline **/ 71 | #ifndef INLINE 72 | # ifdef _MSC_VER 73 | # define INLINE __inline 74 | # else 75 | # define INLINE inline 76 | # endif 77 | #endif 78 | 79 | /** for VC++ warning **/ 80 | #ifdef _MSC_VER 81 | #pragma warning(disable: 4127) 82 | #endif 83 | 84 | 85 | #ifdef __cplusplus 86 | } /* extern "C" */ 87 | #endif /* __cplusplus */ 88 | 89 | #endif /* _CONFIG_H */ 90 | -------------------------------------------------------------------------------- /tests/test_none.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import detools 4 | from detools.create import NoneCompressor 5 | from detools.apply import NoneDecompressor 6 | 7 | 8 | class DetoolsNoneTest(unittest.TestCase): 9 | 10 | def test_compress(self): 11 | datas = [ 12 | ( [b''], b''), 13 | ( [b'A'], b'A'), 14 | ( [b'ABBCC', b'CBBA'], b'ABBCCCBBA'), 15 | ( [126 * b'A', b'', b'A'], 127 * b'A') 16 | ] 17 | 18 | for chunks, compressed in datas: 19 | compressor = NoneCompressor() 20 | data = b'' 21 | 22 | for chunk in chunks: 23 | data += compressor.compress(chunk) 24 | 25 | data += compressor.flush() 26 | 27 | self.assertEqual(data, compressed) 28 | 29 | def test_decompress_no_data(self): 30 | compressed = b'' 31 | 32 | decompressor = NoneDecompressor(len(compressed)) 33 | 34 | self.assertEqual(decompressor.needs_input, False) 35 | self.assertEqual(decompressor.eof, True) 36 | 37 | def test_decompress(self): 38 | datas = [ 39 | ( [b'A'], b'A'), 40 | ( [b'ABBCCCBBA'], b'ABBCCCBBA'), 41 | ( [127 * b'A'], 127 * b'A') 42 | ] 43 | 44 | for chunks, decompressed in datas: 45 | decompressor = NoneDecompressor(sum([len(c) for c in chunks])) 46 | 47 | for chunk in chunks: 48 | self.assertEqual(decompressor.needs_input, True) 49 | self.assertEqual(decompressor.eof, False) 50 | decompressor.decompress(chunk, 0) 51 | 52 | self.assertEqual(decompressor.needs_input, False) 53 | 54 | data = b'' 55 | 56 | while not decompressor.eof: 57 | data += decompressor.decompress(b'', 1) 58 | 59 | self.assertEqual(data, decompressed) 60 | 61 | def test_decompress_at_eof(self): 62 | decompressor = NoneDecompressor(1) 63 | 64 | self.assertEqual(decompressor.decompress(b'5', 1), b'5') 65 | self.assertEqual(decompressor.eof, True) 66 | 67 | with self.assertRaises(detools.Error) as cm: 68 | decompressor.decompress(b'6', 1) 69 | 70 | self.assertEqual(str(cm.exception), 'Already at end of stream.') 71 | 72 | with self.assertRaises(detools.Error) as cm: 73 | decompressor.decompress(b'', 1) 74 | 75 | self.assertEqual(str(cm.exception), 'Already at end of stream.') 76 | 77 | def test_decompress_ignore_extra_data(self): 78 | compressed = b'A' 79 | decompressor = NoneDecompressor(len(compressed)) 80 | 81 | self.assertEqual(decompressor.decompress(compressed + b'B', 1), b'A') 82 | self.assertEqual(decompressor.eof, True) 83 | 84 | 85 | if __name__ == '__main__': 86 | unittest.main() 87 | -------------------------------------------------------------------------------- /c/tst/test_command_line.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "nala.h" 3 | 4 | TEST(apply_patch_foo) 5 | { 6 | ASSERT_EQ(system("../detools apply_patch " 7 | "../../tests/files/foo/old " 8 | "../../tests/files/foo/patch " 9 | "build/foo.new"), 10 | 0); 11 | ASSERT_FILE_EQ("build/foo.new", "../../tests/files/foo/new"); 12 | } 13 | 14 | TEST(apply_patch_foo_heatshrink) 15 | { 16 | ASSERT_EQ(system("../detools apply_patch " 17 | "../../tests/files/foo/old " 18 | "../../tests/files/foo/heatshrink.patch " 19 | "build/heatshrink.foo.new"), 20 | 0); 21 | ASSERT_FILE_EQ("build/heatshrink.foo.new", "../../tests/files/foo/new"); 22 | } 23 | 24 | TEST(apply_patch_in_place_foo) 25 | { 26 | ASSERT_EQ(system("cp ../../tests/files/foo/old build/foo.mem"), 0); 27 | ASSERT_EQ(system("../detools apply_patch_in_place " 28 | "build/foo.mem " 29 | "../../tests/files/foo/in-place-3000-500.patch"), 0); 30 | ASSERT_FILE_EQ("build/foo.mem", 31 | "../../tests/files/foo/in-place-3000-500.mem"); 32 | } 33 | 34 | TEST(usage) 35 | { 36 | CAPTURE_OUTPUT(output, errput) { 37 | ASSERT_EQ(system("../detools"), 256); 38 | } 39 | 40 | ASSERT_EQ(output, "Usage: ../detools {apply_patch, apply_patch_in_place}\n"); 41 | ASSERT_EQ(errput, ""); 42 | } 43 | 44 | TEST(usage_apply_patch) 45 | { 46 | CAPTURE_OUTPUT(output, errput) { 47 | ASSERT_EQ(system("../detools apply_patch"), 256); 48 | } 49 | 50 | ASSERT_EQ(output, 51 | "Usage: ../detools apply_patch \n"); 52 | ASSERT_EQ(errput, ""); 53 | } 54 | 55 | TEST(usage_apply_patch_missing_to_file) 56 | { 57 | CAPTURE_OUTPUT(output, errput) { 58 | ASSERT_EQ(system("../detools apply_patch " 59 | "../../tests/files/foo/old " 60 | "../../tests/files/foo/patch"), 61 | 256); 62 | } 63 | 64 | ASSERT_EQ(output, 65 | "Usage: ../detools apply_patch \n"); 66 | ASSERT_EQ(errput, ""); 67 | } 68 | 69 | TEST(usage_apply_patch_in_place) 70 | { 71 | CAPTURE_OUTPUT(output, errput) { 72 | ASSERT_EQ(system("../detools apply_patch_in_place"), 256); 73 | } 74 | 75 | ASSERT_EQ(output, 76 | "Usage: ../detools apply_patch_in_place \n"); 77 | ASSERT_EQ(errput, ""); 78 | } 79 | 80 | TEST(usage_apply_patch_in_place_missing_mem_file) 81 | { 82 | CAPTURE_OUTPUT(output, errput) { 83 | ASSERT_EQ(system("../detools apply_patch_in_place tests/files/foo/old"), 84 | 256); 85 | } 86 | 87 | ASSERT_EQ(output, 88 | "Usage: ../detools apply_patch_in_place \n"); 89 | ASSERT_EQ(errput, ""); 90 | } 91 | -------------------------------------------------------------------------------- /tests/benchmark.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PYTHON="${PYTHON:-python3}" 4 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" 5 | 6 | echo "Python: $PYTHON" 7 | 8 | export PYTHONPATH=$SCRIPT_DIR/.. 9 | 10 | function create_patch() { 11 | echo "================= $1 =================" 12 | \time -f "RSS=%M elapsed=%E" \ 13 | $PYTHON -m detools create_patch $1 $from_file $to_file $2 \ 14 | > /dev/null 15 | ls -lh $2 16 | } 17 | 18 | function apply_patch() { 19 | echo "================= $1 =================" 20 | \time -f "RSS=%M elapsed=%E" \ 21 | $PYTHON -m detools apply_patch $from_file $1 Python.tar \ 22 | > /dev/null 23 | cmp Python.tar $to_file 24 | \time -f "RSS=%M elapsed=%E" \ 25 | c/detools apply_patch $from_file $1 Python.tar 26 | cmp Python.tar $to_file 27 | } 28 | 29 | function print_files() { 30 | echo "***************************************************************" 31 | echo "* From: $from_file" 32 | echo "* To: $to_file" 33 | echo "***************************************************************" 34 | } 35 | 36 | if [ ! -e Python-3.7.3.tar ] ; then 37 | wget https://www.python.org/ftp/python/3.7.3/Python-3.7.3.tgz 38 | gunzip Python-3.7.3.tgz 39 | fi 40 | 41 | if [ ! -e Python-3.8.1.tar ] ; then 42 | wget https://www.python.org/ftp/python/3.8.1/Python-3.8.1.tgz 43 | gunzip Python-3.8.1.tgz 44 | fi 45 | 46 | from_file=Python-3.7.3.tar 47 | to_file=Python-3.8.1.tar 48 | 49 | print_files 50 | 51 | for algorithm in bsdiff hdiffpatch match-blocks ; do 52 | for patch_type in sequential hdiffpatch ; do 53 | for compression in lzma none ; do 54 | create_patch \ 55 | "-a $algorithm -t $patch_type -c $compression" \ 56 | "$algorithm-$patch_type-$compression.patch" 57 | done 58 | done 59 | done 60 | 61 | for algorithm in bsdiff hdiffpatch match-blocks ; do 62 | for patch_type in sequential hdiffpatch ; do 63 | for compression in lzma none ; do 64 | apply_patch "$algorithm-$patch_type-$compression.patch" 65 | done 66 | done 67 | done 68 | 69 | from_file=tests/files/micropython/esp8266-20180511-v1.9.4.bin 70 | to_file=tests/files/micropython/esp8266-20190125-v1.10.bin 71 | 72 | print_files 73 | 74 | for algorithm in bsdiff hdiffpatch match-blocks ; do 75 | for patch_type in sequential hdiffpatch ; do 76 | for compression in lzma none ; do 77 | create_patch \ 78 | "-a $algorithm -t $patch_type -c $compression" \ 79 | "$algorithm-$patch_type-$compression.patch" 80 | done 81 | done 82 | done 83 | 84 | for algorithm in bsdiff hdiffpatch match-blocks ; do 85 | for patch_type in sequential hdiffpatch ; do 86 | for compression in lzma none ; do 87 | apply_patch "$algorithm-$patch_type-$compression.patch" 88 | done 89 | done 90 | done 91 | 92 | exit 0 93 | -------------------------------------------------------------------------------- /c/main.c: -------------------------------------------------------------------------------- 1 | /** 2 | * BSD 2-Clause License 3 | * 4 | * Copyright (c) 2019, Erik Moqvist 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * 14 | * * Redistributions in binary form must reproduce the above copyright 15 | * notice, this list of conditions and the following disclaimer in 16 | * the documentation and/or other materials provided with the 17 | * distribution. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 22 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 23 | * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 24 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 25 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 28 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 30 | * OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #include 34 | #include "detools.h" 35 | 36 | static void print_usage_and_exit(const char *name_p) 37 | { 38 | printf("Usage: %s {apply_patch, apply_patch_in_place}\n", name_p); 39 | exit(1); 40 | } 41 | 42 | static void print_apply_patch_usage_and_exit(const char *name_p) 43 | { 44 | printf("Usage: %s apply_patch \n", 45 | name_p); 46 | exit(1); 47 | } 48 | 49 | static void print_apply_patch_in_place_usage_and_exit(const char *name_p) 50 | { 51 | printf("Usage: %s apply_patch_in_place \n", 52 | name_p); 53 | exit(1); 54 | } 55 | 56 | int main(int argc, const char *argv[]) 57 | { 58 | int res; 59 | 60 | if (argc < 2) { 61 | print_usage_and_exit(argv[0]); 62 | } 63 | 64 | if (strcmp("apply_patch", argv[1]) == 0) { 65 | if (argc != 5) { 66 | print_apply_patch_usage_and_exit(argv[0]); 67 | } 68 | 69 | res = detools_apply_patch_filenames(argv[2], argv[3], argv[4]); 70 | } else if (strcmp("apply_patch_in_place", argv[1]) == 0) { 71 | if (argc != 4) { 72 | print_apply_patch_in_place_usage_and_exit(argv[0]); 73 | } 74 | 75 | res = detools_apply_patch_in_place_filenames(argv[2], 76 | argv[3], 77 | NULL, 78 | NULL); 79 | } else { 80 | print_usage_and_exit(argv[0]); 81 | } 82 | 83 | if (res > 0) { 84 | res = 0; 85 | } else if (res < 0) { 86 | res *= -1; 87 | 88 | printf("error: %s (error code %d)\n", 89 | detools_error_as_string(res), 90 | res); 91 | } 92 | 93 | return (res); 94 | } 95 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS := \ 2 | -Wall \ 3 | -Wextra \ 4 | -Wdouble-promotion \ 5 | -Wfloat-equal \ 6 | -Wformat=2 \ 7 | -Wshadow \ 8 | -Werror \ 9 | -Wpedantic \ 10 | -std=c99 \ 11 | -g \ 12 | --coverage \ 13 | -Ic/heatshrink 14 | 15 | FUZZER_CFLAGS = \ 16 | -fprofile-instr-generate \ 17 | -fcoverage-mapping \ 18 | -Itests/files/c_source \ 19 | -Ic/heatshrink \ 20 | -g \ 21 | -fsanitize=address,fuzzer \ 22 | -fsanitize=signed-integer-overflow \ 23 | -fno-sanitize-recover=all 24 | FUZZER_EXECUTION_TIME ?= 30 25 | 26 | test: 27 | env CFLAGS=--coverage python3 setup.py test 28 | $(MAKE) test-sdist 29 | $(MAKE) test-c 30 | find . -name "*.gcno" -exec gcov {} + 31 | $(MAKE) test-c-fuzzer FUZZER_EXECUTION_TIME=1 32 | $(MAKE) -C c/tst fuzz-corpus-patch FUZZER_EXECUTION_TIME=1 33 | 34 | test-sdist: 35 | rm -rf dist 36 | python3 setup.py sdist 37 | cd dist && \ 38 | mkdir test && \ 39 | cd test && \ 40 | tar xf ../*.tar.gz && \ 41 | cd detools-* && \ 42 | python3 setup.py test 43 | 44 | test-c: 45 | $(MAKE) -C c/tst SANITIZE=yes 46 | $(CC) $(CFLAGS) -DDETOOLS_CONFIG_FILE_IO=0 -Ic/heatshrink \ 47 | -c c/detools.c -o detools.no-file-io.o 48 | $(CC) $(CFLAGS) -DDETOOLS_CONFIG_COMPRESSION_NONE=0 -Ic/heatshrink \ 49 | -c c/detools.c -o detools.no-none.o 50 | $(CC) $(CFLAGS) -DDETOOLS_CONFIG_COMPRESSION_LZMA=0 -Ic/heatshrink \ 51 | -c c/detools.c -o detools.no-lzma.o 52 | $(CC) $(CFLAGS) -DDETOOLS_CONFIG_COMPRESSION_CRLE=0 -Ic/heatshrink \ 53 | -c c/detools.c -o detools.no-crle.o 54 | $(CC) $(CFLAGS) -DDETOOLS_CONFIG_COMPRESSION_HEATSHRINK=0 -Ic/heatshrink \ 55 | -c c/detools.c -o detools.no-crle.o 56 | $(CC) $(CFLAGS) \ 57 | -DDETOOLS_CONFIG_COMPRESSION_NONE=0 \ 58 | -DDETOOLS_CONFIG_COMPRESSION_CRLE=0 \ 59 | -Ic/heatshrink -c c/detools.c -o detools.no-crle.o 60 | $(CC) $(CFLAGS) \ 61 | -DDETOOLS_CONFIG_COMPRESSION_NONE=0 \ 62 | -DDETOOLS_CONFIG_COMPRESSION_LZMA=0 \ 63 | -DDETOOLS_CONFIG_COMPRESSION_CRLE=0 \ 64 | -Ic/heatshrink -c c/detools.c -o detools.no-crle.o 65 | $(CC) $(CFLAGS) \ 66 | -DDETOOLS_CONFIG_COMPRESSION_NONE=0 \ 67 | -DDETOOLS_CONFIG_COMPRESSION_CRLE=0 \ 68 | -DDETOOLS_CONFIG_COMPRESSION_HEATSHRINK=0 \ 69 | -Ic/heatshrink -c c/detools.c -o detools.no-crle.o 70 | $(MAKE) -C c library 71 | $(MAKE) -C c/examples/in_place all 72 | $(MAKE) -C c/examples/in_place heatshrink 73 | $(MAKE) -C c/examples/in_place crle 74 | $(MAKE) -C c/examples/dump_restore 75 | $(MAKE) -C c clean 76 | $(MAKE) -C c test 77 | $(MAKE) -C c clean 78 | $(MAKE) -C c CFLAGS_EXTRA="-DHEATSHRINK_DYNAMIC_ALLOC=1" test 79 | 80 | test-c-fuzzer: 81 | clang $(FUZZER_CFLAGS) \ 82 | c/detools.c \ 83 | c/heatshrink/heatshrink_decoder.c \ 84 | tests/fuzzer.c \ 85 | -l lzma -o fuzzer 86 | rm -f fuzzer.profraw 87 | LLVM_PROFILE_FILE="fuzzer.profraw" \ 88 | ./fuzzer \ 89 | -max_total_time=$(FUZZER_EXECUTION_TIME) \ 90 | -print_final_stats 91 | llvm-profdata merge \ 92 | -sparse fuzzer.profraw \ 93 | -o fuzzer.profdata 94 | llvm-cov show ./fuzzer \ 95 | -instr-profile=fuzzer.profdata 96 | 97 | release-to-pypi: 98 | python3 setup.py sdist 99 | twine upload dist/* 100 | 101 | benchmark: 102 | $(MAKE) -C c 103 | python3 setup.py test -s \ 104 | tests.test_detools.DetoolsTest.test_create_and_apply_patch_foo 105 | tests/benchmark.sh 106 | -------------------------------------------------------------------------------- /c/tst/test.mk: -------------------------------------------------------------------------------- 1 | NALA ?= nala 2 | BUILD = build 3 | EXE = $(BUILD)/app 4 | INC += $(BUILD) 5 | INC += $(CURDIR) 6 | INC += $(shell $(NALA) include_dir) 7 | SRC += $(BUILD)/nala_mocks.c 8 | SRC += $(shell $(NALA) c_sources) 9 | SRC += $(TESTS) 10 | # To evaluate once for fewer nala include_dir/c_sources calls. 11 | INC := $(INC) 12 | SRC := $(SRC) 13 | OBJ = $(patsubst %,$(BUILD)%,$(abspath $(SRC:%.c=%.o))) 14 | OBJDEPS = $(OBJ:%.o=%.d) 15 | MOCKGENDEPS = $(BUILD)/nala_mocks.ldflags.d 16 | DEPS = $(OBJDEPS) $(MOCKGENDEPS) 17 | CFLAGS += $(INC:%=-I%) 18 | CFLAGS += -g 19 | CFLAGS += -O0 20 | CFLAGS += -no-pie 21 | CFLAGS += -coverage 22 | CFLAGS += -Wall 23 | CFLAGS += -Wextra 24 | CFLAGS += -Wpedantic 25 | CFLAGS += -Werror 26 | CFLAGS += -Wno-unused-command-line-argument 27 | ifeq ($(SANITIZE), yes) 28 | CFLAGS += -fsanitize=address 29 | CFLAGS += -fsanitize=undefined 30 | endif 31 | CFLAGS += -DNALA_INCLUDE_NALA_MOCKS_H 32 | MOCKGENFLAGS += $(IMPLEMENTATION:%=-i %) 33 | MOCKGENFLAGS += $(NO_IMPLEMENTATION:%=-n %) 34 | REPORT_JSON = $(BUILD)/report.json 35 | EXEARGS += $(ARGS) 36 | EXEARGS += $(JOBS:%=-j %) 37 | EXEARGS += $(REPORT_JSON:%=-r %) 38 | 39 | .PHONY: all build generate clean coverage gdb gdb-run auto auto-run help 40 | 41 | all: build 42 | $(EXE) $(EXEARGS) 43 | 44 | auto: 45 | $(MAKE) || true 46 | while true ; do \ 47 | $(MAKE) auto-run ; \ 48 | done 49 | 50 | auto-run: 51 | for f in $(OBJDEPS) ; do \ 52 | ls -1 $$(cat $$f | sed s/\\\\//g | sed s/.*://g) ; \ 53 | done | sort | uniq | grep -v $(BUILD) | entr -d -p $(MAKE) 54 | 55 | build: generate 56 | $(MAKE) $(EXE) 57 | 58 | generate: $(BUILD)/nala_mocks.ldflags 59 | 60 | clean: 61 | rm -rf $(BUILD) 62 | 63 | coverage: 64 | gcovr $(GCOVR_ARGS) --html-details --output index.html $(BUILD) 65 | mkdir -p $(BUILD)/coverage 66 | mv index.* $(BUILD)/coverage 67 | @echo "Code coverage report: $$(readlink -f $(BUILD)/coverage/index.html)" 68 | 69 | # Recursive make for helpful output. 70 | gdb: 71 | test_file_func=$$($(EXE) --print-test-file-func $(TEST)) && \ 72 | $(MAKE) gdb-run TEST_FILE_FUNC=$$test_file_func 73 | 74 | gdb-run: 75 | gdb \ 76 | -ex "b $(TEST_FILE_FUNC)_before_fork" \ 77 | -ex "r $(TEST)" \ 78 | -ex "set follow-fork-mode child" \ 79 | -ex c \ 80 | $(EXE) 81 | 82 | help: 83 | @echo "TARGET DESCRIPTION" 84 | @echo "---------------------------------------------------------" 85 | @echo "all Build and run with given ARGS." 86 | @echo "auto Build and run with given ARGS on source change." 87 | @echo "clean Remove build output." 88 | @echo "coverage Create the code coverage report." 89 | @echo "gdb Debug given test TEST with gdb." 90 | 91 | $(EXE): $(OBJ) 92 | echo "LD $@" 93 | $(CC) $(CFLAGS) @$(BUILD)/nala_mocks.ldflags $^ $(LIBS:%=-l%) -o $@ 94 | 95 | define COMPILE_template 96 | $(patsubst %.c,$(BUILD)%.o,$(abspath $1)): $1 97 | @echo "CC $1" 98 | mkdir -p $$(@D) 99 | $$(CC) -MMD $$(CFLAGS) -c -o $$@ $$< 100 | $(NALA) wrap_internal_symbols $(BUILD)/nala_mocks.ldflags $$@ 101 | endef 102 | $(foreach file,$(SRC),$(eval $(call COMPILE_template,$(file)))) 103 | 104 | $(BUILD)/nala_mocks.ldflags: $(TESTS) 105 | echo "MOCKGEN $(TESTS)" 106 | mkdir -p $(@D) 107 | [ -f $(BUILD)/nala_mocks.h ] || touch $(BUILD)/nala_mocks.h 108 | cat $(TESTS) \ 109 | | $(CC) $(CFLAGS) -DNALA_GENERATE_MOCKS -x c -E - \ 110 | | $(NALA) generate_mocks $(MOCKGENFLAGS) -o $(BUILD) 111 | cat $(TESTS) \ 112 | | $(CC) -MM -MT $@ $(CFLAGS) -x c -o $@.d - 113 | touch $@ 114 | 115 | -include $(DEPS) 116 | -------------------------------------------------------------------------------- /c/tst/utils.c: -------------------------------------------------------------------------------- 1 | #include "nala.h" 2 | #include "utils.h" 3 | 4 | struct files_t utils_files; 5 | 6 | int utils_from_read(void *arg_p, uint8_t *buf_p, size_t size) 7 | { 8 | (void)arg_p; 9 | 10 | size_t number_of_members_read; 11 | 12 | number_of_members_read = fread(buf_p, size, 1, utils_files.from.file_p); 13 | ASSERT_EQ(number_of_members_read, 1); 14 | 15 | return (0); 16 | } 17 | 18 | int utils_from_seek(void *arg_p, int offset) 19 | { 20 | (void)arg_p; 21 | 22 | ASSERT_EQ(fseek(utils_files.from.file_p, offset, SEEK_CUR), 0); 23 | 24 | return (0); 25 | } 26 | 27 | int utils_to_write(void *arg_p, const uint8_t *buf_p, size_t size) 28 | { 29 | (void)arg_p; 30 | 31 | size_t number_of_members_written; 32 | 33 | number_of_members_written = fwrite(buf_p, size, 1, utils_files.to.file_p); 34 | ASSERT_EQ(number_of_members_written, 1); 35 | utils_files.to.offset += size; 36 | 37 | return (0); 38 | } 39 | 40 | int utils_state_write(void *arg_p, const void *buf_p, size_t size) 41 | { 42 | (void)arg_p; 43 | 44 | size_t number_of_members_written; 45 | 46 | number_of_members_written = fwrite(buf_p, size, 1, utils_files.state.file_p); 47 | ASSERT_EQ(number_of_members_written, 1); 48 | 49 | return (0); 50 | } 51 | 52 | int utils_state_read(void *arg_p, void *buf_p, size_t size) 53 | { 54 | (void)arg_p; 55 | 56 | size_t number_of_members_read; 57 | 58 | number_of_members_read = fread(buf_p, size, 1, utils_files.state.file_p); 59 | ASSERT_EQ(number_of_members_read, 1); 60 | 61 | return (0); 62 | } 63 | 64 | const uint8_t *utils_read_file(const char *filename_p, size_t *size_p) 65 | { 66 | FILE *file_p; 67 | char *buf_p; 68 | 69 | file_p = fopen(filename_p, "rb"); 70 | ASSERT_NE(file_p, NULL); 71 | ASSERT_EQ(fseek(file_p, 0, SEEK_END), 0); 72 | *size_p = ftell(file_p); 73 | ASSERT_GT(*size_p, 0); 74 | buf_p = malloc(*size_p); 75 | ASSERT_NE(buf_p, NULL); 76 | ASSERT_EQ(fseek(file_p, 0, SEEK_SET), 0); 77 | ASSERT_EQ(fread(buf_p, *size_p, 1, file_p), 1); 78 | fclose(file_p); 79 | 80 | return ((const uint8_t *)buf_p); 81 | } 82 | 83 | void utils_files_init(const char *from_filename_p, 84 | const char *patch_filename_p, 85 | const char *expected_new_filename_p) 86 | { 87 | utils_files.from.file_p = fopen(from_filename_p, "rb"); 88 | ASSERT_NE(utils_files.from.file_p, NULL); 89 | utils_files.to.file_p = open_memstream(&utils_files.to.buf_p, 90 | &utils_files.to.size); 91 | ASSERT_NE(utils_files.to.file_p, NULL); 92 | utils_files.to.offset = 0; 93 | utils_files.to.saved_offset = -1; 94 | utils_files.state.file_p = open_memstream(&utils_files.state.buf_p, 95 | &utils_files.state.size); 96 | ASSERT_NE(utils_files.state.file_p, NULL); 97 | utils_files.patch.buf_p = utils_read_file(patch_filename_p, 98 | &utils_files.patch.size); 99 | utils_files.expected_new.buf_p = utils_read_file( 100 | expected_new_filename_p, 101 | &utils_files.expected_new.size); 102 | } 103 | 104 | void utils_files_destroy(void) 105 | { 106 | fclose(utils_files.to.file_p); 107 | free((void *)utils_files.expected_new.buf_p); 108 | fclose(utils_files.from.file_p); 109 | free(utils_files.to.buf_p); 110 | fclose(utils_files.state.file_p); 111 | free(utils_files.state.buf_p); 112 | free((void *)utils_files.patch.buf_p); 113 | } 114 | 115 | void utils_files_assert_and_destroy(void) 116 | { 117 | fflush(utils_files.to.file_p); 118 | 119 | ASSERT_EQ(utils_files.to.size, utils_files.expected_new.size); 120 | ASSERT_MEMORY_EQ(utils_files.to.buf_p, 121 | utils_files.expected_new.buf_p, 122 | utils_files.expected_new.size); 123 | 124 | utils_files_destroy(); 125 | } 126 | 127 | void utils_files_reopen_from(void) 128 | { 129 | ASSERT_EQ(fseek(utils_files.from.file_p, 0, SEEK_SET), 0); 130 | } 131 | -------------------------------------------------------------------------------- /tests/files/READ-ME.rst: -------------------------------------------------------------------------------- 1 | |buildstatus|_ 2 | |coverage|_ 3 | |codecov|_ 4 | 5 | About 6 | ===== 7 | 8 | Binary delta encoding in Python 3, using C extensions. 9 | 10 | Based on http://www.daemonology.net/bsdiff/, with the following 11 | differences: 12 | 13 | - LZMA compression instead of BZ2. 14 | 15 | - Linear patch file access pattern to allow streaming. 16 | 17 | - `SA-IS`_ instead of qsufsort. 18 | 19 | - Variable length size fields. 20 | 21 | Project homepage: https://github.com/eerimoq/detools 22 | 23 | Documentation: http://detools.readthedocs.org/en/latest 24 | 25 | Installation 26 | ============ 27 | 28 | .. code-block:: python 29 | 30 | pip install detools 31 | 32 | Statistics 33 | ========== 34 | 35 | `To compressed` is the size of `To file` compressed using ``lzma 36 | --best``. 37 | 38 | All sizes are in bytes. 39 | 40 | +--------------------+-------------------+-----------+-----------+------------+---------------+ 41 | | From file | To file | From size | To size | Patch size | To compressed | 42 | +====================+===================+===========+===========+============+===============+ 43 | | micropython v1.9.4 | micropython v1.10 | 604872 | 615388 | 71868 | 367500 | 44 | +--------------------+-------------------+-----------+-----------+------------+---------------+ 45 | | python 3.5 | python 3.6 | 4464400 | 4568920 | 1451788 | 1402663 | 46 | +--------------------+-------------------+-----------+-----------+------------+---------------+ 47 | | foo.old | foo.new | 2780 | 2780 | 184 | 1934 | 48 | +--------------------+-------------------+-----------+-----------+------------+---------------+ 49 | 50 | Example usage 51 | ============= 52 | 53 | Command line tool 54 | ----------------- 55 | 56 | The create patch subcommand 57 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ 58 | 59 | Create a patch ``foo.patch`` from ``tests/files/foo.old`` to 60 | ``tests/files/foo.new``. 61 | 62 | .. code-block:: text 63 | 64 | $ detools create_patch tests/files/foo.old tests/files/foo.new foo.patch 65 | $ ls -l foo.patch 66 | -rw-rw-r-- 1 erik erik 184 feb 21 07:28 foo.patch 67 | 68 | The apply patch subcommand 69 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 70 | 71 | Apply the patch ``foo.patch`` to ``tests/files/foo.old`` to create 72 | ``foo.new``. 73 | 74 | .. code-block:: text 75 | 76 | $ detools apply_patch tests/files/foo.old foo.patch foo.new 77 | $ ls -l foo.new 78 | -rw-rw-r-- 1 erik erik 2780 feb 21 07:30 foo.new 79 | 80 | The patch info subcommand 81 | ^^^^^^^^^^^^^^^^^^^^^^^^^ 82 | 83 | Print information about the patch ``foo.patch``. 84 | 85 | .. code-block:: text 86 | 87 | $ detools patch_info foo.patch 88 | Patch size: 184 bytes 89 | To size: 2.78 KB 90 | Patch/to ratio: 6.6 % (lower is better) 91 | Size/data ratio: 0.3 % (lower is better) 92 | 93 | Number of diffs: 2 94 | Total diff size: 2.75 KB 95 | Average diff size: 1.38 KB 96 | Median diff size: 1.38 KB 97 | 98 | Number of extras: 2 99 | Total extra size: 28 bytes 100 | Average extra size: 14 bytes 101 | Median extra size: 14 bytes 102 | 103 | Contributing 104 | ============ 105 | 106 | #. Fork the repository. 107 | 108 | #. Install prerequisites. 109 | 110 | .. code-block:: text 111 | 112 | pip install -r requirements.txt 113 | 114 | #. Implement the new feature or bug fix. 115 | 116 | #. Implement test case(s) to ensure that future changes do not break 117 | legacy. 118 | 119 | #. Run the tests. 120 | 121 | .. code-block:: text 122 | 123 | make test 124 | 125 | #. Create a pull request. 126 | 127 | .. |buildstatus| image:: https://travis-ci.org/eerimoq/detools.svg?branch=master 128 | .. _buildstatus: https://travis-ci.org/eerimoq/detools 129 | 130 | .. |coverage| image:: https://coveralls.io/repos/github/eerimoq/detools/badge.svg?branch=master 131 | .. _coverage: https://coveralls.io/github/eerimoq/detools 132 | 133 | .. |codecov| image:: https://codecov.io/gh/eerimoq/detools/branch/master/graph/badge.svg 134 | .. _codecov: https://codecov.io/gh/eerimoq/detools 135 | 136 | .. _SA-IS: https://sites.google.com/site/yuta256/sais 137 | -------------------------------------------------------------------------------- /c/examples/in_place/main.c: -------------------------------------------------------------------------------- 1 | /** 2 | * BSD 2-Clause License 3 | * 4 | * Copyright (c) 2019, Erik Moqvist 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions 9 | * are met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * 14 | * * Redistributions in binary form must reproduce the above copyright 15 | * notice, this list of conditions and the following disclaimer in 16 | * the documentation and/or other materials provided with the 17 | * distribution. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 22 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 23 | * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 24 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 25 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 28 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 30 | * OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #include 34 | #include "../../detools.h" 35 | 36 | /* Helper functions. */ 37 | static int flash_read(void *arg_p, void *dst_p, uintptr_t src, size_t size) 38 | { 39 | return (0); 40 | } 41 | 42 | static int flash_write(void *arg_p, uintptr_t dst, void *src_p, size_t size) 43 | { 44 | return (0); 45 | } 46 | 47 | static int flash_erase(void *arg_p, uintptr_t addr, size_t size) 48 | { 49 | return (0); 50 | } 51 | 52 | static int step_set(void *arg_p, int step) 53 | { 54 | return (0); 55 | } 56 | 57 | static int step_get(void *arg_p, int *step_p) 58 | { 59 | return (0); 60 | } 61 | 62 | static int serial_read(uint8_t *buf_p, size_t size) 63 | { 64 | return (0); 65 | } 66 | 67 | static int verify_written_data(int to_size, uint32_t to_crc) 68 | { 69 | return (0); 70 | } 71 | 72 | static int update(size_t patch_size, uint32_t to_crc) 73 | { 74 | struct detools_apply_patch_in_place_t apply_patch; 75 | uint8_t buf[256]; 76 | size_t left; 77 | int res; 78 | 79 | /* Initialize the in-place apply patch object. */ 80 | res = detools_apply_patch_in_place_init(&apply_patch, 81 | flash_read, 82 | flash_write, 83 | flash_erase, 84 | step_set, 85 | step_get, 86 | patch_size, 87 | NULL); 88 | 89 | if (res != 0) { 90 | return (res); 91 | } 92 | 93 | left = patch_size; 94 | 95 | /* Incrementally process patch data until the whole patch has been 96 | applied or an error occurrs. */ 97 | while ((left > 0) && (res == 0)) { 98 | res = serial_read(&buf[0], sizeof(buf)); 99 | 100 | if (res > 0) { 101 | left -= res; 102 | res = detools_apply_patch_in_place_process(&apply_patch, 103 | &buf[0], 104 | res); 105 | } 106 | } 107 | 108 | /* Finalize patching and verify written data. */ 109 | if (res == 0) { 110 | res = detools_apply_patch_in_place_finalize(&apply_patch); 111 | 112 | if (res >= 0) { 113 | res = verify_written_data(res, to_crc); 114 | } 115 | } else { 116 | (void)detools_apply_patch_in_place_finalize(&apply_patch); 117 | } 118 | 119 | return (res); 120 | } 121 | 122 | int main(int argc, const char *argv[]) 123 | { 124 | return (update(atoi(argv[1]), atoi(argv[2]))); 125 | } 126 | -------------------------------------------------------------------------------- /c/heatshrink/heatshrink_decoder.h: -------------------------------------------------------------------------------- 1 | #ifndef HEATSHRINK_DECODER_H 2 | #define HEATSHRINK_DECODER_H 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include "heatshrink_common.h" 11 | #include "heatshrink_config.h" 12 | 13 | typedef enum { 14 | HSDR_SINK_OK, /* data sunk, ready to poll */ 15 | HSDR_SINK_FULL, /* out of space in internal buffer */ 16 | HSDR_SINK_ERROR_NULL=-1, /* NULL argument */ 17 | } HSD_sink_res; 18 | 19 | typedef enum { 20 | HSDR_POLL_EMPTY, /* input exhausted */ 21 | HSDR_POLL_MORE, /* more data remaining, call again w/ fresh output buffer */ 22 | HSDR_POLL_ERROR_NULL=-1, /* NULL arguments */ 23 | HSDR_POLL_ERROR_UNKNOWN=-2, 24 | } HSD_poll_res; 25 | 26 | typedef enum { 27 | HSDR_FINISH_DONE, /* output is done */ 28 | HSDR_FINISH_MORE, /* more output remains */ 29 | HSDR_FINISH_ERROR_NULL=-1, /* NULL arguments */ 30 | } HSD_finish_res; 31 | 32 | #if HEATSHRINK_DYNAMIC_ALLOC 33 | #define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(BUF) \ 34 | ((BUF)->input_buffer_size) 35 | #define HEATSHRINK_DECODER_WINDOW_BITS(BUF) \ 36 | ((BUF)->window_sz2) 37 | #define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ 38 | ((BUF)->lookahead_sz2) 39 | #else 40 | #define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_) \ 41 | HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 42 | #define HEATSHRINK_DECODER_WINDOW_BITS(_) \ 43 | (HEATSHRINK_STATIC_WINDOW_BITS) 44 | #define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ 45 | (HEATSHRINK_STATIC_LOOKAHEAD_BITS) 46 | #endif 47 | 48 | typedef struct { 49 | uint16_t input_size; /* bytes in input buffer */ 50 | uint16_t input_index; /* offset to next unprocessed input byte */ 51 | uint16_t output_count; /* how many bytes to output */ 52 | uint16_t output_index; /* index for bytes to output */ 53 | uint16_t head_index; /* head of window buffer */ 54 | uint8_t state; /* current state machine node */ 55 | uint8_t current_byte; /* current byte of input */ 56 | uint8_t bit_index; /* current bit index */ 57 | 58 | #if HEATSHRINK_DYNAMIC_ALLOC 59 | /* Fields that are only used if dynamically allocated. */ 60 | uint8_t window_sz2; /* window buffer bits */ 61 | uint8_t lookahead_sz2; /* lookahead bits */ 62 | uint16_t input_buffer_size; /* input buffer size */ 63 | 64 | /* Input buffer, then expansion window buffer */ 65 | uint8_t buffers[]; 66 | #else 67 | /* Input buffer, then expansion window buffer */ 68 | uint8_t buffers[(1 << HEATSHRINK_DECODER_WINDOW_BITS(_)) 69 | + HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_)]; 70 | #endif 71 | } heatshrink_decoder; 72 | 73 | #if HEATSHRINK_DYNAMIC_ALLOC 74 | /* Allocate a decoder with an input buffer of INPUT_BUFFER_SIZE bytes, 75 | * an expansion buffer size of 2^WINDOW_SZ2, and a lookahead 76 | * size of 2^lookahead_sz2. (The window buffer and lookahead sizes 77 | * must match the settings used when the data was compressed.) 78 | * Returns NULL on error. */ 79 | heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, 80 | uint8_t expansion_buffer_sz2, uint8_t lookahead_sz2); 81 | 82 | /* Free a decoder. */ 83 | void heatshrink_decoder_free(heatshrink_decoder *hsd); 84 | #endif 85 | 86 | /* Reset a decoder. */ 87 | void heatshrink_decoder_reset(heatshrink_decoder *hsd); 88 | 89 | /* Sink at most SIZE bytes from IN_BUF into the decoder. *INPUT_SIZE is set to 90 | * indicate how many bytes were actually sunk (in case a buffer was filled). */ 91 | HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, 92 | uint8_t *in_buf, size_t size, size_t *input_size); 93 | 94 | /* Poll for output from the decoder, copying at most OUT_BUF_SIZE bytes into 95 | * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ 96 | HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, 97 | uint8_t *out_buf, size_t out_buf_size, size_t *output_size); 98 | 99 | /* Notify the dencoder that the input stream is finished. 100 | * If the return value is HSDR_FINISH_MORE, there is still more output, so 101 | * call heatshrink_decoder_poll and repeat. */ 102 | HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd); 103 | 104 | #ifdef __cplusplus 105 | } 106 | #endif 107 | 108 | #endif 109 | -------------------------------------------------------------------------------- /detools/suffix_array.c: -------------------------------------------------------------------------------- 1 | /* 2 | * sais.c for sais-lite 3 | * Copyright (c) 2008-2010 Yuta Mori All Rights Reserved. 4 | * Copyright (c) 2019, Erik Moqvist (Python wrapper). 5 | * 6 | * Permission is hereby granted, free of charge, to any person 7 | * obtaining a copy of this software and associated documentation 8 | * files (the "Software"), to deal in the Software without 9 | * restriction, including without limitation the rights to use, 10 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | * copies of the Software, and to permit persons to whom the 12 | * Software is furnished to do so, subject to the following 13 | * conditions: 14 | * 15 | * The above copyright notice and this permission notice shall be 16 | * included in all copies or substantial portions of the Software. 17 | * 18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 20 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 21 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 22 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 23 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 25 | * OTHER DEALINGS IN THE SOFTWARE. 26 | */ 27 | 28 | #include 29 | #include 30 | #include "sais/sais.h" 31 | #include "libdivsufsort/divsufsort.h" 32 | 33 | typedef int32_t (*create_t)(const uint8_t *buf_p, 34 | int32_t *suffix_array_p, 35 | int32_t length); 36 | 37 | static PyObject *create(PyObject *self_p, 38 | PyObject* args_p, 39 | create_t create_callback) 40 | { 41 | int res; 42 | Py_buffer from_view; 43 | Py_buffer suffix_array_view; 44 | PyObject *from_p; 45 | PyObject *suffix_array_buffer_p; 46 | int32_t *suffix_array_p; 47 | 48 | res = PyArg_ParseTuple(args_p, 49 | "OO", 50 | &from_p, 51 | &suffix_array_buffer_p); 52 | 53 | if (res == 0) { 54 | return (NULL); 55 | } 56 | 57 | /* Input argument conversion. */ 58 | res = PyObject_GetBuffer(from_p, &from_view, PyBUF_CONTIG_RO); 59 | 60 | if (res == -1) { 61 | return (NULL); 62 | } 63 | 64 | res = PyObject_GetBuffer(suffix_array_buffer_p, 65 | &suffix_array_view, 66 | PyBUF_CONTIG); 67 | 68 | if (res == -1) { 69 | goto err1; 70 | } 71 | 72 | suffix_array_p = (int32_t *)suffix_array_view.buf; 73 | suffix_array_p[0] = (int32_t)from_view.len; 74 | 75 | /* Execute the SA-IS algorithm. */ 76 | res = create_callback((uint8_t *)from_view.buf, 77 | &suffix_array_p[1], 78 | (int32_t)from_view.len); 79 | 80 | if (res != 0) { 81 | goto err2; 82 | } 83 | 84 | PyBuffer_Release(&from_view); 85 | PyBuffer_Release(&suffix_array_view); 86 | Py_INCREF(Py_None); 87 | 88 | return (Py_None); 89 | 90 | err2: 91 | PyBuffer_Release(&suffix_array_view); 92 | 93 | err1: 94 | PyBuffer_Release(&from_view); 95 | 96 | return (NULL); 97 | } 98 | 99 | /** 100 | * def sais(data) -> suffix array 101 | */ 102 | static PyObject *m_sais(PyObject *self_p, PyObject* args_p) 103 | { 104 | return (create(self_p, args_p, sais)); 105 | } 106 | 107 | /** 108 | * def divsufsort(data) -> suffix array 109 | */ 110 | static PyObject *m_divsufsort(PyObject *self_p, PyObject* args_p) 111 | { 112 | return (create(self_p, args_p, divsufsort)); 113 | } 114 | 115 | static PyMethodDef module_methods[] = { 116 | { "sais", m_sais, METH_VARARGS }, 117 | { "divsufsort", m_divsufsort, METH_VARARGS }, 118 | { NULL } 119 | }; 120 | 121 | static PyModuleDef module = { 122 | PyModuleDef_HEAD_INIT, 123 | .m_name = "suffix_array", 124 | .m_doc = NULL, 125 | .m_size = -1, 126 | .m_methods = module_methods 127 | }; 128 | 129 | PyMODINIT_FUNC PyInit_suffix_array(void) 130 | { 131 | PyObject *m_p; 132 | 133 | /* Module creation. */ 134 | m_p = PyModule_Create(&module); 135 | 136 | if (m_p == NULL) { 137 | return (NULL); 138 | } 139 | 140 | return (m_p); 141 | } 142 | -------------------------------------------------------------------------------- /tests/test_crle.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import detools 4 | from detools.create import CrleCompressor 5 | from detools.apply import CrleDecompressor 6 | 7 | 8 | class DetoolsCrleTest(unittest.TestCase): 9 | 10 | def test_compress(self): 11 | datas = [ 12 | ( [b''], b'\x00\x00'), 13 | ( [b'A'], b'\x00\x01A'), 14 | ( [5 * b'A'], b'\x00\x05AAAAA'), 15 | ( [6 * b'A'], b'\x01\x06A'), 16 | ( [b'ABBCC', b'CBBA'], b'\x00\x09ABBCCCBBA'), 17 | ( [126 * b'A', b'', b'A'], b'\x01\x7fA'), 18 | ( [128 * b'A'], b'\x01\x80\x01A'), 19 | ( [1000 * b'A'], b'\x01\xe8\x07A'), 20 | ( [69999 * b'A', b'A'], b'\x01\xf0\xa2\x04A'), 21 | ([10 * b'A', b'BC', 8 * b'A'], b'\x01\x0aA\x00\x02BC\x01\x08A'), 22 | ( [10 * b'A' + 8 * b'B'], b'\x01\x0aA\x01\x08B') 23 | ] 24 | 25 | for chunks, compressed in datas: 26 | compressor = CrleCompressor() 27 | data = b'' 28 | 29 | for chunk in chunks: 30 | data += compressor.compress(chunk) 31 | 32 | data += compressor.flush() 33 | 34 | self.assertEqual(data, compressed) 35 | 36 | def test_decompress_no_data(self): 37 | compressed = b'\x00\x00' 38 | 39 | decompressor = CrleDecompressor(len(compressed)) 40 | 41 | self.assertEqual(decompressor.needs_input, True) 42 | self.assertEqual(decompressor.decompress(compressed, 1), b'') 43 | self.assertEqual(decompressor.eof, True) 44 | 45 | def test_decompress(self): 46 | datas = [ 47 | ( [b'\x00\x01A'], b'A'), 48 | ( [b'\x00\x07AAAAAAA'], 7 * b'A'), 49 | ( [b'\x01\x08A'], 8 * b'A'), 50 | ( [b'\x00\x09ABBCCCBBA'], b'ABBCCCBBA'), 51 | ( [b'\x01\x7f', b'A'], 127 * b'A'), 52 | ( [b'\x01\x80\x01A'], 128 * b'A'), 53 | ( [b'\x01\xe8\x07A'], 1000 * b'A'), 54 | ( [b'\x01\xf0', b'\xa2\x04A'], 70000 * b'A'), 55 | ([b'\x01\x0aA\x00\x02BC\x01\x08A'], 10 * b'A' + b'BC' + 8 * b'A'), 56 | ( [b'\x01\x0aA\x01\x08B'], 10 * b'A' + 8 * b'B') 57 | ] 58 | 59 | for chunks, decompressed in datas: 60 | decompressor = CrleDecompressor(sum([len(c) for c in chunks])) 61 | 62 | for chunk in chunks: 63 | self.assertEqual(decompressor.needs_input, True) 64 | self.assertEqual(decompressor.eof, False) 65 | decompressor.decompress(chunk, 0) 66 | 67 | self.assertEqual(decompressor.needs_input, False) 68 | 69 | data = b'' 70 | 71 | while not decompressor.eof: 72 | data += decompressor.decompress(b'', 1) 73 | 74 | self.assertEqual(data, decompressed) 75 | 76 | def test_decompress_bad_kind(self): 77 | decompressor = CrleDecompressor(3) 78 | 79 | with self.assertRaises(detools.Error) as cm: 80 | decompressor.decompress(b'\x02\x01A', 1) 81 | 82 | self.assertEqual( 83 | str(cm.exception), 84 | 'Expected kind scattered(0) or repeated(1), but got 2.') 85 | 86 | def test_decompress_at_eof(self): 87 | compressed = b'\x00\x01A' 88 | decompressor = CrleDecompressor(len(compressed)) 89 | 90 | self.assertEqual(decompressor.decompress(compressed, 1), b'A') 91 | self.assertEqual(decompressor.eof, True) 92 | 93 | with self.assertRaises(detools.Error) as cm: 94 | decompressor.decompress(b'6', 1) 95 | 96 | self.assertEqual(str(cm.exception), 'Already at end of stream.') 97 | 98 | with self.assertRaises(detools.Error) as cm: 99 | decompressor.decompress(b'', 1) 100 | 101 | self.assertEqual(str(cm.exception), 'Already at end of stream.') 102 | 103 | def test_decompress_ignore_extra_data(self): 104 | compressed = b'\x00\x01A' 105 | decompressor = CrleDecompressor(len(compressed)) 106 | 107 | self.assertEqual(decompressor.decompress(compressed + b'B', 1), b'A') 108 | self.assertEqual(decompressor.eof, True) 109 | 110 | 111 | if __name__ == '__main__': 112 | unittest.main() 113 | -------------------------------------------------------------------------------- /c/README.rst: -------------------------------------------------------------------------------- 1 | About 2 | ===== 3 | 4 | An implementation of detools in the C programming language. 5 | 6 | Features: 7 | 8 | - Incremental apply of `sequential`_ and `in-place`_ patches. 9 | 10 | - bsdiff algorithm. 11 | 12 | - LZMA, `heatshrink`_ or CRLE compression. 13 | 14 | - Dump (and store) the apply patch state at any time. Restore it 15 | later, possibly after a system reboot or program crash. Only 16 | `heatshrink`_ and CRLE compressions are currently supported. 17 | 18 | Goals: 19 | 20 | - Easy to use. 21 | 22 | - Low RAM usage. 23 | 24 | - Small code size. 25 | 26 | - Portable. 27 | 28 | ToDo: 29 | 30 | - Implement dump and restore for LZMA and/or other compression 31 | algorithms. Requires implementing dump and restore functions in 32 | these libraries, which may not be trivial. 33 | 34 | API Overview 35 | ============ 36 | 37 | See `detools.h`_ for details. 38 | 39 | Using files 40 | ----------- 41 | 42 | .. code-block:: c 43 | 44 | int detools_apply_patch_filenames(const char *from_p, 45 | const char *patch_p, 46 | const char *to_p); 47 | 48 | Using callbacks 49 | --------------- 50 | 51 | .. code-block:: c 52 | 53 | int detools_apply_patch_callbacks(detools_read_t from_read, 54 | detools_seek_t from_seek, 55 | detools_read_t patch_read, 56 | size_t patch_size, 57 | detools_write_t to_write, 58 | void *arg_p); 59 | 60 | Low level 61 | --------- 62 | 63 | .. code-block:: c 64 | 65 | int detools_apply_patch_init(struct detools_apply_patch_t *self_p, 66 | detools_read_t from_read, 67 | detools_seek_t from_seek, 68 | size_t patch_size, 69 | detools_write_t to_write, 70 | void *arg_p); 71 | 72 | int detools_apply_patch_process(struct detools_apply_patch_t *self_p, 73 | const uint8_t *patch_p, 74 | size_t size); 75 | 76 | int detools_apply_patch_finalize(struct detools_apply_patch_t *self_p); 77 | 78 | Configuration 79 | ============= 80 | 81 | Use the build time configuration to customize detools for your 82 | application needs. See ``DETOOLS_CONFIG_*`` defines in `detools.h`_ 83 | for details. 84 | 85 | Examples 86 | ======== 87 | 88 | There are examples in the `examples folder`_. 89 | 90 | Command line utility 91 | ==================== 92 | 93 | Build the command line utility. 94 | 95 | .. code-block:: text 96 | 97 | $ make 98 | 99 | Apply a sequential patch. 100 | 101 | .. code-block:: text 102 | 103 | $ ./detools apply_patch \ 104 | ../tests/files/foo/old ../tests/files/foo/patch foo.new 105 | 106 | Apply an in-place patch. 107 | 108 | .. code-block:: text 109 | 110 | $ cp ../tests/files/foo/old foo.mem 111 | $ ./detools apply_patch_in_place \ 112 | foo.mem ../tests/files/foo/in-place-3000-500.patch 113 | 114 | Code size 115 | ========= 116 | 117 | Build an in-place apply patch application using gcc. The code size 118 | will likely be smaller when cross compiling for an embedded device. 119 | 120 | All functionality enabled. 121 | 122 | .. code-block:: text 123 | 124 | $ make -s -C examples/in_place all 125 | text data bss dec hex filename 126 | 9048 664 8 9720 25f8 in-place 127 | 128 | Only heatshrink decompression. 129 | 130 | .. code-block:: text 131 | 132 | $ make -s -C examples/in_place heatshrink 133 | text data bss dec hex filename 134 | 6582 600 8 7190 1c16 in-place-heatshrink 135 | 136 | Only CRLE decompression. 137 | 138 | .. code-block:: text 139 | 140 | $ make -s -C examples/in_place crle 141 | text data bss dec hex filename 142 | 5954 600 8 6562 19a2 in-place-crle 143 | 144 | .. _heatshrink: https://github.com/atomicobject/heatshrink 145 | 146 | .. _sequential: https://detools.readthedocs.io/en/latest/#id1 147 | 148 | .. _in-place: https://detools.readthedocs.io/en/latest/#id3 149 | 150 | .. _detools.h: https://github.com/eerimoq/detools/blob/master/c/detools.h 151 | 152 | .. _examples folder: https://github.com/eerimoq/detools/tree/master/c/examples 153 | -------------------------------------------------------------------------------- /detools/common.py: -------------------------------------------------------------------------------- 1 | import os 2 | import struct 3 | from io import BytesIO 4 | import bitstruct 5 | from .errors import Error 6 | from .bsdiff import pack_size 7 | 8 | 9 | PATCH_TYPE_SEQUENTIAL = 0 10 | PATCH_TYPE_IN_PLACE = 1 11 | PATCH_TYPE_HDIFFPATCH = 2 12 | 13 | PATCH_TYPES = { 14 | 'sequential': PATCH_TYPE_SEQUENTIAL, 15 | 'in-place': PATCH_TYPE_IN_PLACE, 16 | 'hdiffpatch': PATCH_TYPE_HDIFFPATCH 17 | } 18 | 19 | COMPRESSION_NONE = 0 20 | COMPRESSION_LZMA = 1 21 | COMPRESSION_CRLE = 2 22 | COMPRESSION_BZ2 = 3 23 | COMPRESSION_HEATSHRINK = 4 24 | COMPRESSION_ZSTD = 5 25 | COMPRESSION_LZ4 = 6 26 | 27 | COMPRESSIONS = { 28 | 'none': COMPRESSION_NONE, 29 | 'lzma': COMPRESSION_LZMA, 30 | 'crle': COMPRESSION_CRLE, 31 | 'bz2': COMPRESSION_BZ2, 32 | 'heatshrink': COMPRESSION_HEATSHRINK, 33 | 'zstd': COMPRESSION_ZSTD, 34 | 'lz4': COMPRESSION_LZ4 35 | } 36 | 37 | DATA_FORMAT_ARM_CORTEX_M4 = 0 38 | DATA_FORMAT_AARCH64 = 1 39 | DATA_FORMAT_XTENSA_LX106 = 2 40 | 41 | DATA_FORMATS = { 42 | 'arm-cortex-m4': DATA_FORMAT_ARM_CORTEX_M4, 43 | 'aarch64': DATA_FORMAT_AARCH64, 44 | 'xtensa-lx106': DATA_FORMAT_XTENSA_LX106 45 | } 46 | 47 | 48 | def format_or(items): 49 | items = [str(item) for item in items] 50 | 51 | if len(items) == 1: 52 | return items[0] 53 | else: 54 | return '{} or {}'.format(', '.join(items[:-1]), 55 | items[-1]) 56 | 57 | 58 | def format_bad_compression_string(compression): 59 | return "Expected compression {}, but got {}.".format( 60 | format_or(sorted(COMPRESSIONS)), 61 | compression) 62 | 63 | 64 | def format_bad_compression_number(compression): 65 | items = sorted([(n, '{}({})'.format(s, n)) for s, n in COMPRESSIONS.items()]) 66 | 67 | return "Expected compression {}, but got {}.".format( 68 | format_or([v for _, v in items]), 69 | compression) 70 | 71 | 72 | def format_bad_data_format(data_format): 73 | return 'Expected data format {}, but got {}.'.format( 74 | format_or(sorted(DATA_FORMATS)), 75 | data_format) 76 | 77 | 78 | def format_bad_data_format_number(data_format): 79 | items = sorted([(n, '{}({})'.format(s, n)) for s, n in DATA_FORMATS.items()]) 80 | 81 | return "Expected data format {}, but got {}.".format( 82 | format_or([v for _, v in items]), 83 | data_format) 84 | 85 | 86 | def compression_string_to_number(compression): 87 | try: 88 | return COMPRESSIONS[compression] 89 | except KeyError: 90 | raise Error(format_bad_compression_string(compression)) 91 | 92 | 93 | def data_format_number_to_string(data_format): 94 | for string, number in DATA_FORMATS.items(): 95 | if data_format == number: 96 | return string 97 | 98 | raise Error(format_bad_data_format_number(data_format)) 99 | 100 | 101 | def div_ceil(a, b): 102 | return (a + b - 1) // b 103 | 104 | 105 | def file_size(f): 106 | position = f.tell() 107 | f.seek(0, os.SEEK_END) 108 | size = f.tell() 109 | f.seek(position, os.SEEK_SET) 110 | 111 | return size 112 | 113 | 114 | def file_read(f): 115 | f.seek(0, os.SEEK_SET) 116 | 117 | return f.read() 118 | 119 | 120 | def unpack_size_with_length(fin): 121 | try: 122 | byte = fin.read(1)[0] 123 | except IndexError: 124 | raise Error('Failed to read first size byte.') 125 | 126 | is_signed = (byte & 0x40) 127 | value = (byte & 0x3f) 128 | offset = 6 129 | 130 | while byte & 0x80: 131 | try: 132 | byte = fin.read(1)[0] 133 | except IndexError: 134 | raise Error('Failed to read consecutive size byte.') 135 | 136 | value |= ((byte & 0x7f) << offset) 137 | offset += 7 138 | 139 | if is_signed: 140 | value *= -1 141 | 142 | return value, ((offset - 6) / 7 + 1) 143 | 144 | 145 | def unpack_size(fin): 146 | return unpack_size_with_length(fin)[0] 147 | 148 | 149 | def unpack_size_bytes(data): 150 | return unpack_size(BytesIO(data)) 151 | 152 | 153 | def pack_usize(value): 154 | return pack_size(struct.unpack('>q', struct.pack('>Q', value))[0]) 155 | 156 | 157 | def unpack_usize(fin): 158 | return struct.unpack('>Q', struct.pack('>q', unpack_size(fin)))[0] 159 | 160 | 161 | class DataSegment(object): 162 | 163 | def __init__(self, 164 | from_data_offset_begin, 165 | from_data_offset_end, 166 | from_data_begin, 167 | from_data_end, 168 | from_code_begin, 169 | from_code_end, 170 | to_data_offset_begin, 171 | to_data_offset_end, 172 | to_data_begin, 173 | to_data_end, 174 | to_code_begin, 175 | to_code_end): 176 | self.from_data_offset_begin = from_data_offset_begin 177 | self.from_data_offset_end = from_data_offset_end 178 | self.from_data_begin = from_data_begin 179 | self.from_data_end = from_data_end 180 | self.from_code_begin = from_code_begin 181 | self.from_code_end = from_code_end 182 | self.to_data_offset_begin = to_data_offset_begin 183 | self.to_data_offset_end = to_data_offset_end 184 | self.to_data_begin = to_data_begin 185 | self.to_data_end = to_data_end 186 | self.to_code_begin = to_code_begin 187 | self.to_code_end = to_code_end 188 | 189 | 190 | def unpack_header(data): 191 | return bitstruct.unpack('p1u3u4', data) 192 | 193 | 194 | def peek_header_type(fpatch): 195 | position = fpatch.tell() 196 | header = fpatch.read(1) 197 | fpatch.seek(position, os.SEEK_SET) 198 | 199 | if len(header) != 1: 200 | raise Error('Failed to read the patch header.') 201 | 202 | return unpack_header(header)[0] 203 | -------------------------------------------------------------------------------- /detools/data_format/elf.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from operator import itemgetter 3 | 4 | from elftools.elf.sections import SymbolTableSection 5 | 6 | from ..errors import Error 7 | 8 | 9 | class AddressRange(object): 10 | 11 | def __init__(self, begin=0, end=0, section_index=None): 12 | self.begin = begin 13 | self.end = end 14 | self.section_index = section_index 15 | 16 | @property 17 | def size(self): 18 | return self.end - self.begin 19 | 20 | def __str__(self): 21 | return 'Range: 0x{:08x}-0x{:08x} ({}) ({})'.format( 22 | self.begin, 23 | self.end, 24 | self.size, 25 | self.section_index) 26 | 27 | 28 | def find_data_and_code_addresses_per_section(elffile): 29 | """Returns data and code address ranges found in given ELF file by 30 | examining the symbols table. 31 | 32 | """ 33 | 34 | # Find all function and object symbols. 35 | symbols_per_section = defaultdict(list) 36 | 37 | for section in elffile.iter_sections(): 38 | if not isinstance(section, SymbolTableSection): 39 | continue 40 | 41 | if section['sh_entsize'] == 0: 42 | continue 43 | 44 | for symbol in section.iter_symbols(): 45 | if symbol['st_info']['type'] not in ['STT_FUNC', 'STT_OBJECT']: 46 | continue 47 | 48 | if symbol['st_size'] == 0: 49 | continue 50 | 51 | section_index = find_section_index_for_symbol(elffile, symbol) 52 | symbols_per_section[section_index].append(symbol) 53 | 54 | # Find all consecutive function and object ranges per section. 55 | for symbols in symbols_per_section.values(): 56 | symbols.sort(key=itemgetter('st_value')) 57 | 58 | func_ranges_per_section = defaultdict(list) 59 | obj_ranges_per_section = defaultdict(list) 60 | 61 | for section_index, symbols in symbols_per_section.items(): 62 | symbol = symbols[0] 63 | block_type = symbol['st_info']['type'] 64 | block_begin = symbol['st_value'] 65 | block_end = symbol['st_size'] 66 | 67 | for symbol in symbols: 68 | if symbol['st_info']['type'] != block_type: 69 | address_range = AddressRange(block_begin, 70 | block_end, 71 | section_index) 72 | 73 | if block_type == 'STT_FUNC': 74 | func_ranges_per_section[section_index].append(address_range) 75 | else: 76 | obj_ranges_per_section[section_index].append(address_range) 77 | 78 | block_type = symbol['st_info']['type'] 79 | block_begin = symbol['st_value'] 80 | 81 | block_end = symbol['st_value'] + symbol['st_size'] 82 | 83 | address_range = AddressRange(block_begin, 84 | block_end, 85 | section_index) 86 | 87 | if block_type == 'STT_FUNC': 88 | func_ranges_per_section[section_index].append(address_range) 89 | else: 90 | obj_ranges_per_section[section_index].append(address_range) 91 | 92 | return func_ranges_per_section, obj_ranges_per_section 93 | 94 | 95 | def find_section_index_for_symbol(elffile, symbol): 96 | address = symbol['st_value'] 97 | 98 | for index, section in enumerate(elffile.iter_sections()): 99 | begin = section['sh_addr'] 100 | size = section['sh_size'] 101 | 102 | if begin <= address < begin + size: 103 | return index 104 | 105 | raise Error("Symbol '{}' not part of any section.".format(symbol.name)) 106 | 107 | 108 | def create_code_range(code_ranges_per_section): 109 | largest_range = AddressRange() 110 | 111 | for code_ranges in code_ranges_per_section.values(): 112 | code_range = AddressRange(code_ranges[0].begin, 113 | code_ranges[-1].end, 114 | code_ranges[0].section_index) 115 | 116 | if code_range.size > largest_range.size: 117 | largest_range = code_range 118 | 119 | return largest_range 120 | 121 | 122 | def create_data_range(data_ranges_per_section, code_range): 123 | largest_range = AddressRange() 124 | 125 | for data_ranges in data_ranges_per_section.values(): 126 | for data_range in data_ranges: 127 | if data_range.begin < code_range.begin < data_range.end: 128 | left_data_range = AddressRange(data_range.begin, 129 | code_range.begin, 130 | data_range.section_index) 131 | else: 132 | left_data_range = None 133 | 134 | if data_range.begin < code_range.end < data_range.end: 135 | right_data_range = AddressRange(code_range.end, 136 | data_range.end, 137 | data_range.section_index) 138 | else: 139 | right_data_range = None 140 | 141 | if left_data_range is not None and right_data_range is not None: 142 | if left_data_range.size > right_data_range.size: 143 | data_range = left_data_range 144 | else: 145 | data_range = right_data_range 146 | elif left_data_range is not None: 147 | data_range = left_data_range 148 | elif right_data_range is not None: 149 | data_range = right_data_range 150 | 151 | if data_range.size > largest_range.size: 152 | largest_range = data_range 153 | 154 | return largest_range 155 | 156 | 157 | def from_file(elffile): 158 | """Returns data format parameters by examining given ELF file. 159 | 160 | """ 161 | 162 | (code_ranges_per_section, 163 | data_ranges_per_section) = find_data_and_code_addresses_per_section( 164 | elffile) 165 | code_range = create_code_range(code_ranges_per_section) 166 | data_range = create_data_range(data_ranges_per_section, code_range) 167 | 168 | return (code_range, data_range) 169 | -------------------------------------------------------------------------------- /detools/info.py: -------------------------------------------------------------------------------- 1 | import os 2 | from .errors import Error 3 | from .apply import read_header_sequential 4 | from .apply import read_header_in_place 5 | from .apply import read_header_hdiffpatch 6 | from .apply import PatchReader 7 | from .common import PATCH_TYPE_SEQUENTIAL 8 | from .common import PATCH_TYPE_IN_PLACE 9 | from .common import PATCH_TYPE_HDIFFPATCH 10 | from .common import file_size 11 | from .common import unpack_size 12 | from .common import unpack_size_with_length 13 | from .common import data_format_number_to_string 14 | from .common import peek_header_type 15 | from .compression.heatshrink import HeatshrinkDecompressor 16 | from .data_format import info as data_format_info 17 | 18 | 19 | def _compression_info(patch_reader): 20 | info = None 21 | 22 | if patch_reader: 23 | decompressor = patch_reader.decompressor 24 | 25 | if isinstance(decompressor, HeatshrinkDecompressor): 26 | info = { 27 | 'window-sz2': decompressor.window_sz2, 28 | 'lookahead-sz2': decompressor.lookahead_sz2 29 | } 30 | 31 | return info 32 | 33 | 34 | def patch_info_sequential_inner(patch_reader, to_size): 35 | to_pos = 0 36 | number_of_size_bytes = 0 37 | diff_sizes = [] 38 | extra_sizes = [] 39 | adjustment_sizes = [] 40 | 41 | while to_pos < to_size: 42 | # Diff data. 43 | size, number_of_bytes = unpack_size_with_length(patch_reader) 44 | 45 | if to_pos + size > to_size: 46 | raise Error("Patch diff data too long.") 47 | 48 | diff_sizes.append(size) 49 | number_of_size_bytes += number_of_bytes 50 | patch_reader.decompress(size) 51 | to_pos += size 52 | 53 | # Extra data. 54 | size, number_of_bytes = unpack_size_with_length(patch_reader) 55 | number_of_size_bytes += number_of_bytes 56 | 57 | if to_pos + size > to_size: 58 | raise Error("Patch extra data too long.") 59 | 60 | extra_sizes.append(size) 61 | patch_reader.decompress(size) 62 | to_pos += size 63 | 64 | # Adjustment. 65 | size, number_of_bytes = unpack_size_with_length(patch_reader) 66 | number_of_size_bytes += number_of_bytes 67 | adjustment_sizes.append(size) 68 | 69 | return (to_size, 70 | diff_sizes, 71 | extra_sizes, 72 | adjustment_sizes, 73 | number_of_size_bytes) 74 | 75 | 76 | def patch_info_sequential(fpatch, fsize): 77 | patch_size = file_size(fpatch) 78 | compression, to_size = read_header_sequential(fpatch) 79 | dfpatch_size = 0 80 | data_format = None 81 | dfpatch_info = None 82 | patch_reader = None 83 | 84 | if to_size == 0: 85 | info = (0, [], [], [], 0) 86 | else: 87 | patch_reader = PatchReader(fpatch, compression) 88 | dfpatch_size = unpack_size(patch_reader) 89 | 90 | if dfpatch_size > 0: 91 | data_format = unpack_size(patch_reader) 92 | patch = patch_reader.decompress(dfpatch_size) 93 | dfpatch_info = data_format_info(data_format, patch, fsize) 94 | data_format = data_format_number_to_string(data_format) 95 | 96 | info = patch_info_sequential_inner(patch_reader, to_size) 97 | 98 | if not patch_reader.eof: 99 | raise Error('End of patch not found.') 100 | 101 | return (patch_size, 102 | compression, 103 | _compression_info(patch_reader), 104 | dfpatch_size, 105 | data_format, 106 | dfpatch_info, 107 | *info) 108 | 109 | 110 | def patch_info_in_place(fpatch): 111 | patch_size = file_size(fpatch) 112 | (compression, 113 | memory_size, 114 | segment_size, 115 | shift_size, 116 | from_size, 117 | to_size) = read_header_in_place(fpatch) 118 | segments = [] 119 | patch_reader = None 120 | 121 | if to_size > 0: 122 | patch_reader = PatchReader(fpatch, compression) 123 | 124 | for to_pos in range(0, to_size, segment_size): 125 | segment_to_size = min(segment_size, to_size - to_pos) 126 | dfpatch_size = unpack_size(patch_reader) 127 | 128 | if dfpatch_size > 0: 129 | data_format = unpack_size(patch_reader) 130 | data_format = data_format_number_to_string(data_format) 131 | patch_reader.decompress(dfpatch_size) 132 | else: 133 | data_format = None 134 | 135 | info = patch_info_sequential_inner(patch_reader, segment_to_size) 136 | segments.append((dfpatch_size, data_format, info)) 137 | 138 | return (patch_size, 139 | compression, 140 | _compression_info(patch_reader), 141 | memory_size, 142 | segment_size, 143 | shift_size, 144 | from_size, 145 | to_size, 146 | segments) 147 | 148 | 149 | def patch_info_hdiffpatch(fpatch): 150 | patch_size = file_size(fpatch) 151 | compression, to_size, _ = read_header_hdiffpatch(fpatch) 152 | patch_reader = None 153 | 154 | if to_size > 0: 155 | patch_reader = PatchReader(fpatch, compression) 156 | 157 | return (patch_size, 158 | compression, 159 | _compression_info(patch_reader), 160 | to_size) 161 | 162 | 163 | def patch_info(fpatch, fsize=None): 164 | """Get patch information from given file-like patch object `fpatch`. 165 | 166 | """ 167 | 168 | if fsize is None: 169 | fsize = str 170 | 171 | patch_type = peek_header_type(fpatch) 172 | 173 | if patch_type == PATCH_TYPE_SEQUENTIAL: 174 | return 'sequential', patch_info_sequential(fpatch, fsize) 175 | elif patch_type == PATCH_TYPE_IN_PLACE: 176 | return 'in-place', patch_info_in_place(fpatch) 177 | elif patch_type == PATCH_TYPE_HDIFFPATCH: 178 | return 'hdiffpatch', patch_info_hdiffpatch(fpatch) 179 | else: 180 | raise Error('Bad patch type {}.'.format(patch_type)) 181 | 182 | 183 | def patch_info_filename(patchfile, fsize=None): 184 | """Same as :func:`~detools.patch_info()`, but with a filename instead 185 | of a file-like object. 186 | 187 | """ 188 | 189 | with open(patchfile, 'rb') as fpatch: 190 | return patch_info(fpatch, fsize) 191 | -------------------------------------------------------------------------------- /detools/libdivsufsort/divsufsort_private.h: -------------------------------------------------------------------------------- 1 | /* 2 | * divsufsort_private.h for libdivsufsort 3 | * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person 6 | * obtaining a copy of this software and associated documentation 7 | * files (the "Software"), to deal in the Software without 8 | * restriction, including without limitation the rights to use, 9 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the 11 | * Software is furnished to do so, subject to the following 12 | * conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be 15 | * included in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 19 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 21 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | * OTHER DEALINGS IN THE SOFTWARE. 25 | */ 26 | 27 | #ifndef _DIVSUFSORT_PRIVATE_H 28 | #define _DIVSUFSORT_PRIVATE_H 1 29 | 30 | #ifdef __cplusplus 31 | extern "C" { 32 | #endif /* __cplusplus */ 33 | 34 | #if HAVE_CONFIG_H 35 | # include "config.h" 36 | #endif 37 | #include 38 | #include 39 | #if HAVE_STRING_H 40 | # include 41 | #endif 42 | #if HAVE_STDLIB_H 43 | # include 44 | #endif 45 | #if HAVE_MEMORY_H 46 | # include 47 | #endif 48 | #if HAVE_STDDEF_H 49 | # include 50 | #endif 51 | #if HAVE_STRINGS_H 52 | # include 53 | #endif 54 | #if HAVE_INTTYPES_H 55 | # include 56 | #else 57 | # if HAVE_STDINT_H 58 | # include 59 | # endif 60 | #endif 61 | #if defined(BUILD_DIVSUFSORT64) 62 | # include "divsufsort64.h" 63 | # ifndef SAIDX_T 64 | # define SAIDX_T 65 | # define saidx_t saidx64_t 66 | # endif /* SAIDX_T */ 67 | # ifndef PRIdSAIDX_T 68 | # define PRIdSAIDX_T PRIdSAIDX64_T 69 | # endif /* PRIdSAIDX_T */ 70 | # define divsufsort divsufsort64 71 | # define divbwt divbwt64 72 | # define divsufsort_version divsufsort64_version 73 | # define bw_transform bw_transform64 74 | # define inverse_bw_transform inverse_bw_transform64 75 | # define sufcheck sufcheck64 76 | # define sa_search sa_search64 77 | # define sa_simplesearch sa_simplesearch64 78 | # define sssort sssort64 79 | # define trsort trsort64 80 | #else 81 | # include "divsufsort.h" 82 | #endif 83 | 84 | 85 | /*- Constants -*/ 86 | #if !defined(UINT8_MAX) 87 | # define UINT8_MAX (255) 88 | #endif /* UINT8_MAX */ 89 | #if defined(ALPHABET_SIZE) && (ALPHABET_SIZE < 1) 90 | # undef ALPHABET_SIZE 91 | #endif 92 | #if !defined(ALPHABET_SIZE) 93 | # define ALPHABET_SIZE (UINT8_MAX + 1) 94 | #endif 95 | /* for divsufsort.c */ 96 | #define BUCKET_A_SIZE (ALPHABET_SIZE) 97 | #define BUCKET_B_SIZE (ALPHABET_SIZE * ALPHABET_SIZE) 98 | /* for sssort.c */ 99 | #if defined(SS_INSERTIONSORT_THRESHOLD) 100 | # if SS_INSERTIONSORT_THRESHOLD < 1 101 | # undef SS_INSERTIONSORT_THRESHOLD 102 | # define SS_INSERTIONSORT_THRESHOLD (1) 103 | # endif 104 | #else 105 | # define SS_INSERTIONSORT_THRESHOLD (8) 106 | #endif 107 | #if defined(SS_BLOCKSIZE) 108 | # if SS_BLOCKSIZE < 0 109 | # undef SS_BLOCKSIZE 110 | # define SS_BLOCKSIZE (0) 111 | # elif 32768 <= SS_BLOCKSIZE 112 | # undef SS_BLOCKSIZE 113 | # define SS_BLOCKSIZE (32767) 114 | # endif 115 | #else 116 | # define SS_BLOCKSIZE (1024) 117 | #endif 118 | /* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */ 119 | #if SS_BLOCKSIZE == 0 120 | # if defined(BUILD_DIVSUFSORT64) 121 | # define SS_MISORT_STACKSIZE (96) 122 | # else 123 | # define SS_MISORT_STACKSIZE (64) 124 | # endif 125 | #elif SS_BLOCKSIZE <= 4096 126 | # define SS_MISORT_STACKSIZE (16) 127 | #else 128 | # define SS_MISORT_STACKSIZE (24) 129 | #endif 130 | #if defined(BUILD_DIVSUFSORT64) 131 | # define SS_SMERGE_STACKSIZE (64) 132 | #else 133 | # define SS_SMERGE_STACKSIZE (32) 134 | #endif 135 | /* for trsort.c */ 136 | #define TR_INSERTIONSORT_THRESHOLD (8) 137 | #if defined(BUILD_DIVSUFSORT64) 138 | # define TR_STACKSIZE (96) 139 | #else 140 | # define TR_STACKSIZE (64) 141 | #endif 142 | 143 | 144 | /*- Macros -*/ 145 | #ifndef SWAP 146 | # define SWAP(_a, _b) do { t = (_a); (_a) = (_b); (_b) = t; } while(0) 147 | #endif /* SWAP */ 148 | #ifndef MIN 149 | # define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b)) 150 | #endif /* MIN */ 151 | #ifndef MAX 152 | # define MAX(_a, _b) (((_a) > (_b)) ? (_a) : (_b)) 153 | #endif /* MAX */ 154 | #define STACK_PUSH(_a, _b, _c, _d)\ 155 | do {\ 156 | assert(ssize < STACK_SIZE);\ 157 | stack[ssize].a = (_a), stack[ssize].b = (_b),\ 158 | stack[ssize].c = (_c), stack[ssize++].d = (_d);\ 159 | } while(0) 160 | #define STACK_PUSH5(_a, _b, _c, _d, _e)\ 161 | do {\ 162 | assert(ssize < STACK_SIZE);\ 163 | stack[ssize].a = (_a), stack[ssize].b = (_b),\ 164 | stack[ssize].c = (_c), stack[ssize].d = (_d), stack[ssize++].e = (_e);\ 165 | } while(0) 166 | #define STACK_POP(_a, _b, _c, _d)\ 167 | do {\ 168 | assert(0 <= ssize);\ 169 | if(ssize == 0) { return; }\ 170 | (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ 171 | (_c) = stack[ssize].c, (_d) = stack[ssize].d;\ 172 | } while(0) 173 | #define STACK_POP5(_a, _b, _c, _d, _e)\ 174 | do {\ 175 | assert(0 <= ssize);\ 176 | if(ssize == 0) { return; }\ 177 | (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ 178 | (_c) = stack[ssize].c, (_d) = stack[ssize].d, (_e) = stack[ssize].e;\ 179 | } while(0) 180 | /* for divsufsort.c */ 181 | #define BUCKET_A(_c0) bucket_A[(_c0)] 182 | #if ALPHABET_SIZE == 256 183 | #define BUCKET_B(_c0, _c1) (bucket_B[((_c1) << 8) | (_c0)]) 184 | #define BUCKET_BSTAR(_c0, _c1) (bucket_B[((_c0) << 8) | (_c1)]) 185 | #else 186 | #define BUCKET_B(_c0, _c1) (bucket_B[(_c1) * ALPHABET_SIZE + (_c0)]) 187 | #define BUCKET_BSTAR(_c0, _c1) (bucket_B[(_c0) * ALPHABET_SIZE + (_c1)]) 188 | #endif 189 | 190 | 191 | /*- Private Prototypes -*/ 192 | /* sssort.c */ 193 | void 194 | sssort(const sauchar_t *Td, const saidx_t *PA, 195 | saidx_t *first, saidx_t *last, 196 | saidx_t *buf, saidx_t bufsize, 197 | saidx_t depth, saidx_t n, saint_t lastsuffix); 198 | /* trsort.c */ 199 | void 200 | trsort(saidx_t *ISA, saidx_t *SA, saidx_t n, saidx_t depth); 201 | 202 | 203 | #ifdef __cplusplus 204 | } /* extern "C" */ 205 | #endif /* __cplusplus */ 206 | 207 | #endif /* _DIVSUFSORT_PRIVATE_H */ 208 | -------------------------------------------------------------------------------- /detools/libdivsufsort/divsufsort.h: -------------------------------------------------------------------------------- 1 | /* 2 | * divsufsort.h for libdivsufsort 3 | * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person 6 | * obtaining a copy of this software and associated documentation 7 | * files (the "Software"), to deal in the Software without 8 | * restriction, including without limitation the rights to use, 9 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the 11 | * Software is furnished to do so, subject to the following 12 | * conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be 15 | * included in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 19 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 21 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | * OTHER DEALINGS IN THE SOFTWARE. 25 | */ 26 | 27 | #ifndef _DIVSUFSORT_H 28 | #define _DIVSUFSORT_H 1 29 | 30 | #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 31 | # include //for uint8_t,int32_t 32 | #else 33 | # if (_MSC_VER >= 1300) 34 | typedef unsigned __int8 uint8_t; 35 | typedef signed __int32 int32_t; 36 | # else 37 | typedef unsigned char uint8_t; 38 | typedef signed int int32_t; 39 | # endif 40 | #endif 41 | 42 | #ifdef __cplusplus 43 | extern "C" { 44 | #endif /* __cplusplus */ 45 | 46 | #ifndef PRId32 47 | # define PRId32 "d" 48 | #endif 49 | 50 | #ifndef DIVSUFSORT_API 51 | # ifdef DIVSUFSORT_BUILD_DLL 52 | # define DIVSUFSORT_API 53 | # else 54 | # define DIVSUFSORT_API 55 | # endif 56 | #endif 57 | 58 | /*- Datatypes -*/ 59 | #ifndef SAUCHAR_T 60 | #define SAUCHAR_T 61 | typedef uint8_t sauchar_t; 62 | #endif /* SAUCHAR_T */ 63 | #ifndef SAINT_T 64 | #define SAINT_T 65 | typedef int32_t saint_t; 66 | #endif /* SAINT_T */ 67 | #ifndef SAIDX_T 68 | #define SAIDX_T 69 | typedef int32_t saidx_t; 70 | #endif /* SAIDX_T */ 71 | #ifndef PRIdSAINT_T 72 | #define PRIdSAINT_T PRId32 73 | #endif /* PRIdSAINT_T */ 74 | #ifndef PRIdSAIDX_T 75 | #define PRIdSAIDX_T PRId32 76 | #endif /* PRIdSAIDX_T */ 77 | 78 | 79 | /*- Prototypes -*/ 80 | 81 | /** 82 | * Constructs the suffix array of a given string. 83 | * @param T[0..n-1] The input string. 84 | * @param SA[0..n-1] The output array of suffixes. 85 | * @param n The length of the given string. 86 | * @return 0 if no error occurred, -1 or -2 otherwise. 87 | */ 88 | DIVSUFSORT_API 89 | saint_t 90 | divsufsort(const sauchar_t *T, saidx_t *SA, saidx_t n); 91 | 92 | /** 93 | * Constructs the burrows-wheeler transformed string of a given string. 94 | * @param T[0..n-1] The input string. 95 | * @param U[0..n-1] The output string. (can be T) 96 | * @param A[0..n-1] The temporary array. (can be NULL) 97 | * @param n The length of the given string. 98 | * @return The primary index if no error occurred, -1 or -2 otherwise. 99 | */ 100 | DIVSUFSORT_API 101 | saidx_t 102 | divbwt(const sauchar_t *T, sauchar_t *U, saidx_t *A, saidx_t n); 103 | 104 | /** 105 | * Returns the version of the divsufsort library. 106 | * @return The version number string. 107 | */ 108 | DIVSUFSORT_API 109 | const char * 110 | divsufsort_version(void); 111 | 112 | 113 | /** 114 | * Constructs the burrows-wheeler transformed string of a given string and suffix array. 115 | * @param T[0..n-1] The input string. 116 | * @param U[0..n-1] The output string. (can be T) 117 | * @param SA[0..n-1] The suffix array. (can be NULL) 118 | * @param n The length of the given string. 119 | * @param idx The output primary index. 120 | * @return 0 if no error occurred, -1 or -2 otherwise. 121 | */ 122 | DIVSUFSORT_API 123 | saint_t 124 | bw_transform(const sauchar_t *T, sauchar_t *U, 125 | saidx_t *SA /* can NULL */, 126 | saidx_t n, saidx_t *idx); 127 | 128 | /** 129 | * Inverse BW-transforms a given BWTed string. 130 | * @param T[0..n-1] The input string. 131 | * @param U[0..n-1] The output string. (can be T) 132 | * @param A[0..n-1] The temporary array. (can be NULL) 133 | * @param n The length of the given string. 134 | * @param idx The primary index. 135 | * @return 0 if no error occurred, -1 or -2 otherwise. 136 | */ 137 | DIVSUFSORT_API 138 | saint_t 139 | inverse_bw_transform(const sauchar_t *T, sauchar_t *U, 140 | saidx_t *A /* can NULL */, 141 | saidx_t n, saidx_t idx); 142 | 143 | /** 144 | * Checks the correctness of a given suffix array. 145 | * @param T[0..n-1] The input string. 146 | * @param SA[0..n-1] The input suffix array. 147 | * @param n The length of the given string. 148 | * @param verbose The verbose mode. 149 | * @return 0 if no error occurred. 150 | */ 151 | DIVSUFSORT_API 152 | saint_t 153 | sufcheck(const sauchar_t *T, const saidx_t *SA, saidx_t n, saint_t verbose); 154 | 155 | /** 156 | * Search for the pattern P in the string T. 157 | * @param T[0..Tsize-1] The input string. 158 | * @param Tsize The length of the given string. 159 | * @param P[0..Psize-1] The input pattern string. 160 | * @param Psize The length of the given pattern string. 161 | * @param SA[0..SAsize-1] The input suffix array. 162 | * @param SAsize The length of the given suffix array. 163 | * @param idx The output index. 164 | * @return The count of matches if no error occurred, -1 otherwise. 165 | */ 166 | DIVSUFSORT_API 167 | saidx_t 168 | sa_search(const sauchar_t *T, saidx_t Tsize, 169 | const sauchar_t *P, saidx_t Psize, 170 | const saidx_t *SA, saidx_t SAsize, 171 | saidx_t *left); 172 | 173 | /** 174 | * Search for the character c in the string T. 175 | * @param T[0..Tsize-1] The input string. 176 | * @param Tsize The length of the given string. 177 | * @param SA[0..SAsize-1] The input suffix array. 178 | * @param SAsize The length of the given suffix array. 179 | * @param c The input character. 180 | * @param idx The output index. 181 | * @return The count of matches if no error occurred, -1 otherwise. 182 | */ 183 | DIVSUFSORT_API 184 | saidx_t 185 | sa_simplesearch(const sauchar_t *T, saidx_t Tsize, 186 | const saidx_t *SA, saidx_t SAsize, 187 | saint_t c, saidx_t *left); 188 | 189 | 190 | #ifdef __cplusplus 191 | } /* extern "C" */ 192 | #endif /* __cplusplus */ 193 | 194 | #endif /* _DIVSUFSORT_H */ 195 | -------------------------------------------------------------------------------- /c/tst/test_fuzzer.c: -------------------------------------------------------------------------------- 1 | /* #include */ 2 | #include 3 | #include 4 | #include 5 | #include "nala.h" 6 | #include "../detools.h" 7 | 8 | static const uint8_t from[256] = { 0, }; 9 | 10 | static void create_from_file(void) 11 | { 12 | FILE *file_p; 13 | 14 | file_p = fopen("from", "wb"); 15 | ASSERT_NE(file_p, NULL); 16 | ASSERT_EQ(fwrite(&from[0], sizeof(from), 1, file_p), 1); 17 | fclose(file_p); 18 | } 19 | 20 | static bool is_file(const char *path_p) 21 | { 22 | struct stat statbuf; 23 | 24 | ASSERT_EQ(stat(path_p, &statbuf), 0); 25 | 26 | return (S_ISREG(statbuf.st_mode)); 27 | } 28 | 29 | static void create_patch_file(const char *corpus_file_path_p) 30 | { 31 | FILE *file_p; 32 | uint8_t size; 33 | uint8_t patch[256]; 34 | 35 | /* Read corpus file. */ 36 | file_p = fopen(corpus_file_path_p, "rb"); 37 | ASSERT_NE(file_p, NULL); 38 | ASSERT_EQ(fread(&size, 1, 1, file_p), 1); 39 | 40 | if (size > 0) { 41 | ASSERT_EQ(fread(&patch[0], size, 1, file_p), 1); 42 | /* dbgh(&patch[0], size); */ 43 | } 44 | 45 | fclose(file_p); 46 | 47 | /* Create patch file. */ 48 | file_p = fopen("patch", "wb"); 49 | ASSERT_NE(file_p, NULL); 50 | 51 | if (size > 0) { 52 | ASSERT_EQ(fwrite(&patch[0], size, 1, file_p), 1); 53 | } 54 | 55 | fclose(file_p); 56 | } 57 | 58 | TEST(all_ok) 59 | { 60 | DIR *dir_p; 61 | struct dirent *item_p; 62 | char corpus_file_path[512]; 63 | int res; 64 | 65 | dir_p = opendir("corpus"); 66 | 67 | if (dir_p != NULL) { 68 | create_from_file(); 69 | 70 | while ((item_p = readdir(dir_p)) != NULL) { 71 | sprintf(&corpus_file_path[0], "corpus/%s", item_p->d_name); 72 | 73 | if (!is_file(&corpus_file_path[0])) { 74 | continue; 75 | } 76 | 77 | printf("\n"); 78 | printf("Corpus file: '%s'\n", &corpus_file_path[0]); 79 | create_patch_file(&corpus_file_path[0]); 80 | res = detools_apply_patch_filenames("from", "patch", "to"); 81 | printf("Patch result: %s (%d)\n", detools_error_as_string(res), res); 82 | } 83 | 84 | closedir(dir_p); 85 | } 86 | } 87 | 88 | static const uint8_t from_buf[256] = { 0, }; 89 | static int from_offset = 0; 90 | 91 | static int from_read(void *arg_p, uint8_t *buf_p, size_t size) 92 | { 93 | (void)arg_p; 94 | 95 | if (((size_t)from_offset + size) > 256) { 96 | return (-1); 97 | } 98 | 99 | memcpy(buf_p, &from_buf[from_offset], size); 100 | from_offset += size; 101 | 102 | return (0); 103 | } 104 | 105 | static int from_seek(void *arg_p, int offset) 106 | { 107 | (void)arg_p; 108 | 109 | if ((from_offset + offset) < 0) { 110 | return (-1); 111 | } 112 | 113 | from_offset += offset; 114 | 115 | return (0); 116 | } 117 | 118 | static int to_write(void *arg_p, const uint8_t *buf_p, size_t size) 119 | { 120 | (void)arg_p; 121 | (void)buf_p; 122 | (void)size; 123 | 124 | return (0); 125 | } 126 | 127 | static void test_one(const uint8_t *patch_buf_p, 128 | size_t patch_size, 129 | int process_res, 130 | int finalize_res) 131 | { 132 | struct detools_apply_patch_t apply_patch; 133 | int res; 134 | 135 | res = detools_apply_patch_init(&apply_patch, 136 | from_read, 137 | from_seek, 138 | patch_size, 139 | to_write, 140 | NULL); 141 | ASSERT_EQ(res, 0); 142 | 143 | res = detools_apply_patch_process(&apply_patch, patch_buf_p, patch_size); 144 | 145 | WITH_MESSAGE("Failed with '%s' (%d).", detools_error_as_string(res), res) { 146 | ASSERT_EQ(res, process_res); 147 | } 148 | 149 | res = detools_apply_patch_finalize(&apply_patch); 150 | 151 | WITH_MESSAGE("Failed with '%s' (%d).", detools_error_as_string(res), res) { 152 | ASSERT_EQ(res, finalize_res); 153 | } 154 | } 155 | 156 | TEST(size_overflow) 157 | { 158 | const uint8_t patch[] = { 159 | 0x04, 0x0c, 0x44, 0xfd, 0xff, 0x00, 0x00, 0x00, 0x5d, 0x00, 160 | 0x2b, 0x06, 0x66 161 | }; 162 | 163 | test_one(&patch[0], 164 | sizeof(patch), 165 | -DETOOLS_CORRUPT_PATCH_OVERFLOW, 166 | -DETOOLS_ALREADY_FAILED); 167 | } 168 | 169 | TEST(infinite_loop) 170 | { 171 | const uint8_t patch[] = { 172 | 0x02, 0x3a, 0x01, 0xce, 0xce, 0xce, 0xfe, 0xff, 0x00, 0x00, 173 | 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 174 | }; 175 | 176 | test_one(&patch[0], 177 | sizeof(patch), 178 | -DETOOLS_CORRUPT_PATCH_OVERFLOW, 179 | -DETOOLS_ALREADY_FAILED); 180 | } 181 | 182 | TEST(lzma_decode_failure) 183 | { 184 | const uint8_t patch[] = { 185 | 0x01, 0x5b, 0x00, 0x00, 0x00, 0x00, 0xe2, 0xff, 0x8c, 0x8c, 186 | 0x8c, 0x00, 0x00, 0x00, 0x0f, 0x02 187 | }; 188 | 189 | test_one(&patch[0], 190 | sizeof(patch), 191 | -DETOOLS_LZMA_DECODE, 192 | -DETOOLS_ALREADY_FAILED); 193 | } 194 | 195 | TEST(corrupt_crle_idle_out_of_data) 196 | { 197 | const uint8_t patch[] = { 198 | 0x02, 0x7a 199 | }; 200 | 201 | test_one(&patch[0], sizeof(patch), 0, -DETOOLS_NOT_ENOUGH_PATCH_DATA); 202 | } 203 | 204 | TEST(dfpatch_not_implemented) 205 | { 206 | const uint8_t patch[] = { 207 | 0x02, 0x08, 0x00, 0x40, 0x05, 0xfe 208 | }; 209 | 210 | test_one(&patch[0], 211 | sizeof(patch), 212 | -DETOOLS_NOT_IMPLEMENTED, 213 | -DETOOLS_ALREADY_FAILED); 214 | } 215 | 216 | TEST(corrupt_crle_kind) 217 | { 218 | const uint8_t patch[] = { 219 | 0x02, 0x0a, 0x3d 220 | }; 221 | 222 | test_one(&patch[0], 223 | sizeof(patch), 224 | -DETOOLS_CORRUPT_PATCH_CRLE_KIND, 225 | -DETOOLS_ALREADY_FAILED); 226 | } 227 | 228 | TEST(bad_from_read_error) 229 | { 230 | const uint8_t patch[] = { 231 | 0x04, 0xee, 0xee, 0x18, 0x44, 0x00, 0x00, 0x00, 0x04, 0x00, 232 | 0x00, 0xce, 0xc1, 0x27, 0x28, 0x09, 0xcf, 0xee, 0xce, 0xc1, 233 | 0x27, 0x28, 0x09, 0xcf 234 | }; 235 | 236 | test_one(&patch[0], 237 | sizeof(patch), 238 | -DETOOLS_IO_FAILED, 239 | -DETOOLS_ALREADY_FAILED); 240 | } 241 | 242 | TEST(size_overflow_header) 243 | { 244 | const uint8_t patch[] = { 245 | 0x04, 0xfc, 0xf7, 0xfe, 0xfb, 0x04 246 | }; 247 | 248 | test_one(&patch[0], 249 | sizeof(patch), 250 | -DETOOLS_CORRUPT_PATCH_OVERFLOW, 251 | -DETOOLS_ALREADY_FAILED); 252 | } 253 | -------------------------------------------------------------------------------- /detools/libdivsufsort/divsufsort64.h: -------------------------------------------------------------------------------- 1 | /* 2 | * divsufsort64.h for libdivsufsort64 3 | * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person 6 | * obtaining a copy of this software and associated documentation 7 | * files (the "Software"), to deal in the Software without 8 | * restriction, including without limitation the rights to use, 9 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the 11 | * Software is furnished to do so, subject to the following 12 | * conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be 15 | * included in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 19 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 21 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | * OTHER DEALINGS IN THE SOFTWARE. 25 | */ 26 | 27 | #ifndef _DIVSUFSORT64_H 28 | #define _DIVSUFSORT64_H 1 29 | 30 | #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 31 | # include //for uint8_t,int32_t 32 | #else 33 | # if (_MSC_VER >= 1300) 34 | typedef unsigned __int8 uint8_t; 35 | typedef signed __int32 int32_t; 36 | # else 37 | typedef unsigned char uint8_t; 38 | typedef signed int int32_t; 39 | # endif 40 | #endif 41 | 42 | #ifdef __cplusplus 43 | extern "C" { 44 | #endif /* __cplusplus */ 45 | #ifndef PRId32 46 | # define PRId32 "d" 47 | #endif 48 | #ifdef _MSC_VER 49 | typedef signed __int64 llong_t; 50 | # ifndef PRId64 51 | # define PRId64 "I64d" 52 | # endif 53 | #else 54 | typedef signed long long llong_t; 55 | # ifndef PRId64 56 | # define PRId64 "lld" 57 | # endif 58 | #endif 59 | 60 | #ifndef DIVSUFSORT_API 61 | # ifdef DIVSUFSORT_BUILD_DLL 62 | # define DIVSUFSORT_API 63 | # else 64 | # define DIVSUFSORT_API 65 | # endif 66 | #endif 67 | 68 | /*- Datatypes -*/ 69 | #ifndef SAUCHAR_T 70 | #define SAUCHAR_T 71 | typedef uint8_t sauchar_t; 72 | #endif /* SAUCHAR_T */ 73 | #ifndef SAINT_T 74 | #define SAINT_T 75 | typedef int32_t saint_t; 76 | #endif /* SAINT_T */ 77 | #ifndef SAIDX64_T 78 | #define SAIDX64_T 79 | typedef llong_t saidx64_t; 80 | #endif /* SAIDX64_T */ 81 | #ifndef PRIdSAINT_T 82 | #define PRIdSAINT_T PRId32 83 | #endif /* PRIdSAINT_T */ 84 | #ifndef PRIdSAIDX64_T 85 | #define PRIdSAIDX64_T PRId64 86 | #endif /* PRIdSAIDX64_T */ 87 | 88 | 89 | /*- Prototypes -*/ 90 | 91 | /** 92 | * Constructs the suffix array of a given string. 93 | * @param T[0..n-1] The input string. 94 | * @param SA[0..n-1] The output array of suffixes. 95 | * @param n The length of the given string. 96 | * @return 0 if no error occurred, -1 or -2 otherwise. 97 | */ 98 | DIVSUFSORT_API 99 | saint_t 100 | divsufsort64(const sauchar_t *T, saidx64_t *SA, saidx64_t n); 101 | 102 | /** 103 | * Constructs the burrows-wheeler transformed string of a given string. 104 | * @param T[0..n-1] The input string. 105 | * @param U[0..n-1] The output string. (can be T) 106 | * @param A[0..n-1] The temporary array. (can be NULL) 107 | * @param n The length of the given string. 108 | * @return The primary index if no error occurred, -1 or -2 otherwise. 109 | */ 110 | DIVSUFSORT_API 111 | saidx64_t 112 | divbwt64(const sauchar_t *T, sauchar_t *U, saidx64_t *A, saidx64_t n); 113 | 114 | /** 115 | * Returns the version of the divsufsort library. 116 | * @return The version number string. 117 | */ 118 | DIVSUFSORT_API 119 | const char * 120 | divsufsort64_version(void); 121 | 122 | 123 | /** 124 | * Constructs the burrows-wheeler transformed string of a given string and suffix array. 125 | * @param T[0..n-1] The input string. 126 | * @param U[0..n-1] The output string. (can be T) 127 | * @param SA[0..n-1] The suffix array. (can be NULL) 128 | * @param n The length of the given string. 129 | * @param idx The output primary index. 130 | * @return 0 if no error occurred, -1 or -2 otherwise. 131 | */ 132 | DIVSUFSORT_API 133 | saint_t 134 | bw_transform64(const sauchar_t *T, sauchar_t *U, 135 | saidx64_t *SA /* can NULL */, 136 | saidx64_t n, saidx64_t *idx); 137 | 138 | /** 139 | * Inverse BW-transforms a given BWTed string. 140 | * @param T[0..n-1] The input string. 141 | * @param U[0..n-1] The output string. (can be T) 142 | * @param A[0..n-1] The temporary array. (can be NULL) 143 | * @param n The length of the given string. 144 | * @param idx The primary index. 145 | * @return 0 if no error occurred, -1 or -2 otherwise. 146 | */ 147 | DIVSUFSORT_API 148 | saint_t 149 | inverse_bw_transform64(const sauchar_t *T, sauchar_t *U, 150 | saidx64_t *A /* can NULL */, 151 | saidx64_t n, saidx64_t idx); 152 | 153 | /** 154 | * Checks the correctness of a given suffix array. 155 | * @param T[0..n-1] The input string. 156 | * @param SA[0..n-1] The input suffix array. 157 | * @param n The length of the given string. 158 | * @param verbose The verbose mode. 159 | * @return 0 if no error occurred. 160 | */ 161 | DIVSUFSORT_API 162 | saint_t 163 | sufcheck64(const sauchar_t *T, const saidx64_t *SA, saidx64_t n, saint_t verbose); 164 | 165 | /** 166 | * Search for the pattern P in the string T. 167 | * @param T[0..Tsize-1] The input string. 168 | * @param Tsize The length of the given string. 169 | * @param P[0..Psize-1] The input pattern string. 170 | * @param Psize The length of the given pattern string. 171 | * @param SA[0..SAsize-1] The input suffix array. 172 | * @param SAsize The length of the given suffix array. 173 | * @param idx The output index. 174 | * @return The count of matches if no error occurred, -1 otherwise. 175 | */ 176 | DIVSUFSORT_API 177 | saidx64_t 178 | sa_search64(const sauchar_t *T, saidx64_t Tsize, 179 | const sauchar_t *P, saidx64_t Psize, 180 | const saidx64_t *SA, saidx64_t SAsize, 181 | saidx64_t *left); 182 | 183 | /** 184 | * Search for the character c in the string T. 185 | * @param T[0..Tsize-1] The input string. 186 | * @param Tsize The length of the given string. 187 | * @param SA[0..SAsize-1] The input suffix array. 188 | * @param SAsize The length of the given suffix array. 189 | * @param c The input character. 190 | * @param idx The output index. 191 | * @return The count of matches if no error occurred, -1 otherwise. 192 | */ 193 | DIVSUFSORT_API 194 | saidx64_t 195 | sa_simplesearch64(const sauchar_t *T, saidx64_t Tsize, 196 | const saidx64_t *SA, saidx64_t SAsize, 197 | saint_t c, saidx64_t *left); 198 | 199 | 200 | #ifdef __cplusplus 201 | } /* extern "C" */ 202 | #endif /* __cplusplus */ 203 | 204 | #endif /* _DIVSUFSORT64_H */ 205 | -------------------------------------------------------------------------------- /tests/files/bsdiff.py: -------------------------------------------------------------------------------- 1 | # 2 | # Based on the implementation in bsdiff.c. 3 | # 4 | # Copyright 2003-2005 Colin Percival 5 | # Copyright (c) 2019, Erik Moqvist 6 | # All rights reserved 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted providing that the following conditions 10 | # are met: 11 | # 1. Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # 2. Redistributions in binary form must reproduce the above copyright 14 | # notice, this list of conditions and the following disclaimer in the 15 | # documentation and/or other materials provided with the distribution. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 | # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 25 | # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 26 | # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | # POSSIBILITY OF SUCH DAMAGE. 28 | # 29 | 30 | def matchlen(from_data, to_data): 31 | length = min(len(from_data), len(to_data)) 32 | 33 | for i in range(length): 34 | if from_data[i] != to_data[i]: 35 | return i 36 | 37 | return length 38 | 39 | 40 | def memcmp(b1, b2): 41 | for a, b in zip(b1, b2): 42 | if a > b: 43 | return 1 44 | elif a < b: 45 | return -1 46 | 47 | return 0 48 | 49 | 50 | def search(suffix_array, from_data, to_data, st, en): 51 | if en - st < 2: 52 | x = matchlen(from_data[suffix_array[st]:], to_data) 53 | y = matchlen(from_data[suffix_array[en]:], to_data) 54 | 55 | if x > y: 56 | return x, suffix_array[st] 57 | else: 58 | return y, suffix_array[en] 59 | 60 | x = (st + (en - st) // 2) 61 | length = min(len(from_data) - suffix_array[x], len(to_data)) 62 | 63 | if memcmp(from_data[suffix_array[x]:suffix_array[x] + length], to_data[:length]) < 0: 64 | return search(suffix_array, from_data, to_data, x, en) 65 | else: 66 | return search(suffix_array, from_data, to_data, st, x) 67 | 68 | 69 | def pack_size(value): 70 | packed = bytearray() 71 | 72 | if value == 0: 73 | packed.append(0) 74 | elif value < 0x8000000000000000: 75 | if value > 0: 76 | packed.append(0) 77 | else: 78 | packed.append(0x40) 79 | value *= -1 80 | 81 | packed[0] |= (0x80 | (value & 0x3f)) 82 | value >>= 6 83 | 84 | while value > 0: 85 | packed.append(0x80 | (value & 0x7f)) 86 | value >>= 7 87 | else: 88 | raise Exception('Size too big.') 89 | 90 | packed[-1] &= 0x7f 91 | 92 | return packed 93 | 94 | 95 | def append_buffer(chunks, buf): 96 | chunks.append(pack_size(len(buf))) 97 | chunks.append(buf) 98 | 99 | 100 | def create_patch(suffix_array, from_data, to_data): 101 | """Return chunks of data. 102 | 103 | """ 104 | 105 | scan = 0 106 | length = 0 107 | last_scan = 0 108 | last_pos = 0 109 | last_offset = 0 110 | pos = 0 111 | chunks = [] 112 | 113 | while scan < len(to_data): 114 | from_score = 0 115 | scsc = scan 116 | scan += length 117 | 118 | while scan < len(to_data): 119 | length, pos = search(suffix_array, 120 | from_data, 121 | to_data[scan:], 122 | 0, 123 | len(from_data)) 124 | 125 | while scsc < scan + length: 126 | if ((scsc + last_offset < len(from_data)) 127 | and (from_data[scsc + last_offset] == to_data[scsc])): 128 | from_score += 1 129 | 130 | scsc += 1 131 | 132 | if ((length == from_score) and (length != 0)) or (length > from_score + 8): 133 | break 134 | 135 | if ((scan + last_offset < len(from_data)) 136 | and (from_data[scan + last_offset] == to_data[scan])): 137 | from_score -= 1 138 | 139 | scan += 1 140 | 141 | if (length != from_score) or (scan == len(to_data)): 142 | s = 0 143 | sf = 0 144 | lenf = 0 145 | i = 0 146 | 147 | while (last_scan + i < scan) and (last_pos + i < len(from_data)): 148 | if from_data[last_pos + i] == to_data[last_scan + i]: 149 | s += 1 150 | 151 | i += 1 152 | 153 | if s * 2 - i > sf * 2 - lenf: 154 | sf = s 155 | lenf = i 156 | 157 | lenb = 0 158 | 159 | if scan < len(to_data): 160 | s = 0 161 | sb = 0 162 | i = 1 163 | 164 | while (scan >= last_scan + i) and (pos >= i): 165 | if from_data[pos - i] == to_data[scan - i]: 166 | s += 1 167 | 168 | if s * 2 - i > sb * 2 - lenb: 169 | sb = s 170 | lenb = i 171 | 172 | i += 1 173 | 174 | if last_scan + lenf > scan - lenb: 175 | overlap = (last_scan + lenf) - (scan - lenb) 176 | s = 0 177 | ss = 0 178 | lens = 0 179 | 180 | for i in range(overlap): 181 | if (to_data[last_scan + lenf - overlap + i] 182 | == from_data[last_pos + lenf - overlap + i]): 183 | s += 1 184 | 185 | if to_data[scan - lenb + i] == from_data[pos - lenb + i]: 186 | s -= 1 187 | 188 | if s > ss: 189 | ss = s 190 | lens = (i + 1) 191 | 192 | lenf += (lens - overlap) 193 | lenb -= lens 194 | 195 | db = bytearray( 196 | to_data[last_scan + i] - from_data[last_pos + i] 197 | for i in range(lenf) 198 | ) 199 | 200 | eb = bytearray( 201 | to_data[last_scan + lenf + i] 202 | for i in range((scan - lenb) - (last_scan + lenf)) 203 | ) 204 | 205 | 206 | # Diff, extra and adjustment. 207 | append_buffer(chunks, db) 208 | append_buffer(chunks, eb) 209 | chunks.append(pack_size((pos - lenb) - (last_pos + lenf))) 210 | 211 | last_scan = (scan - lenb) 212 | last_pos = (pos - lenb) 213 | last_offset = (pos - scan) 214 | 215 | return chunks 216 | -------------------------------------------------------------------------------- /detools/compression/crle.py: -------------------------------------------------------------------------------- 1 | """Conditional Run Length Encoding (CRLE) compresses repeated bytes 2 | with RLE, but leaves other data sequences as is. 3 | 4 | It compresses diffs fairly well, but extras poorly. Not very useful in 5 | general. 6 | 7 | """ 8 | 9 | import struct 10 | from ..errors import Error 11 | 12 | 13 | MINIMUM_REPEATED_SIZE = 6 14 | 15 | SCATTERED = 0 16 | REPEATED = 1 17 | 18 | 19 | class CrleCompressor(object): 20 | 21 | def __init__(self): 22 | self._data = b'' 23 | self._flushing = False 24 | self._number_of_compressed_bytes = 0 25 | 26 | def compress(self, data): 27 | """Compress `data` and return any compressed data. 28 | 29 | """ 30 | 31 | self._data += data 32 | 33 | return self.compress_segment() 34 | 35 | def flush(self): 36 | """Compress and return remaining data. 37 | 38 | """ 39 | 40 | if self._number_of_compressed_bytes == len(self._data) == 0: 41 | compressed = struct.pack('B', SCATTERED) 42 | compressed += pack_size(0) 43 | else: 44 | self._flushing = True 45 | compressed = [] 46 | 47 | while True: 48 | chunk = self.compress_segment() 49 | 50 | if not chunk: 51 | break 52 | 53 | compressed.append(chunk) 54 | 55 | compressed = b''.join(compressed) 56 | 57 | return compressed 58 | 59 | def find_repeated_segment(self): 60 | """Find the first repeated segment in the data and return its offset 61 | and length. Return ``(None, None)`` if no repeated segment was 62 | found. 63 | 64 | """ 65 | 66 | for offset in range(len(self._data)): 67 | byte = self._data[offset] 68 | length = 0 69 | 70 | while ((offset + length < len(self._data)) 71 | and (byte == self._data[offset + length])): 72 | length += 1 73 | 74 | if length >= MINIMUM_REPEATED_SIZE: 75 | return offset, length 76 | 77 | return None, None 78 | 79 | def get_segment(self): 80 | """Get a segment of scattered or repeated data. Returns the segment 81 | kind and data. Returns ``(None, None)`` if no segment was 82 | found. 83 | 84 | """ 85 | 86 | offset, length = self.find_repeated_segment() 87 | 88 | if offset is None: 89 | if self._flushing: 90 | kind = SCATTERED 91 | data = self._data 92 | self._data = b'' 93 | else: 94 | kind = None 95 | data = None 96 | elif offset > 0: 97 | data = self._data[:offset] 98 | self._data = self._data[offset:] 99 | kind = SCATTERED 100 | elif offset + length < len(self._data) or self._flushing: 101 | data = self._data[:length] 102 | self._data = self._data[length:] 103 | kind = REPEATED 104 | else: 105 | kind = None 106 | data = None 107 | 108 | return kind, data 109 | 110 | def compress_segment(self): 111 | """Compress one segment and return it. 112 | 113 | """ 114 | 115 | if len(self._data) == 0: 116 | return b'' 117 | 118 | kind, data = self.get_segment() 119 | 120 | if kind is None: 121 | return b'' 122 | 123 | compressed = struct.pack('B', kind) 124 | compressed += pack_size(len(data)) 125 | 126 | if kind == SCATTERED: 127 | compressed += data 128 | else: 129 | compressed += data[:1] 130 | 131 | self._number_of_compressed_bytes += len(compressed) 132 | 133 | return compressed 134 | 135 | 136 | class CrleDecompressor(object): 137 | 138 | def __init__(self, number_of_bytes): 139 | self._number_of_indata_bytes_left = number_of_bytes 140 | self._indata = b'' 141 | self._outdata = b'' 142 | self._number_of_scattered_bytes_left = 0 143 | 144 | def decompress(self, data, size): 145 | """Decompress up to size bytes. 146 | 147 | """ 148 | 149 | if self.eof: 150 | raise Error('Already at end of stream.') 151 | 152 | if len(data) > self._number_of_indata_bytes_left: 153 | data = data[:self._number_of_indata_bytes_left] 154 | 155 | self._indata += data 156 | self._number_of_indata_bytes_left -= len(data) 157 | self._outdata += self.decompress_segments() 158 | data = self._outdata[:size] 159 | self._outdata = self._outdata[size:] 160 | 161 | return data 162 | 163 | @property 164 | def needs_input(self): 165 | return len(self._outdata) == 0 and not self.eof 166 | 167 | @property 168 | def eof(self): 169 | return (self._number_of_indata_bytes_left == 0 170 | and len(self._outdata) == 0 171 | and len(self._indata) == 0) 172 | 173 | def decompress_segments(self): 174 | segments = [] 175 | 176 | try: 177 | while True: 178 | segments.append(self.decompress_segment()) 179 | except IndexError: 180 | pass 181 | 182 | return b''.join(segments) 183 | 184 | def decompress_segment(self): 185 | """Try to decompress a segment. Raises IndexError if not enough data 186 | is available.. 187 | 188 | """ 189 | 190 | if self._number_of_scattered_bytes_left == 0: 191 | kind = self._indata[0] 192 | 193 | if kind == SCATTERED: 194 | length, offset = unpack_size(self._indata, 1) 195 | remaining = (offset + length - len(self._indata)) 196 | 197 | if remaining > 0: 198 | self._number_of_scattered_bytes_left = remaining 199 | length -= remaining 200 | 201 | repetitions = 1 202 | elif kind == REPEATED: 203 | repetitions, offset = unpack_size(self._indata, 1) 204 | length = 1 205 | else: 206 | raise Error( 207 | 'Expected kind scattered(0) or repeated(1), but got {}.'.format( 208 | kind)) 209 | elif len(self._indata) > 0: 210 | length = min(len(self._indata), self._number_of_scattered_bytes_left) 211 | offset = 0 212 | repetitions = 1 213 | self._number_of_scattered_bytes_left -= length 214 | else: 215 | raise IndexError 216 | 217 | if len(self._indata) < offset + length: 218 | raise IndexError 219 | 220 | data = repetitions * self._indata[offset:offset + length] 221 | self._indata = self._indata[offset + length:] 222 | 223 | return data 224 | 225 | 226 | def pack_size(value): 227 | if value >= 0x8000000000000000: 228 | raise Error('Size too big.') 229 | 230 | packed = bytearray() 231 | packed.append(0) 232 | packed[0] |= (0x80 | (value & 0x7f)) 233 | value >>= 7 234 | 235 | while value > 0: 236 | packed.append(0x80 | (value & 0x7f)) 237 | value >>= 7 238 | 239 | packed[-1] &= 0x7f 240 | 241 | return packed 242 | 243 | 244 | def unpack_size(buf, position): 245 | byte = 0x80 246 | value = 0 247 | offset = 0 248 | 249 | while byte & 0x80: 250 | byte = buf[position] 251 | value |= ((byte & 0x7f) << offset) 252 | offset += 7 253 | position += 1 254 | 255 | return value, position 256 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/detools.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/detools.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/detools" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/detools" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 2> nul 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\detools.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\detools.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. cantools documentation master file, created by 2 | sphinx-quickstart on Sat Apr 25 11:54:09 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | Binary delta encoding utility 10 | ============================= 11 | 12 | .. include:: ../README.rst 13 | 14 | Patch types 15 | =========== 16 | 17 | Sequential 18 | ------ 19 | 20 | A sequential patch uses two memory regions or files. One contains the 21 | from-data and the to-data is written to the other. The patch is 22 | accesses sequentially from the beginning to the end when applying the 23 | patch. 24 | 25 | .. code-block:: text 26 | 27 | $ detools create_patch tests/files/foo.old tests/files/foo.new foo.patch 28 | 29 | Patch layout: 30 | 31 | +--------+--------+---------+--------+--------+---------+--------+-----+ 32 | | header | diff 1 | extra 1 | adj. 1 | diff 2 | extra 2 | adj. 2 | ... | 33 | +--------+--------+---------+--------+--------+---------+--------+-----+ 34 | 35 | The first part of the header is not compressed. The rest of the patch 36 | is compressed. 37 | 38 | HDiffPatch 39 | ---------- 40 | 41 | Patches of this type are slightly smaller than sequential patches. 42 | 43 | .. code-block:: text 44 | 45 | $ detools create_patch --patch-type hdiffpatch \ 46 | tests/files/foo.old tests/files/foo.new foo.patch 47 | 48 | Patch layout: 49 | 50 | +--------+--------+------------------+---------------+-------+ 51 | | header | covers | RLE diff control | RLE diff code | extra | 52 | +--------+--------+------------------+---------------+-------+ 53 | 54 | The header is not compressed. The other four parts are compressed 55 | separately. 56 | 57 | In-place 58 | -------- 59 | 60 | The in-place patch type is designed to update an application in 61 | place. It is useful when flash operations are faster than the external 62 | interface transfer speed. 63 | 64 | Use ``create_patch_in_place`` to create an in-place patch. The to 65 | options ``--memory-size`` and ``--segment-size`` are required, while 66 | ``--minimum-shift-size`` is optional. 67 | 68 | .. code-block:: text 69 | 70 | $ detools create_patch --type in-place --memory-size 131072 --segment-size 32768 \ 71 | tests/files/foo.old tests/files/foo.new foo.patch 72 | 73 | Here is an example of an in-place application update from version 1 to 74 | version 2. The two applications are represented by the character 75 | sequences below for clarity. 76 | 77 | .. code-block:: text 78 | 79 | Version 1: 0123456789abcdefghijklmnopqr 80 | Version 2: ABCDEFGHIJKLMNOPQRSTUVWXYZstuvwxyz 81 | 82 | #. Before the update application version 1 is found in memory segments 83 | 0 to 3. 84 | 85 | .. code-block:: text 86 | 87 | 0 1 2 3 4 5 88 | +-------+-------+-------+-------+-------+-------+ 89 | |0123456789abcdefghijklmnopqr| | 90 | +-------+-------+-------+-------+-------+-------+ 91 | 92 | #. The update starts by moving the application two segments to the 93 | right to make room for the new version. 94 | 95 | .. code-block:: text 96 | 97 | 0 1 2 3 4 5 98 | +-------+-------+-------+-------+-------+-------+ 99 | | |0123456789abcdefghijklmnopqr| | 100 | +-------+-------+-------+-------+-------+-------+ 101 | 102 | #. The first part of the patch is received and combined with 103 | application version 1. The combined data is written to segment 0. 104 | 105 | .. code-block:: text 106 | 107 | 0 1 2 3 4 5 108 | +-------+-------+-------+-------+-------+-------+ 109 | |ABCDEFG| |0123456789abcdefghijklmnopqr| | 110 | +-------+-------+-------+-------+-------+-------+ 111 | 112 | #. Same as the previous step, but the combined data is written to 113 | segment 1. 114 | 115 | .. code-block:: text 116 | 117 | 0 1 2 3 4 5 118 | +-------+-------+-------+-------+-------+-------+ 119 | |ABCDEFGHIJKLMNO|0123456789abcdefghijklmnopqr| | 120 | +-------+-------+-------+-------+-------+-------+ 121 | 122 | #. Segment 2 is erased to make room for the next part of the patch. 123 | 124 | .. code-block:: text 125 | 126 | 0 1 2 3 4 5 127 | +-------+-------+-------+-------+-------+-------+ 128 | |ABCDEFGHIJKLMNO| |89abcdefghijklmnopqr| | 129 | +-------+-------+-------+-------+-------+-------+ 130 | 131 | #. Combined data written to segment 2. 132 | 133 | .. code-block:: text 134 | 135 | 0 1 2 3 4 5 136 | +-------+-------+-------+-------+-------+-------+ 137 | |ABCDEFGHIJKLMNOPQRSTUVW|89abcdefghijklmnopqr| | 138 | +-------+-------+-------+-------+-------+-------+ 139 | 140 | #. Segment 3 is erased. 141 | 142 | .. code-block:: text 143 | 144 | 0 1 2 3 4 5 145 | +-------+-------+-------+-------+-------+-------+ 146 | |ABCDEFGHIJKLMNOPQRSTUVW| |ghijklmnopqr| | 147 | +-------+-------+-------+-------+-------+-------+ 148 | 149 | #. Combined data written to segment 3. 150 | 151 | .. code-block:: text 152 | 153 | 0 1 2 3 4 5 154 | +-------+-------+-------+-------+-------+-------+ 155 | |ABCDEFGHIJKLMNOPQRSTUVWXYZstuvw|ghijklmnopqr| | 156 | +-------+-------+-------+-------+-------+-------+ 157 | 158 | #. Segment 4 is erased. 159 | 160 | .. code-block:: text 161 | 162 | 0 1 2 3 4 5 163 | +-------+-------+-------+-------+-------+-------+ 164 | |ABCDEFGHIJKLMNOPQRSTUVWXYZstuvw| |opqr| | 165 | +-------+-------+-------+-------+-------+-------+ 166 | 167 | #. Combined data written to segment 4. 168 | 169 | .. code-block:: text 170 | 171 | 0 1 2 3 4 5 172 | +-------+-------+-------+-------+-------+-------+ 173 | |ABCDEFGHIJKLMNOPQRSTUVWXYZstuvwxyz| |opqr| | 174 | +-------+-------+-------+-------+-------+-------+ 175 | 176 | #. Optionally, segment 5 is erased. 177 | 178 | .. code-block:: text 179 | 180 | 0 1 2 3 4 5 181 | +-------+-------+-------+-------+-------+-------+ 182 | |ABCDEFGHIJKLMNOPQRSTUVWXYZstuvwxyz| | 183 | +-------+-------+-------+-------+-------+-------+ 184 | 185 | #. Update to application version 2 complete! 186 | 187 | An interrupted in-place update can be resumed by introducing a step 188 | state, persistentely stored in a separate memory region. Also store 189 | the patch header persistentely. Reject any other patch until the 190 | currently active patch has been successfully applied. 191 | 192 | .. code-block:: text 193 | 194 | 0 1 2 3 4 5 195 | +-------+-------+-------+-------+-------+-------+ 196 | |0123456789abcdefghijklmnopqr| | Step: 0 197 | +-------+-------+-------+-------+-------+-------+ 198 | |0123456789abcdefghijklmnopqr| |opqr| | Step: 1 199 | +-------+-------+-------+-------+-------+-------+ 200 | |0123456789abcdefghijklmnopqr| |ghijklmnopqr| | Step: 2 201 | +-------+-------+-------+-------+-------+-------+ 202 | |0123456789abcdefghijklm|89abcdefghijklmnopqr| | Step: 3 203 | +-------+-------+-------+-------+-------+-------+ 204 | |0123456789abcde|0123456789abcdefghijklmnopqr| | Step: 4 205 | +-------+-------+-------+-------+-------+-------+ 206 | |ABCDEFG789abcde|0123456789abcdefghijklmnopqr| | Step: 5 207 | +-------+-------+-------+-------+-------+-------+ 208 | |ABCDEFGHIJKLMNO|0123456789abcdefghijklmnopqr| | Step: 6 209 | +-------+-------+-------+-------+-------+-------+ 210 | |ABCDEFGHIJKLMNOPQRSTUVW|89abcdefghijklmnopqr| | Step: 7 211 | +-------+-------+-------+-------+-------+-------+ 212 | |ABCDEFGHIJKLMNOPQRSTUVWXYZstuvw|ghijklmnopqr| | Step: 8 213 | +-------+-------+-------+-------+-------+-------+ 214 | |ABCDEFGHIJKLMNOPQRSTUVWXYZstuvwxyz| |opqr| | Step: 9 215 | +-------+-------+-------+-------+-------+-------+ 216 | 217 | Functions and classes 218 | ===================== 219 | 220 | .. autofunction:: detools.create_patch 221 | 222 | .. autofunction:: detools.apply_patch 223 | 224 | .. autofunction:: detools.apply_patch_in_place 225 | 226 | .. autofunction:: detools.patch_info 227 | 228 | .. autofunction:: detools.create_patch_filenames 229 | 230 | .. autofunction:: detools.apply_patch_filenames 231 | 232 | .. autofunction:: detools.apply_patch_in_place_filenames 233 | 234 | .. autofunction:: detools.patch_info_filename 235 | -------------------------------------------------------------------------------- /c/tst/test_dump_restore.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "nala.h" 3 | #include "utils.h" 4 | #include "../detools.h" 5 | 6 | static void init(struct detools_apply_patch_t *apply_patch_p, 7 | size_t patch_size) 8 | { 9 | int res; 10 | 11 | /* Simulate program restart. */ 12 | memset(apply_patch_p, 0, sizeof(*apply_patch_p)); 13 | utils_files_reopen_from(); 14 | 15 | res = detools_apply_patch_init(apply_patch_p, 16 | utils_from_read, 17 | utils_from_seek, 18 | patch_size, 19 | utils_to_write, 20 | NULL); 21 | ASSERT_EQ(res, DETOOLS_OK); 22 | } 23 | 24 | static void dump(struct detools_apply_patch_t *apply_patch_p) 25 | { 26 | int res; 27 | 28 | res = detools_apply_patch_dump(apply_patch_p, utils_state_write); 29 | ASSERT_EQ(res, DETOOLS_OK); 30 | ASSERT_EQ(fseek(utils_files.state.file_p, 0, SEEK_SET), 0); 31 | } 32 | 33 | static size_t restore(struct detools_apply_patch_t *apply_patch_p) 34 | { 35 | int res; 36 | 37 | res = detools_apply_patch_restore(apply_patch_p, utils_state_read); 38 | ASSERT_EQ(res, DETOOLS_OK); 39 | ASSERT_EQ(fseek(utils_files.state.file_p, 0, SEEK_SET), 0); 40 | ASSERT_EQ(fseek(utils_files.to.file_p, 41 | detools_apply_patch_get_to_offset(apply_patch_p), 42 | SEEK_SET), 0); 43 | 44 | return (detools_apply_patch_get_patch_offset(apply_patch_p)); 45 | } 46 | 47 | static void process(struct detools_apply_patch_t *apply_patch_p, 48 | const uint8_t *buf_p, 49 | size_t size) 50 | { 51 | int res; 52 | 53 | res = detools_apply_patch_process(apply_patch_p, buf_p, size); 54 | ASSERT_EQ(res, DETOOLS_OK); 55 | } 56 | 57 | static void finalize(struct detools_apply_patch_t *apply_patch_p, 58 | size_t size) 59 | { 60 | int res; 61 | 62 | res = detools_apply_patch_finalize(apply_patch_p); 63 | ASSERT_EQ(res, size); 64 | } 65 | 66 | TEST(foo_none_at_offset_0) 67 | { 68 | struct detools_apply_patch_t apply_patch; 69 | 70 | utils_files_init("../../tests/files/foo/old", 71 | "../../tests/files/foo/none.patch", 72 | "../../tests/files/foo/new"); 73 | 74 | /* Init and dump. */ 75 | init(&apply_patch, utils_files.patch.size); 76 | dump(&apply_patch); 77 | 78 | /* Init again, restore and apply the patch. */ 79 | init(&apply_patch, 0); 80 | ASSERT_EQ(restore(&apply_patch), 0); 81 | process(&apply_patch, &utils_files.patch.buf_p[0], utils_files.patch.size); 82 | finalize(&apply_patch, 2780); 83 | 84 | utils_files_assert_and_destroy(); 85 | } 86 | 87 | TEST(foo_none_at_offset_100_and_2791) 88 | { 89 | struct detools_apply_patch_t apply_patch; 90 | 91 | utils_files_init("../../tests/files/foo/old", 92 | "../../tests/files/foo/none.patch", 93 | "../../tests/files/foo/new"); 94 | 95 | /* Init, process 100 bytes and dump. Process another 50 bytes, 96 | which will be "lost" when later restoring. */ 97 | init(&apply_patch, utils_files.patch.size); 98 | process(&apply_patch, &utils_files.patch.buf_p[0], 100); 99 | dump(&apply_patch); 100 | process(&apply_patch, &utils_files.patch.buf_p[100], 50); 101 | 102 | /* Init again, restore, apply all but one byte and dump again. */ 103 | init(&apply_patch, 0); 104 | ASSERT_EQ(restore(&apply_patch), 100); 105 | process(&apply_patch, &utils_files.patch.buf_p[100], 2691); 106 | dump(&apply_patch); 107 | 108 | /* Init once again, restore and apply the last byte. */ 109 | init(&apply_patch, 0); 110 | ASSERT_EQ(restore(&apply_patch), 2791); 111 | process(&apply_patch, &utils_files.patch.buf_p[2791], 1); 112 | finalize(&apply_patch, 2780); 113 | 114 | utils_files_assert_and_destroy(); 115 | } 116 | 117 | TEST(foo_none_one_byte_at_a_time) 118 | { 119 | struct detools_apply_patch_t apply_patch; 120 | int i; 121 | 122 | utils_files_init("../../tests/files/foo/old", 123 | "../../tests/files/foo/none.patch", 124 | "../../tests/files/foo/new"); 125 | 126 | /* Init, process 10 bytes and dump. */ 127 | init(&apply_patch, utils_files.patch.size); 128 | process(&apply_patch, &utils_files.patch.buf_p[0], 10); 129 | dump(&apply_patch); 130 | 131 | /* Init again, restore, process one byte and dump again. */ 132 | for (i = 10; i < 2792; i++) { 133 | init(&apply_patch, 0); 134 | ASSERT_EQ(restore(&apply_patch), i); 135 | process(&apply_patch, &utils_files.patch.buf_p[i], 1); 136 | dump(&apply_patch); 137 | } 138 | 139 | utils_files_assert_and_destroy(); 140 | } 141 | 142 | TEST(foo_none_dump_state_write_error) 143 | { 144 | struct detools_apply_patch_t apply_patch; 145 | int res; 146 | 147 | utils_files_init("../../tests/files/foo/old", 148 | "../../tests/files/foo/none.patch", 149 | "../../tests/files/foo/new"); 150 | 151 | init(&apply_patch, utils_files.patch.size); 152 | 153 | utils_state_write_mock_once(sizeof(apply_patch), -1); 154 | res = detools_apply_patch_dump(&apply_patch, utils_state_write); 155 | ASSERT_EQ(res, -DETOOLS_IO_FAILED); 156 | 157 | utils_files_destroy(); 158 | } 159 | 160 | TEST(foo_crle_at_offset_100_101_164_and_189) 161 | { 162 | struct detools_apply_patch_t apply_patch; 163 | 164 | utils_files_init("../../tests/files/foo/old", 165 | "../../tests/files/foo/crle.patch", 166 | "../../tests/files/foo/new"); 167 | 168 | /* Init, process 100 bytes and dump. Process another 50 bytes, 169 | which will be "lost" when later restoring. */ 170 | init(&apply_patch, utils_files.patch.size); 171 | process(&apply_patch, &utils_files.patch.buf_p[0], 100); 172 | dump(&apply_patch); 173 | process(&apply_patch, &utils_files.patch.buf_p[100], 50); 174 | 175 | /* Init again, restore, process one byte and dump again. */ 176 | init(&apply_patch, 0); 177 | ASSERT_EQ(restore(&apply_patch), 100); 178 | process(&apply_patch, &utils_files.patch.buf_p[100], 1); 179 | dump(&apply_patch); 180 | 181 | /* Init again, restore, process 63 bytes and dump again. */ 182 | init(&apply_patch, 0); 183 | ASSERT_EQ(restore(&apply_patch), 101); 184 | process(&apply_patch, &utils_files.patch.buf_p[101], 63); 185 | dump(&apply_patch); 186 | 187 | /* Init again, restore, apply all but one byte and dump again. */ 188 | init(&apply_patch, 0); 189 | ASSERT_EQ(restore(&apply_patch), 164); 190 | process(&apply_patch, &utils_files.patch.buf_p[164], 25); 191 | dump(&apply_patch); 192 | 193 | /* Init once again, restore and apply the last byte. */ 194 | init(&apply_patch, 0); 195 | ASSERT_EQ(restore(&apply_patch), 189); 196 | process(&apply_patch, &utils_files.patch.buf_p[189], 1); 197 | finalize(&apply_patch, 2780); 198 | 199 | utils_files_assert_and_destroy(); 200 | } 201 | 202 | TEST(foo_crle_one_byte_at_a_time) 203 | { 204 | struct detools_apply_patch_t apply_patch; 205 | int i; 206 | 207 | utils_files_init("../../tests/files/foo/old", 208 | "../../tests/files/foo/crle.patch", 209 | "../../tests/files/foo/new"); 210 | 211 | /* Init, process 10 bytes and dump. */ 212 | init(&apply_patch, utils_files.patch.size); 213 | process(&apply_patch, &utils_files.patch.buf_p[0], 10); 214 | dump(&apply_patch); 215 | 216 | /* Init again, restore, process one byte and dump again. */ 217 | for (i = 10; i < 190; i++) { 218 | init(&apply_patch, 0); 219 | ASSERT_EQ(restore(&apply_patch), i); 220 | process(&apply_patch, &utils_files.patch.buf_p[i], 1); 221 | dump(&apply_patch); 222 | } 223 | 224 | utils_files_assert_and_destroy(); 225 | } 226 | 227 | TEST(foo_heatshrink_at_offset_10_and_100) 228 | { 229 | struct detools_apply_patch_t apply_patch; 230 | 231 | utils_files_init("../../tests/files/foo/old", 232 | "../../tests/files/foo/heatshrink.patch", 233 | "../../tests/files/foo/new"); 234 | 235 | /* Init, process 10 bytes and dump. Process another 50 bytes, 236 | which will be "lost" when later restoring. */ 237 | init(&apply_patch, utils_files.patch.size); 238 | process(&apply_patch, &utils_files.patch.buf_p[0], 10); 239 | dump(&apply_patch); 240 | process(&apply_patch, &utils_files.patch.buf_p[10], 50); 241 | 242 | /* Init again, restore, process 90 bytes and dump again. */ 243 | init(&apply_patch, 0); 244 | ASSERT_EQ(restore(&apply_patch), 10); 245 | process(&apply_patch, &utils_files.patch.buf_p[10], 90); 246 | dump(&apply_patch); 247 | 248 | /* Init once again, restore and process remaining 25 bytes. */ 249 | init(&apply_patch, 0); 250 | ASSERT_EQ(restore(&apply_patch), 100); 251 | process(&apply_patch, &utils_files.patch.buf_p[100], 26); 252 | finalize(&apply_patch, 2780); 253 | 254 | utils_files_assert_and_destroy(); 255 | } 256 | 257 | TEST(foo_heatshrink_one_byte_at_a_time) 258 | { 259 | struct detools_apply_patch_t apply_patch; 260 | int i; 261 | 262 | utils_files_init("../../tests/files/foo/old", 263 | "../../tests/files/foo/heatshrink.patch", 264 | "../../tests/files/foo/new"); 265 | 266 | /* Init, process 10 bytes and dump. */ 267 | init(&apply_patch, utils_files.patch.size); 268 | process(&apply_patch, &utils_files.patch.buf_p[0], 10); 269 | dump(&apply_patch); 270 | 271 | /* Init again, restore, process one byte and dump again. */ 272 | for (i = 10; i < 125; i++) { 273 | init(&apply_patch, 0); 274 | ASSERT_EQ(restore(&apply_patch), i); 275 | process(&apply_patch, &utils_files.patch.buf_p[i], 1); 276 | dump(&apply_patch); 277 | } 278 | 279 | utils_files_assert_and_destroy(); 280 | } 281 | --------------------------------------------------------------------------------