├── .editorconfig ├── .gitignore ├── .gitmodules ├── .travis.yml ├── LICENSE ├── Makefile ├── README.rst ├── build_cli.sh ├── performance.py ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── build-wheels.sh ├── check_tag.py ├── requirements.txt └── test_main.py └── xdelta3 ├── __init__.py ├── _xdelta3.c ├── main.py └── version.py /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | # Unix-style newlines with a newline ending every file 4 | [*] 5 | end_of_line = lf 6 | insert_final_newline = true 7 | line_length = 120 8 | trim_trailing_whitespace = true 9 | 10 | [*.c] 11 | indent_style = space 12 | indent_size = 2 13 | 14 | [*.py] 15 | indent_style = space 16 | indent_size = 4 17 | 18 | [Makefile] 19 | indent_style = tab 20 | 21 | [*.md,*.rst] 22 | indent_style = space 23 | indent_size = 4 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | env/ 3 | env35/ 4 | *.py[cod] 5 | *.egg-info/ 6 | build/ 7 | dist/ 8 | .cache/ 9 | test.py 10 | .coverage 11 | htmlcov/ 12 | *.so 13 | /*.txt 14 | /*.whl 15 | /xdelta3/lib/ 16 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "xdelta"] 2 | path = xdelta 3 | url = https://github.com/jmacd/xdelta.git 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | cache: pip 4 | 5 | python: 6 | - '3.5' 7 | - '3.6' 8 | - 'nightly' # currently 3.7 9 | 10 | matrix: 11 | allow_failures: 12 | - python: 'nightly' 13 | 14 | #env: 15 | #- DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 16 | #services: 17 | #- docker 18 | 19 | addons: 20 | apt: 21 | sources: 22 | - ubuntu-toolchain-r-test 23 | packages: 24 | - gcc-5 25 | 26 | install: 27 | - sudo unlink /usr/bin/gcc && sudo ln -s /usr/bin/gcc-5 /usr/bin/gcc 28 | - gcc -v 29 | #- docker pull $DOCKER_IMAGE 30 | - pip freeze 31 | 32 | script: 33 | - make test 34 | - make lint 35 | - ./tests/check_tag.py 36 | # TODO manylinux is currently broken as gcc-5 isn't available on centos 5 37 | #- docker run --rm -v `pwd`:/io $DOCKER_IMAGE /io/tests/build-wheels.sh 38 | 39 | after_success: 40 | - ls -lha 41 | - bash <(curl -s https://codecov.io/bash) 42 | 43 | deploy: 44 | provider: pypi 45 | user: samuelcolvin 46 | password: 47 | secure: Qrh0eve9uLknhDZ7q5DQaSNaL8gG/9bXaYAjJB3s9hMAXtTM+4zVBpke0Dl5Hziy0sWN304OSuAwHSzzBaZxChZ9mpaE7cuuEVNo5RZXNZJxbsHQLlTngV1w6khd7VhgmgG6QbYqoFXgpkB3p88bLxyAc4I2PbTVY4omeRCzKg8av20pu+C3mCN0e5dvE7u2CFDgd/s0V4fgt0Cjhw88sS/7fqsUZpbXy8J9Q+V1IP40CAmr8nXPEva4nSmTOFEyt6NOijWWizea5LvT9MJwK7tugM6BeH4IT1OaA58eRuOlDenaTDbhfOMQain/PTEWwr3KnYa8v6dvaVSU2uMEm2v+hVCYqQeFwAdFEJLY/bpl5kTRQ++dmvEPrWdU0cOf22vDzjfLLSHtWRQqtfp5BXpfL7jXV2IydUTjEmrybok7LxoRc7JhrfvlAScd6t2R/OQ1dyvav68ICkOfAu660h9ETa0wsq917wIj9kErz4Gj+ItLwHsRkCwoEKpk3Xa+FkUkuEMoD/gmPB93R5fpSldc0ZCNXzNPcWJcBWxQpvlhXs/lhdl5gG15BUzDYQbWsK7XaGn8wV+cHNODee4VE/vKUTlo/xNWkxmWMWzQS7t+yZhmorN69Xd4wGcvT2Bed/9Q/bc154G2da+ps89cWVkRgY2lYrYXcu1FM7O9KMs= 48 | distributions: sdist bdist_wheel 49 | skip_upload_docs: true 50 | skip_cleanup: true 51 | on: 52 | tags: true 53 | python: 3.6 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | xdelta3-python - python wrapper for xdelta3 Copyright 2017 Samuel Colvin 2 | 3 | xdelta3 - delta compression tools and library Copyright 2016 Joshua MacDonald 4 | 5 | Apache License 6 | Version 2.0, January 2004 7 | http://www.apache.org/licenses/ 8 | 9 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 10 | 11 | 1. Definitions. 12 | 13 | "License" shall mean the terms and conditions for use, reproduction, 14 | and distribution as defined by Sections 1 through 9 of this document. 15 | 16 | "Licensor" shall mean the copyright owner or entity authorized by 17 | the copyright owner that is granting the License. 18 | 19 | "Legal Entity" shall mean the union of the acting entity and all 20 | other entities that control, are controlled by, or are under common 21 | control with that entity. For the purposes of this definition, 22 | "control" means (i) the power, direct or indirect, to cause the 23 | direction or management of such entity, whether by contract or 24 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 25 | outstanding shares, or (iii) beneficial ownership of such entity. 26 | 27 | "You" (or "Your") shall mean an individual or Legal Entity 28 | exercising permissions granted by this License. 29 | 30 | "Source" form shall mean the preferred form for making modifications, 31 | including but not limited to software source code, documentation 32 | source, and configuration files. 33 | 34 | "Object" form shall mean any form resulting from mechanical 35 | transformation or translation of a Source form, including but 36 | not limited to compiled object code, generated documentation, 37 | and conversions to other media types. 38 | 39 | "Work" shall mean the work of authorship, whether in Source or 40 | Object form, made available under the License, as indicated by a 41 | copyright notice that is included in or attached to the work 42 | (an example is provided in the Appendix below). 43 | 44 | "Derivative Works" shall mean any work, whether in Source or Object 45 | form, that is based on (or derived from) the Work and for which the 46 | editorial revisions, annotations, elaborations, or other modifications 47 | represent, as a whole, an original work of authorship. For the purposes 48 | of this License, Derivative Works shall not include works that remain 49 | separable from, or merely link (or bind by name) to the interfaces of, 50 | the Work and Derivative Works thereof. 51 | 52 | "Contribution" shall mean any work of authorship, including 53 | the original version of the Work and any modifications or additions 54 | to that Work or Derivative Works thereof, that is intentionally 55 | submitted to Licensor for inclusion in the Work by the copyright owner 56 | or by an individual or Legal Entity authorized to submit on behalf of 57 | the copyright owner. For the purposes of this definition, "submitted" 58 | means any form of electronic, verbal, or written communication sent 59 | to the Licensor or its representatives, including but not limited to 60 | communication on electronic mailing lists, source code control systems, 61 | and issue tracking systems that are managed by, or on behalf of, the 62 | Licensor for the purpose of discussing and improving the Work, but 63 | excluding communication that is conspicuously marked or otherwise 64 | designated in writing by the copyright owner as "Not a Contribution." 65 | 66 | "Contributor" shall mean Licensor and any individual or Legal Entity 67 | on behalf of whom a Contribution has been received by Licensor and 68 | subsequently incorporated within the Work. 69 | 70 | 2. Grant of Copyright License. Subject to the terms and conditions of 71 | this License, each Contributor hereby grants to You a perpetual, 72 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 73 | copyright license to reproduce, prepare Derivative Works of, 74 | publicly display, publicly perform, sublicense, and distribute the 75 | Work and such Derivative Works in Source or Object form. 76 | 77 | 3. Grant of Patent License. Subject to the terms and conditions of 78 | this License, each Contributor hereby grants to You a perpetual, 79 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 80 | (except as stated in this section) patent license to make, have made, 81 | use, offer to sell, sell, import, and otherwise transfer the Work, 82 | where such license applies only to those patent claims licensable 83 | by such Contributor that are necessarily infringed by their 84 | Contribution(s) alone or by combination of their Contribution(s) 85 | with the Work to which such Contribution(s) was submitted. If You 86 | institute patent litigation against any entity (including a 87 | cross-claim or counterclaim in a lawsuit) alleging that the Work 88 | or a Contribution incorporated within the Work constitutes direct 89 | or contributory patent infringement, then any patent licenses 90 | granted to You under this License for that Work shall terminate 91 | as of the date such litigation is filed. 92 | 93 | 4. Redistribution. You may reproduce and distribute copies of the 94 | Work or Derivative Works thereof in any medium, with or without 95 | modifications, and in Source or Object form, provided that You 96 | meet the following conditions: 97 | 98 | (a) You must give any other recipients of the Work or 99 | Derivative Works a copy of this License; and 100 | 101 | (b) You must cause any modified files to carry prominent notices 102 | stating that You changed the files; and 103 | 104 | (c) You must retain, in the Source form of any Derivative Works 105 | that You distribute, all copyright, patent, trademark, and 106 | attribution notices from the Source form of the Work, 107 | excluding those notices that do not pertain to any part of 108 | the Derivative Works; and 109 | 110 | (d) If the Work includes a "NOTICE" text file as part of its 111 | distribution, then any Derivative Works that You distribute must 112 | include a readable copy of the attribution notices contained 113 | within such NOTICE file, excluding those notices that do not 114 | pertain to any part of the Derivative Works, in at least one 115 | of the following places: within a NOTICE text file distributed 116 | as part of the Derivative Works; within the Source form or 117 | documentation, if provided along with the Derivative Works; or, 118 | within a display generated by the Derivative Works, if and 119 | wherever such third-party notices normally appear. The contents 120 | of the NOTICE file are for informational purposes only and 121 | do not modify the License. You may add Your own attribution 122 | notices within Derivative Works that You distribute, alongside 123 | or as an addendum to the NOTICE text from the Work, provided 124 | that such additional attribution notices cannot be construed 125 | as modifying the License. 126 | 127 | You may add Your own copyright statement to Your modifications and 128 | may provide additional or different license terms and conditions 129 | for use, reproduction, or distribution of Your modifications, or 130 | for any such Derivative Works as a whole, provided Your use, 131 | reproduction, and distribution of the Work otherwise complies with 132 | the conditions stated in this License. 133 | 134 | 5. Submission of Contributions. Unless You explicitly state otherwise, 135 | any Contribution intentionally submitted for inclusion in the Work 136 | by You to the Licensor shall be under the terms and conditions of 137 | this License, without any additional terms or conditions. 138 | Notwithstanding the above, nothing herein shall supersede or modify 139 | the terms of any separate license agreement you may have executed 140 | with Licensor regarding such Contributions. 141 | 142 | 6. Trademarks. This License does not grant permission to use the trade 143 | names, trademarks, service marks, or product names of the Licensor, 144 | except as required for reasonable and customary use in describing the 145 | origin of the Work and reproducing the content of the NOTICE file. 146 | 147 | 7. Disclaimer of Warranty. Unless required by applicable law or 148 | agreed to in writing, Licensor provides the Work (and each 149 | Contributor provides its Contributions) on an "AS IS" BASIS, 150 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 151 | implied, including, without limitation, any warranties or conditions 152 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 153 | PARTICULAR PURPOSE. You are solely responsible for determining the 154 | appropriateness of using or redistributing the Work and assume any 155 | risks associated with Your exercise of permissions under this License. 156 | 157 | 8. Limitation of Liability. In no event and under no legal theory, 158 | whether in tort (including negligence), contract, or otherwise, 159 | unless required by applicable law (such as deliberate and grossly 160 | negligent acts) or agreed to in writing, shall any Contributor be 161 | liable to You for damages, including any direct, indirect, special, 162 | incidental, or consequential damages of any character arising as a 163 | result of this License or out of the use or inability to use the 164 | Work (including but not limited to damages for loss of goodwill, 165 | work stoppage, computer failure or malfunction, or any and all 166 | other commercial damages or losses), even if such Contributor 167 | has been advised of the possibility of such damages. 168 | 169 | 9. Accepting Warranty or Additional Liability. While redistributing 170 | the Work or Derivative Works thereof, You may choose to offer, 171 | and charge a fee for, acceptance of support, warranty, indemnity, 172 | or other liability obligations and/or rights consistent with this 173 | License. However, in accepting such obligations, You may act only 174 | on Your own behalf and on Your sole responsibility, not on behalf 175 | of any other Contributor, and only if You agree to indemnify, 176 | defend, and hold each Contributor harmless for any liability 177 | incurred by, or claims asserted against, such Contributor by reason 178 | of your accepting any such warranty or additional liability. 179 | 180 | END OF TERMS AND CONDITIONS 181 | 182 | APPENDIX: How to apply the Apache License to your work. 183 | 184 | To apply the Apache License to your work, attach the following 185 | boilerplate notice, with the fields enclosed by brackets "{}" 186 | replaced with your own identifying information. (Don't include 187 | the brackets!) The text should be enclosed in the appropriate 188 | comment syntax for the file format. We also recommend that a 189 | file or class name and description of purpose be included on the 190 | same "printed page" as the copyright notice for easier 191 | identification within third-party archives. 192 | 193 | Copyright {yyyy} {name of copyright owner} 194 | 195 | Licensed under the Apache License, Version 2.0 (the "License"); 196 | you may not use this file except in compliance with the License. 197 | You may obtain a copy of the License at 198 | 199 | http://www.apache.org/licenses/LICENSE-2.0 200 | 201 | Unless required by applicable law or agreed to in writing, software 202 | distributed under the License is distributed on an "AS IS" BASIS, 203 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 204 | See the License for the specific language governing permissions and 205 | limitations under the License. 206 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: prepare 2 | prepare: 3 | rm -rf xdelta3/lib/ || true 4 | mkdir xdelta3/lib/ 5 | cp xdelta/xdelta3/xdelta3*.c xdelta3/lib/ 6 | cp xdelta/xdelta3/xdelta3*.h xdelta3/lib/ 7 | rm xdelta3/lib/*test.h 8 | 9 | .PHONY: install 10 | install: prepare 11 | pip install -U setuptools pip 12 | pip install -U . 13 | pip install -r tests/requirements.txt 14 | 15 | .PHONY: isort 16 | isort: 17 | isort -rc -w 120 xdelta3 18 | isort -rc -w 120 tests 19 | 20 | .PHONY: lint 21 | lint: 22 | python setup.py check -rms 23 | flake8 xdelta3/ tests/ 24 | pytest xdelta3 -p no:sugar -q 25 | 26 | .PHONY: test 27 | test: install 28 | pytest --cov=xdelta3 29 | 30 | .PHONY: quickbuild 31 | quickbuild: prepare 32 | rm *.so || true 33 | python setup.py build --build-lib . && printf "\n build succeeded\n\n" 34 | 35 | .PHONY: testcov 36 | testcov: quickbuild 37 | pytest --cov=xdelta3 && (echo "building coverage html"; coverage html) 38 | 39 | .PHONY: all 40 | all: testcov lint 41 | 42 | .PHONY: clean 43 | clean: 44 | rm -rf `find . -name __pycache__` 45 | rm -f `find . -type f -name '*.py[co]' ` 46 | rm -f `find . -type f -name '*~' ` 47 | rm -f `find . -type f -name '.*~' ` 48 | rm -rf xdelta3/lib/ 49 | rm -rf htmlcov 50 | rm -rf *.egg-info 51 | rm -rf *.so 52 | rm -rf dist 53 | rm -f .coverage 54 | rm -f .coverage.* 55 | rm -rf build 56 | python setup.py clean 57 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | xdelta3-python 2 | ============== 3 | 4 | |BuildStatus| |Coverage| |pypi| 5 | 6 | Fast delta encoding in python using xdelta3. 7 | 8 | Requirements 9 | ------------ 10 | 11 | * **Python 3.5 or 3.6** - it's 2017, you should be using python 3.6 by now anyway. 12 | * **linux** - compilation only tested on ubuntu, might work on other platform. 13 | 14 | Installation 15 | ------------ 16 | 17 | .. code:: shell 18 | 19 | pip install xdelta3 20 | 21 | Usage 22 | ----- 23 | 24 | .. code:: python 25 | 26 | import xdelta3 27 | value_one = b'wonderful string to demonstrate xdelta3, much of these two strings is the same.' 28 | value_two = b'different string to demonstrate xdelta3, much of these two strings is the same.' 29 | delta = xdelta3.encode(value_one, value_two) 30 | # delta is an unreadable byte string: b'\xd6\xc3 ... \x01different\n\x13F\x00' 31 | 32 | print(f'New string length: {len(value_two)}, delta length: {len(delta)}') 33 | value_two_rebuilt = xdelta3.decode(value_one, delta) 34 | if value_two_rebuilt == value_two: 35 | print('Boo Ya! Delta encoding successful.') 36 | 37 | *(with xdelta3 installed this code should run "as is", just copy it into ipython or a file and run)* 38 | 39 | How fast? 40 | --------- 41 | 42 | *xdelta3-python* is a thin wrapper around `xdelta 3.1.1 `_ 43 | which is a highly optimised c library for delta calculation and compression. 44 | It can encode a delta and decode it again for 5 small changes in a 5.5 million character string 45 | (the complete works of shakespeare) in around 10ms (or 30ms with the highest compression level). Boom. 46 | See `performance.py `_. 47 | 48 | .. |BuildStatus| image:: https://travis-ci.org/samuelcolvin/xdelta3-python.svg?branch=master 49 | :target: https://travis-ci.org/samuelcolvin/xdelta3-python 50 | .. |Coverage| image:: https://codecov.io/gh/samuelcolvin/xdelta3-python/branch/master/graph/badge.svg 51 | :target: https://codecov.io/gh/samuelcolvin/xdelta3-python 52 | .. |pypi| image:: https://img.shields.io/pypi/v/xdelta3.svg 53 | :target: https://pypi.python.org/pypi/xdelta3 54 | -------------------------------------------------------------------------------- /build_cli.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # build xdelta3 command line tool in the submodule directory, requires automake and libtool 3 | set -e 4 | set -x 5 | 6 | cd xdelta/xdelta3 7 | git clean -fX 8 | libtoolize 9 | aclocal 10 | autoconf 11 | autoheader 12 | automake --add-missing 13 | ./configure 14 | make 15 | cd ../.. 16 | -------------------------------------------------------------------------------- /performance.py: -------------------------------------------------------------------------------- 1 | """ 2 | To run: 3 | 4 | curl https://ocw.mit.edu/ans7870/6/6.006/s08/lecturenotes/files/t8.shakespeare.txt > shakespeare.txt 5 | cp shakespeare.txt shakespeare_changed.txt 6 | vim shakespeare_changed.txt (and make some changes to shakespeare_changed.txt) 7 | python performance.py 8 | """ 9 | 10 | from pathlib import Path 11 | from statistics import mean, stdev 12 | from time import time 13 | import xdelta3 14 | 15 | 16 | v1 = Path('shakespeare.txt').read_bytes() 17 | v2 = Path('shakespeare_changed.txt').read_bytes() 18 | 19 | times = [] 20 | for i in range(50): 21 | start = time() 22 | delta = xdelta3.encode(v1, v2, xdelta3.Flags.COMPLEVEL_1) 23 | v22 = xdelta3.decode(v1, delta) 24 | time_taken = (time() - start) * 1000 25 | times.append(time_taken) 26 | print(f'{i + 1:3} result_match={v2 == v22} time={time_taken:0.1f}ms') 27 | 28 | print(f'\noriginal length: {len(v1)}') 29 | print(f'changed length: {len(v2)}') 30 | print(f'delta length: {len(delta)}') 31 | print(f'mean time taken to encode and decode: {mean(times):0.3f}ms, stdev {stdev(times):0.3f}ms') 32 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | testpaths = tests 3 | addopts = --isort 4 | timeout = 10 5 | 6 | [flake8] 7 | max-line-length = 120 8 | max-complexity = 10 9 | 10 | [bdist_wheel] 11 | python-tag = py35-py36 12 | plat-name = any 13 | 14 | [coverage:run] 15 | source = xdelta3 16 | branch = True 17 | 18 | [coverage:report] 19 | precision = 2 20 | exclude_lines = 21 | pragma: no cover 22 | raise NotImplementedError 23 | raise NotImplemented 24 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from importlib.machinery import SourceFileLoader 2 | from pathlib import Path 3 | from setuptools import setup, Extension 4 | 5 | THIS_DIR = Path(__file__).resolve().parent 6 | long_description = THIS_DIR.joinpath('README.rst').read_text() 7 | 8 | # avoid loading the package before requirements are installed: 9 | version = SourceFileLoader('version', 'xdelta3/version.py').load_module() 10 | 11 | package = THIS_DIR.joinpath('xdelta3') 12 | package_data = ['_xdelta3.c'] 13 | if package.joinpath('lib').exists(): 14 | package_data += ['lib/' + f.name for f in package.joinpath('lib').iterdir()] 15 | 16 | setup( 17 | name='xdelta3', 18 | version=str(version.VERSION), 19 | description='Fast delta encoding using xdelta3', 20 | long_description=long_description, 21 | author='Samuel Colvin', 22 | author_email='s@muelcolvin.com', 23 | url='https://github.com/samuelcolvin/xdelta3-python', 24 | license='Apache License, Version 2.0', 25 | packages=['xdelta3'], 26 | package_data={ 27 | 'xdelta3': package_data 28 | }, 29 | zip_safe=True, 30 | ext_modules=[ 31 | Extension( 32 | '_xdelta3', 33 | sources=['xdelta3/_xdelta3.c'], 34 | include_dirs=['./xdelta3/lib'], 35 | # use with SECONDARY_LZMA to enabled secondary compression with lzma 36 | # libraries=['lzma'], 37 | define_macros=[ 38 | ('SIZEOF_SIZE_T', '8'), 39 | ('SIZEOF_UNSIGNED_LONG_LONG', '8'), 40 | ('XD3_USE_LARGEFILE64', '1'), 41 | # ('SECONDARY_LZMA', '1'), 42 | # adds verbose debug output to xdelta3 43 | # ('XD3_DEBUG', '3'), 44 | ] 45 | ) 46 | ], 47 | classifiers=[ 48 | 'Development Status :: 4 - Beta', 49 | 'Programming Language :: Python', 50 | 'Programming Language :: Python :: 3', 51 | 'Programming Language :: Python :: 3.5', 52 | 'Programming Language :: Python :: 3.6', 53 | 'License :: OSI Approved :: Apache Software License', 54 | 'Operating System :: Unix', 55 | 'Operating System :: POSIX :: Linux', 56 | 'Topic :: System :: Archiving :: Compression', 57 | ], 58 | ) 59 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/samuelcolvin/xdelta3-python/3b7ddaa1f46a29aac16cdb1bf5b00d6afa79a832/tests/__init__.py -------------------------------------------------------------------------------- /tests/build-wheels.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e -x 3 | 4 | yum install -y gcc 5 | gcc -v 6 | 7 | # Compile wheels 8 | /opt/python/cp36-cp36m/bin/pip install -r /io/tests/requirements.txt 9 | /opt/python/cp36-cp36m/bin/pip wheel /io/ -w wheelhouse/ 10 | 11 | # Bundle external shared libraries into the wheels 12 | for whl in wheelhouse/*.whl; do 13 | auditwheel repair "$whl" -w /io/wheelhouse/ 14 | done 15 | -------------------------------------------------------------------------------- /tests/check_tag.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | import sys 4 | 5 | from xdelta3.version import VERSION 6 | 7 | git_tag = os.getenv('TRAVIS_TAG') 8 | if git_tag: 9 | if git_tag.lower().lstrip('v') != str(VERSION).lower(): 10 | print('✖ "TRAVIS_TAG" environment variable does not match package version: "%s" vs. "%s"' % (git_tag, VERSION)) 11 | sys.exit(1) 12 | else: 13 | print('✓ "TRAVIS_TAG" environment variable matches package version: "%s" vs. "%s"' % (git_tag, VERSION)) 14 | else: 15 | print('✓ "TRAVIS_TAG" not defined, not releasing') 16 | -------------------------------------------------------------------------------- /tests/requirements.txt: -------------------------------------------------------------------------------- 1 | auditwheel==1.7.0 2 | coverage==4.4.2 3 | docutils==0.14 4 | flake8==3.5.0 5 | pycodestyle==2.3.1 6 | pyflakes==1.6.0 7 | pygments==2.2.0 8 | pytest==3.3.1 9 | pytest-cov==2.5.1 10 | pytest-isort==0.1.0 11 | pytest-sugar==0.9.0 12 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import os 3 | import string 4 | 5 | import pytest 6 | 7 | import xdelta3 8 | 9 | value_one = b'this is a short string to test with. It is suitable for delta encoding.' 10 | value_two = b'this is a different short string to test with. It is suitable for delta encoding.' 11 | expected_delta = b'\xd6\xc3\xc4\x00\x00\x01G\x00\x14Q\x00\t\x04\x02different\x1a\n\x13>\x00\t' 12 | 13 | 14 | def test_encode_decode(): 15 | delta = xdelta3.encode(value_one, value_two) 16 | assert delta == expected_delta 17 | value_two2 = xdelta3.decode(value_one, delta) 18 | assert value_two == value_two2 19 | 20 | 21 | def test_long_random(): 22 | v1 = base64.b32encode(os.urandom(1000)) 23 | v2 = b'x' + v1 + b'x' 24 | delta = xdelta3.encode(v1, v2) 25 | v22 = xdelta3.decode(v1, delta) 26 | assert v2 == v22 27 | 28 | 29 | def test_decode_error(): 30 | with pytest.raises(xdelta3.XDeltaError) as exc_info: 31 | xdelta3.decode(expected_delta, value_one) 32 | assert exc_info.value.args[0] == 'Error occur executing xdelta3: XD3_INVALID_INPUT' 33 | 34 | 35 | def test_no_delta(): 36 | with pytest.raises(xdelta3.NoDeltaFound) as exc_info: 37 | xdelta3.encode(b'hello', b'goodbye') 38 | assert exc_info.value.args[0] == 'No delta found shorter than the input value' 39 | 40 | 41 | def test_different_compression(): 42 | all_ascii = (string.ascii_letters + string.digits).encode() 43 | v1 = all_ascii * 1000 44 | v2 = all_ascii * 900 + string.ascii_letters.encode() * 100 + all_ascii * 100 45 | delta_a = xdelta3.encode(v1, v2, xdelta3.Flags.COMPLEVEL_9) 46 | v2_a = xdelta3.decode(v1, delta_a) 47 | assert v2 == v2_a 48 | 49 | delta_b = xdelta3.encode(v1, v2, xdelta3.Flags.COMPLEVEL_1) 50 | v2_b = xdelta3.decode(v1, delta_b) 51 | assert v2 == v2_b 52 | assert len(delta_a) < len(delta_b) 53 | 54 | 55 | def test_version(): 56 | # can't easily test output as capsys doesn't capture output of print_version 57 | xdelta3.print_version() 58 | 59 | 60 | def test_readme(): 61 | value_one = b'wonderful string to demonstrate xdelta3, much of these two strings is the same.' 62 | value_two = b'different string to demonstrate xdelta3, much of these two strings is the same.' 63 | delta = xdelta3.encode(value_one, value_two) 64 | 65 | value_two_rebuilt = xdelta3.decode(value_one, delta) 66 | assert value_two_rebuilt == value_two 67 | -------------------------------------------------------------------------------- /xdelta3/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .main import * 3 | from .version import VERSION 4 | -------------------------------------------------------------------------------- /xdelta3/_xdelta3.c: -------------------------------------------------------------------------------- 1 | #define NOT_MAIN 1 2 | 3 | #include "xdelta3.h" 4 | #include "xdelta3.c" 5 | #include 6 | 7 | static PyObject *NoDeltaFound; 8 | static PyObject *XDeltaError; 9 | 10 | static PyObject * xdelta3_execute(PyObject *self, PyObject *args) 11 | { 12 | uint8_t *input_bytes = NULL, *source_bytes = NULL, *output_buf = NULL; 13 | int input_len, source_len, flags, action, result; 14 | size_t input_size, source_size, output_alloc, output_size; 15 | 16 | if (!PyArg_ParseTuple(args, "y#y#ii", &input_bytes, &input_len, &source_bytes, &source_len, &flags, &action)) 17 | return NULL; 18 | 19 | source_size = (size_t)source_len; 20 | input_size = (size_t)input_len; 21 | 22 | if (action == 0) { 23 | // if the output would be longer than the input itself, there's no point using delta encoding 24 | output_alloc = input_size; 25 | output_buf = main_malloc(output_alloc); 26 | result = xd3_encode_memory(input_bytes, input_size, source_bytes, source_size, 27 | output_buf, &output_size, output_alloc, flags); 28 | } else { 29 | // output shouldn't be bigger than the original plus the delta, but give a little leeway 30 | output_alloc = input_size + source_size * 11 / 10; 31 | output_buf = main_malloc(output_alloc); 32 | result = xd3_decode_memory(input_bytes, input_size, source_bytes, source_size, 33 | output_buf, &output_size, output_alloc, flags); 34 | } 35 | 36 | if (result == 0) { 37 | PyObject *ret = Py_BuildValue("y#", output_buf, output_size); 38 | main_free(output_buf); 39 | return ret; 40 | } 41 | 42 | if(result == ENOSPC) { 43 | if (action == 0) { 44 | // all is well, just not efficient delta could be found 45 | PyErr_SetString(NoDeltaFound, "No delta found shorter than the input value"); 46 | } else { 47 | PyErr_SetString(XDeltaError, "Output of decoding delta longer than expected"); 48 | } 49 | } else { 50 | char exc_str[80]; 51 | sprintf(exc_str, "Error occur executing xdelta3: %s", xd3_strerror(result)); 52 | PyErr_SetString(XDeltaError, exc_str); 53 | 54 | } 55 | main_free(output_buf); 56 | return NULL; 57 | } 58 | 59 | static PyObject * xdelta3_version(PyObject *self, PyObject *args) 60 | { 61 | int result = main_version(); 62 | PyObject *ret = Py_BuildValue("i", result); 63 | return ret; 64 | } 65 | 66 | static PyMethodDef xdelta3_methods[] = { 67 | {"execute", xdelta3_execute, METH_VARARGS, "xdelta3 encode or decode"}, 68 | {"version", xdelta3_version, METH_VARARGS, "print xdelta3 version info"}, 69 | {NULL, NULL, 0, NULL} 70 | }; 71 | 72 | static struct PyModuleDef xdelta3_module = { 73 | PyModuleDef_HEAD_INIT, 74 | "_xdelta3", 75 | NULL, 76 | 0, 77 | xdelta3_methods 78 | }; 79 | 80 | PyMODINIT_FUNC PyInit__xdelta3(void) { 81 | PyObject *m; 82 | 83 | m = PyModule_Create(&xdelta3_module); 84 | if (m == NULL) 85 | return NULL; 86 | 87 | XDeltaError = PyErr_NewException("xdelta3.XDeltaError", NULL, NULL); 88 | Py_INCREF(XDeltaError); 89 | PyModule_AddObject(m, "XDeltaError", XDeltaError); 90 | 91 | NoDeltaFound = PyErr_NewException("xdelta3.NoDeltaFound", NULL, NULL); 92 | Py_INCREF(NoDeltaFound); 93 | PyModule_AddObject(m, "NoDeltaFound", NoDeltaFound); 94 | return m; 95 | } 96 | -------------------------------------------------------------------------------- /xdelta3/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from enum import IntEnum 3 | 4 | import _xdelta3 5 | from _xdelta3 import NoDeltaFound, XDeltaError # noqa 6 | 7 | from .version import VERSION 8 | 9 | __all__ = [ 10 | 'NoDeltaFound', 11 | 'XDeltaError', 12 | 'Flags', 13 | 'encode', 14 | 'decode', 15 | 'print_version', 16 | ] 17 | 18 | 19 | class Flags(IntEnum): 20 | """ 21 | Config flags taken from xdelta3/xdelta3/xdelta3.h 22 | 23 | XD3_ prefix removed for conciseness 24 | """ 25 | # used by VCDIFF tools, see xdelta3-main.h. 26 | JUST_HDR = 1 << 1 27 | # used by VCDIFF tools, see xdelta3-main.h. 28 | SKIP_WINDOW = 1 << 2 29 | # used by VCDIFF tools, see xdelta3-main.h. 30 | SKIP_EMIT = 1 << 3 31 | # flush the stream buffer to prepare for xd3_stream_close(). 32 | FLUSH = 1 << 4 33 | 34 | # use DJW static huffman 35 | SEC_DJW = 1 << 5 36 | # use FGK adaptive huffman 37 | SEC_FGK = 1 << 6 38 | # use LZMA secondary 39 | SEC_LZMA = 1 << 24 40 | 41 | SEC_TYPE = SEC_DJW | SEC_FGK | SEC_LZMA 42 | 43 | # disable secondary compression of the data section. 44 | SEC_NODATA = 1 << 7 45 | # disable secondary compression of the inst section. 46 | SEC_NOINST = 1 << 8 47 | # disable secondary compression of the addr section. 48 | SEC_NOADDR = 1 << 9 49 | 50 | SEC_NOALL = SEC_NODATA | SEC_NOINST | SEC_NOADDR 51 | 52 | # enable checksum computation in the encoder. 53 | ADLER32 = 1 << 10 54 | # disable checksum verification in the decoder. 55 | ADLER32_NOVER = 1 << 11 56 | 57 | # disable ordinary data * compression feature, only search * the source, not the target. 58 | NOCOMPRESS = 1 << 13 59 | # disable the "1.5-pass * algorithm", instead use greedy * matching. Greedy is off by * default. 60 | BEGREEDY = 1 << 14 61 | # used by "recode". 62 | ADLER32_RECODE = 1 << 15 63 | 64 | # 4 bits to set the compression level the same as the command-line setting -1 through -9 65 | # (-0 corresponds to the NOCOMPRESS flag, and is independent of compression level). This is for 66 | # convenience, especially with xd3_encode_memory(). 67 | 68 | COMPLEVEL_SHIFT = 20 69 | COMPLEVEL_MASK = 0xF << COMPLEVEL_SHIFT 70 | COMPLEVEL_1 = 1 << COMPLEVEL_SHIFT 71 | COMPLEVEL_2 = 2 << COMPLEVEL_SHIFT 72 | COMPLEVEL_3 = 3 << COMPLEVEL_SHIFT 73 | COMPLEVEL_6 = 6 << COMPLEVEL_SHIFT 74 | COMPLEVEL_9 = 9 << COMPLEVEL_SHIFT 75 | 76 | 77 | def encode(original: bytes, new_value: bytes, flags: int=Flags.COMPLEVEL_9) -> bytes: 78 | """ 79 | Encode a delta of new_value from original. 80 | 81 | Note the main two arguments original (aka. source) and new_value (aka input) are reversed 82 | here compared to xdelta3 methods as IMHO this makes more sense. 83 | 84 | :param original: source byte string 85 | :param new_value: new byte string differing partially from original 86 | :param flags: see Flags above 87 | :return: delta byte string 88 | """ 89 | return _xdelta3.execute(new_value, original, flags, 0) 90 | 91 | 92 | def decode(original: bytes, delta: bytes, flags: int=Flags.COMPLEVEL_9) -> bytes: 93 | """ 94 | Decode a delta to calculate a new value from the original and a delta. 95 | 96 | Note the main two arguments original (aka. source) and delta (aka input) are reversed 97 | here compared to xdelta3 methods as IMHO this makes more sense. 98 | 99 | :param original: source byte string 100 | :param delta: delta defining changes the original string 101 | :param flags: see Flags above 102 | :return: new byte string from applying delta to original 103 | """ 104 | return _xdelta3.execute(delta, original, flags, 1) 105 | 106 | 107 | def print_version(): 108 | """ 109 | Print version info for xdelta3-python and the xdelta3 c library. 110 | """ 111 | print('xdelta3-python:', VERSION, file=sys.stderr) 112 | print('xdelta3-c library:', file=sys.stderr) 113 | _xdelta3.version() 114 | -------------------------------------------------------------------------------- /xdelta3/version.py: -------------------------------------------------------------------------------- 1 | from distutils.version import StrictVersion 2 | 3 | __all__ = ['VERSION'] 4 | 5 | VERSION = StrictVersion('0.0.6a1') 6 | --------------------------------------------------------------------------------