├── tests ├── __init__.py ├── test_struct.py └── test_quickle.py ├── docs ├── source │ ├── _static │ │ ├── bench-1.png │ │ ├── custom.css │ │ ├── bench-1.json │ │ ├── bench-10k.json │ │ └── bench-1k.json │ ├── _templates │ │ └── help.html │ ├── api.rst │ ├── conf.py │ ├── faq.rst │ ├── benchmarks.rst │ └── index.rst ├── Makefile └── make.bat ├── MANIFEST.in ├── .gitignore ├── setup.cfg ├── benchmarks ├── build_proto.py ├── bench.proto └── bench.py ├── setup.py ├── tools └── build_manylinux.py ├── README.rst ├── github_deploy_key_jcrist_quickle.enc ├── LICENSE └── .travis.yml /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/source/_static/bench-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jcrist/quickle/HEAD/docs/source/_static/bench-1.png -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include quickle.c 2 | include setup.py 3 | include README.rst 4 | include LICENSE 5 | include MANIFEST.in 6 | -------------------------------------------------------------------------------- /docs/source/_static/custom.css: -------------------------------------------------------------------------------- 1 | .centered-radio .bk-inline { 2 | text-align: center; 3 | display: inline-block; 4 | } 5 | -------------------------------------------------------------------------------- /docs/source/_templates/help.html: -------------------------------------------------------------------------------- 1 |

Need help?

2 | 3 |

4 | Open an issue in the issue tracker. 5 |

6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | *.so 4 | *.pem 5 | build/ 6 | dist/ 7 | out/ 8 | docs/build/ 9 | .cache/ 10 | .coverage 11 | .pytest_cache/ 12 | .eggs/ 13 | .DS_Store 14 | htmlcov/ 15 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = 3 | E721, # Comparing types instead of isinstance 4 | E741, # Ambiguous variable names 5 | W503, # line break before binary operator 6 | W504, # line break after binary operator 7 | max-line-length = 95 8 | -------------------------------------------------------------------------------- /benchmarks/build_proto.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import shutil 4 | 5 | # Hacky, I know 6 | os.system("pyrobuf --proto3 --package proto_bench bench.proto") 7 | sos = glob.glob("build/*/*.so") 8 | assert len(sos) == 1 9 | shutil.move(sos[0], ".") 10 | -------------------------------------------------------------------------------- /benchmarks/bench.proto: -------------------------------------------------------------------------------- 1 | message Address { 2 | string street = 1; 3 | string state = 2; 4 | uint32 zip = 3; 5 | } 6 | 7 | message Person { 8 | string first = 1; 9 | string last = 2; 10 | uint32 age = 3; 11 | repeated Address addresses = 4; 12 | string telephone = 5; 13 | string email = 6; 14 | } 15 | 16 | message People { 17 | repeated Person people = 1; 18 | } 19 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | API Docs 2 | ======== 3 | 4 | .. currentmodule:: quickle 5 | 6 | Encoder 7 | ------- 8 | 9 | .. autoclass:: Encoder 10 | :members: dumps 11 | 12 | 13 | Decoder 14 | ------- 15 | 16 | .. autoclass:: Decoder 17 | :members: 18 | 19 | 20 | Struct 21 | ------ 22 | 23 | .. autoclass:: Struct 24 | :members: 25 | 26 | 27 | PickleBuffer 28 | ------------ 29 | 30 | .. autoclass:: PickleBuffer 31 | :members: 32 | 33 | 34 | Functions 35 | --------- 36 | 37 | .. autofunction:: quickle.dumps 38 | 39 | .. autofunction:: quickle.loads 40 | 41 | 42 | Exceptions 43 | ---------- 44 | 45 | .. autoexception:: QuickleError 46 | :show-inheritance: 47 | 48 | .. autoexception:: EncodingError 49 | :show-inheritance: 50 | 51 | .. autoexception:: DecodingError 52 | :show-inheritance: 53 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from setuptools import setup, find_packages 4 | from setuptools.extension import Extension 5 | 6 | ext_modules = [Extension("quickle", ["quickle.c"])] 7 | 8 | setup( 9 | name="quickle", 10 | version="0.4.0", 11 | maintainer="Jim Crist-Harif", 12 | maintainer_email="jcristharif@gmail.com", 13 | url="https://jcristharif.com/quickle/", 14 | project_urls={ 15 | "Documentation": "https://jcristharif.com/quickle/", 16 | "Source": "https://github.com/jcrist/quickle/", 17 | "Issue Tracker": "https://github.com/jcrist/quickle/issues", 18 | }, 19 | description="A quicker pickle", 20 | classifiers=[ 21 | "License :: OSI Approved :: BSD License", 22 | "Programming Language :: Python :: 3.8", 23 | "Programming Language :: Python :: 3.9", 24 | ], 25 | license="BSD", 26 | packages=find_packages(), 27 | ext_modules=ext_modules, 28 | long_description=( 29 | open("README.rst", encoding="utf-8").read() 30 | if os.path.exists("README.rst") 31 | else "" 32 | ), 33 | python_requires=">=3.8", 34 | zip_safe=False, 35 | ) 36 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | import quickle 2 | 3 | project = "quickle 🥒" 4 | copyright = "2020, Jim Crist-Harif" 5 | author = "Jim Crist-Harif" 6 | release = version = quickle.__version__ 7 | 8 | html_theme = "alabaster" 9 | extensions = [ 10 | "sphinx.ext.autodoc", 11 | "sphinx.ext.napoleon", 12 | "sphinx.ext.extlinks", 13 | "sphinx.ext.intersphinx", 14 | ] 15 | intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} 16 | napoleon_numpy_docstring = True 17 | napoleon_google_docstring = False 18 | default_role = "obj" 19 | pygments_style = "sphinx" 20 | 21 | templates_path = ["_templates"] 22 | html_static_path = ["_static"] 23 | html_theme_options = { 24 | "description": "A quicker pickle", 25 | "github_button": True, 26 | "github_count": False, 27 | "github_user": "jcrist", 28 | "github_repo": "quickle", 29 | "travis_button": False, 30 | "show_powered_by": False, 31 | "page_width": "960px", 32 | "sidebar_width": "200px", 33 | "code_font_size": "0.8em", 34 | } 35 | html_sidebars = {"**": ["about.html", "navigation.html", "help.html", "searchbox.html"]} 36 | extlinks = { 37 | "issue": ("https://github.com/jcrist/quickle/issues/%s", "Issue #"), 38 | "pr": ("https://github.com/jcrist/quickle/pull/%s", "PR #"), 39 | } 40 | -------------------------------------------------------------------------------- /tools/build_manylinux.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import subprocess 4 | import sys 5 | 6 | SCRIPT = """\ 7 | cd /tmp 8 | export HOME=/tmp 9 | for py in {pythons}; do 10 | "/opt/python/$py/bin/pip" wheel --no-deps --wheel-dir /tmp /dist/*.tar.gz 11 | done 12 | ls *.whl | xargs -n1 --verbose auditwheel repair --wheel-dir /dist 13 | ls -al /dist 14 | """ 15 | 16 | PACKAGE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | DIST_DIR = os.path.join(PACKAGE_DIR, "dist") 18 | 19 | 20 | def fail(msg): 21 | print(msg, file=sys.stderr) 22 | sys.exit(1) 23 | 24 | 25 | def main(): 26 | python_versions = "cp38-cp38" 27 | 28 | pythons = " ".join(python_versions.split(",")) 29 | 30 | sdists = glob.glob(os.path.join(DIST_DIR, "*.tar.gz")) 31 | if not sdists: 32 | fail("Must build sdist beforehand") 33 | elif len(sdists) > 1: 34 | fail("Must have only one sdist built") 35 | 36 | subprocess.check_call( 37 | [ 38 | "docker", 39 | "run", 40 | "-it", 41 | "--rm", 42 | "--volume", 43 | "{}:/dist:rw".format(DIST_DIR), 44 | "--user", 45 | "{}:{}".format(os.getuid(), os.getgid()), 46 | "quay.io/pypa/manylinux2010_x86_64:latest", 47 | "bash", 48 | "-o", 49 | "pipefail", 50 | "-euxc", 51 | SCRIPT.format(pythons=pythons), 52 | ] 53 | ) 54 | 55 | 56 | if __name__ == "__main__": 57 | main() 58 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | quickle 🥒 2 | =========== 3 | 4 | |travis| |pypi| |conda| 5 | 6 | **Quickle is no longer maintained** 7 | 8 | ``quickle`` was an interesting experiment, but I no longer believe this to be a 9 | good idea. For users looking for a fast and flexible serialization library for 10 | Python, I recommend using `msgspec `__ 11 | instead. Everything ``quickle`` could do, ``msgspec`` can do better and 12 | faster, all while using standardized protocols (JSON and MessagePack 13 | currently), rather than something Python-specific like Pickle. See `the docs 14 | `__ for more information. 15 | 16 | The original README is below: 17 | 18 | ---- 19 | 20 | ``quickle`` is a fast and small serialization format for a subset of Python 21 | types. It's based off of `Pickle 22 | `__, but includes several 23 | optimizations and extensions to provide improved performance and security. For 24 | supported types, serializing a message with ``quickle`` can be *~2-10x faster* 25 | than using ``pickle``. 26 | 27 | .. image:: https://github.com/jcrist/quickle/raw/master/docs/source/_static/bench-1.png 28 | :target: https://jcristharif.com/quickle/benchmarks.html 29 | 30 | See `the documentation `_ for more 31 | information. 32 | 33 | LICENSE 34 | ------- 35 | 36 | New BSD. See the 37 | `License File `_. 38 | 39 | .. |travis| image:: https://travis-ci.com/jcrist/quickle.svg?branch=master 40 | :target: https://travis-ci.com/jcrist/quickle 41 | .. |pypi| image:: https://img.shields.io/pypi/v/quickle.svg 42 | :target: https://pypi.org/project/quickle/ 43 | .. |conda| image:: https://img.shields.io/conda/vn/conda-forge/quickle.svg 44 | :target: https://anaconda.org/conda-forge/quickle 45 | -------------------------------------------------------------------------------- /docs/source/faq.rst: -------------------------------------------------------------------------------- 1 | FAQ 2 | === 3 | 4 | .. _why_not_pickle: 5 | 6 | Why not just use pickle? 7 | ------------------------ 8 | 9 | The builtin pickle_ module 10 | (or other extensions like cloudpickle_) can definitely support more 11 | types, but come with security issues if you're unpickling unknown data. From 12 | `the official docs`_: 13 | 14 | .. warning:: 15 | 16 | The ``pickle`` module **is not secure**. Only unpickle data you trust. 17 | 18 | It is possible to construct malicious pickle data which will **execute 19 | arbitrary code during unpickling**. Never unpickle data that could have come 20 | from an untrusted source, or that could have been tampered with. 21 | 22 | The pickle protocol contains instructions for loading and executing arbitrary 23 | python code - a maliciously crafted pickle could wipe your machine or steal 24 | secrets. ``quickle`` does away with those instructions, removing that 25 | security issue. 26 | 27 | The builtin ``pickle`` module also needs to support multiple protocols, and 28 | includes some optimizations for writing to/reading from files that result in 29 | slowdowns for users wanting fast in-memory performance (as required by 30 | networked services). For common payloads ``quickle`` can be ~2-10x faster at 31 | writing and ~1-3x faster at reading. 32 | 33 | 34 | Why not msgpack, json, etc? 35 | --------------------------- 36 | 37 | There are optimized versions of ``msgpack`` and ``json`` for Python that can be 38 | great for similar use cases. However, both ``msgpack`` and ``json`` have 39 | simpler object models than Python, which makes it tricky to roundtrip all the 40 | rich builtin types Python supports. 41 | 42 | - Both ``msgpack`` and ``json`` only support a single "array" type, which makes 43 | it hard to roundtrip messages where you want to distinguish lists from 44 | tuples. Or sets. 45 | - While ``msgpack`` supports both binary and unicode types, ``json`` requires 46 | all bytes be encoded into something utf8 compatible. 47 | - Quickle supports "memoization" - if a message contains the same object 48 | instance multiple times, it will only be serialized once in the payload. For 49 | messages where this may happen, this can result in a significant reduction in 50 | payload size. (note that ``quickle`` also contains an option to disable 51 | memoization if you don't need it, which can result in further speedups). 52 | - Quickle also supports recursive and self-referential objects, which will cause 53 | recursion errors in other serializers. While uncommon, there are use cases 54 | for such data structures, and quickle supports them natively. 55 | - With the introduction of the `Pickle 5 protocol 56 | `__, Pickle (and Quickle) supports 57 | sending messages containing large binary payloads in a zero-copy fashion. 58 | This is hard (or impossible) to do with either ``msgpack`` or ``json``. 59 | 60 | ``quickle`` is also competitive with common Python `msgpack 61 | `__ and `json 62 | `__ implementations. 63 | 64 | That said, if you're writing a network service that needs to talk to non-python 65 | things, ``json`` or ``msgpack`` will definitely serve you better. Even if 66 | you're writing something only in Python, you might still want to consider using 67 | something more standardized like ``json`` or ``msgpack``. 68 | 69 | When would I use this? 70 | ---------------------- 71 | 72 | I wanted this for writing RPC-style applications in Python. I was unsatisfied 73 | with ``json`` or ``msgpack``, since they didn't support all the rich types I'm 74 | used to in Python. And the existing pickle implementation added measurable 75 | per-message overhead when writing low-latency applications (not to mention 76 | security issues). If you don't have a similar use case, you may be better 77 | served elsewhere. 78 | 79 | .. _pickle: 80 | .. _the official docs: https://docs.python.org/3/library/pickle.html 81 | .. _cloudpickle: https://github.com/cloudpipe/cloudpickle 82 | -------------------------------------------------------------------------------- /github_deploy_key_jcrist_quickle.enc: -------------------------------------------------------------------------------- 1 | gAAAAABfT7YeomCYi0khtnewkbV5YLHyUzY_TNb2Yzt0RHuNzppM_nkq5qLpZBQd9Rr8q6gksPYyqxE8lvb4vtyrfuWnNxLBhb81Q6QN8P-US1Q-yR78-lD8PeWPCBU-hhiwOdqc865IwgwoDRT4grquTnUu5lhVSBg7uO8n8nmqRbljGhe1fDQx5xcIUmRSAxinz9zkdMXyzwwrvp-2vI68P-iF-IP0s1i3Zqf9CWHHJXALvD0ZvBVhi6Jqe4h96WK66XHd2thXqCDlljHuG5drK78I77n9xZYCYzH70LseE0IYlpBZCL5Vc9fz3b_g1zhGTpdJACdN1QzjpEyM0jzFgpMtd4mbWEtnF99HZmDvWVMXcxtlTqotUJT8dhLMeHx54Muhk2zhUAQIEbb2yURLxeqnjUu15cx3V57aPspbJrIc0270lypE2-WKaWWpSjEH2vaOfCv0talXTh9hBAcVgH0Wme-CXkyNmr_fInVedOJ0rkSS8AlMcQUHQs_agxYgvBCp9JlMOuOtsRynLQkG2sL0EE4wnAqZ6FBnCffeo8kBkBwsBVEDCRbwaTaT8UCn10oAuDuU_nRk8kfWXPv05d6qvINDQ2SwrQwa7p3CAz7wkJ7dK26EUN0yagq_0thjkUCiX1_YsxZfg4kjP8Wj6NRYTxjjFbnvV9IwmoAxSwo6ZwH0AXR_DzAtx5Wj4sODZql3Yct2jPdilvmbQTyQca7lpLDWR-bXa_YlnWknyI5eWraK4nl34qrBFKUx6984Zta3yOynS9NH_TvU4ngkGLutVH1sZxbrjsOQIp8GuMTaGkJOZ6TeTuJJG6oUI2JJ0_sCOB716cB6vf76byZ4gK2MlDbCrO0AGC2UVXSVoXi6EsEEuimAv6xKSpNWoUc8yt_ReclGHUbG_OKh4Ur7p-zaEaDJedudCjHYtpgfQKdRi1Gw8aQPZhguUXnr3QiXsy_WMGrioQ4x4F0etFLNh2tJbqJ-0jrMbQWD58Xir6o2WbMje6Y5wxofWQDqfpZCze-4MGPJZ9nzx6q9VqMeIT4zPfvg64rozT1qiaj27Ib5PyjzD3Qces4j7vvQ_nAPvk3wGcfRR24tp5CquIGqXs6_gfoUUNvI2c4nwR9VM6f1yUktun2vzwqlpRpqzLWrYGrlEq2E0mvKFU0pLuI3VnjrQM_Qpo3pDuYA7Ye1Hx-6uWLhytLuqK0WwVIy1z2rQDYV9siKQ5R4gK8PnO1howsbPsXlSrnprpzfAVLDf1m-YFFZBoDIlO2r3MPqaqm90k7pzpm3DVjX0783u5YvFNMOI3287w6sO98agWKB5jGCU5K3jQP4Znot1gmes2JQ8LsKq9-sVC4u2BdmoUIS8cNZkFnsmfdI5S5HNXxC57NkDHHgd1vUoxp9aRrjlb3I3cMrR7Gor6HQEW0B_zP-KShMZhuKzPuZYxsoqwlgBCpX5KBRTU0BLrKyQhVOxNjZG-lgMnnXPlyr1Necc7rZB28IxCHOsxt_Hc4LwB8E7_Bvdzpd0iwfMGdHObuFy2rFUv91IL0VQSQfrmTzxClKY4IUEZKEw2E4vacayP0gbWDkvEYnm9Pwj0derLOyX9_yIlDyqUeiv4_HQFbZVNzGLeaeHz_15xN1bRDiP2_Dh0rgmCQz3dlkQyvqLKLk6KCNZBUOBHGqzOoi59gVxIv2myFzZ0Zy7YuJ1YgpwxdkmJBOlOId38rLDNO4bCk1IxUhwo8QrnjaM0vi_WxJ9QeGY0AbnWn4kxwzCgvpAitswSYwzf16JUWoPYQGLukaHiNR_U3lhFU0fc7o2nn16N03iyM623kxb5WTQVl82vLxenEY5CDJAtjoeo5HB3W9eZ5WHlqZwNBAIToSAD7u95tSfGWQYSzVYpTY8pkbvKg4z2_465TUnxQOaJKYdkrbV_sCnGHKu3XkjAPl2VSkk9xoV7atWzV8a47ztxcQr2xMV_G6mAipd5KsgKV3zwKPH-mLY03dS7Yg5dQEy_LyeabIxrNVgYCdiw2iXwymOM4qTbgo9Vr4xrDwqcihJVvGREcNt_95TsybhiRvpnGTaVxDgn7jcQ3OdnmXp_Qp5rOkeiQoB4E-fPeUfs_HkXTQm5m3E1eNqV54TdNk0pB4RC-Y8J4SgoXZ-rrF7exOdF3ZhiElGktrspkGtUyU33a-3pH3g1KfQ_uzaV-DdsJA5m_SP1NXFuhaWHGHtkYksWscjG0ASN8ngx91gypnZerVGxjCyhbFRZdgsAB14tP4A9P_ZBn2o5TipQHLwG8BlrzpYsTRBfGIYdy1OYKOdnSJHYAwoYKO6Fq5t1faOr4TZNeL7_1-R1Z239igqTUvOCIy_1wZflI_94wYGRGXlpAeZqVQzmQbMf3isAsOXdDiEmGFfzgaAOhGvJHqAhj_foUy8oXDAAMv1nyJdcZMlU4mmkKEJxu1ympGIh3s656Z0nb6yJQFhyklPpPkLXPWL33D3dDFEsGFAaHJS7Elidi9duhAmYhASE_qwHlnPrusyPgAerFZ3KJB97If6ahDN-RHe_JZqezZxFuyvfL_Px3SBrBkKUcJQ5gOivcUrQzjv2DSCHvcgczXoUZpkCRKCWpNS1m9h7OdFsVrRDHuiRyAMVGxipIXQQO_CUra-h8705yDFD3xOogYFTl89-VHFjA_dyFgR2KYMLmSP0puLPzDtov5h71qTiOmMqMqEnu1Zef-U_TZQqgqCe_mmUHNltU9LlySc4V0ZTkUd5dvSNDtFs61dexv8Ri2BFitdfRpV6awziecxmC_r8YTI6tQXVf4342kx_dWIeXB561PTrjdHcpV_JQDNv8Wh8XUQ5-WAuJ_iIGgw2-mq0Pxa8pLqduUoHq-E0hlJWXbNlSdIHiTwNqsqJo8MtlnGbb9txutBGx5RZTyT6eGSeCJLmguJQkKMZlllv6GZooYou-aIFWyWYr9R3nNgRIqXBunoAEPE2J4LYtRCydUu3Pbs3sTMDfMucf6BU1YkeTEEGbkFRf8dx-dHD-zvJQ3cnEwn0aQj9Z-6U_IjtC_9b2RAzxPOgOgHMVlM0DqEWbkiQga711H2mA4sS2HL7heS2SxGBFgolPVjMEk177JrcVAXieo6lfKGv_u9rpVHeD1X9ZWu6VCY0ra737mpNLKvb2BumqjiWBpAPSwBr1li5k2BHLs1kgoXHm96pLI9aXMEH7m2A1YaaXEi15HiDJKCZU_chUQmhR50XCRHofb-IT9IbvA3EHSyozpJ8jBk7SO0oGmFU3R-Ih3e6bRWpxcAqsxL3QlKWTSQGzvq98aWj4tAa3FNQmu6gtM-Hu23xp9pnxnR5Ysf4jKuLD51ySiNlAZ4KKureHdB5UQAhsyY5NkEnglgF7H4AlT7J2ghgC561GC0VeD-gcoONFMrXdFFOpSzdU9zJNqR88DzuQ4MWBEMNpcnovWrCqhG8JfT0oaP_L_e9dQ8k9v_3ch1Wpbec2YGTpwGhc4B6p8B7xj1OZnijVBhGW1L01i9NFbUNg2VPfVTgaf26G_thzMQncmX27keWycw2rb-OqJ6yf_hI5S5ndQyu3bQ2a5YaafHe_bhTpBAIh_6dTk1OAhgDyzfl2OTrGmdJk1gtpWcvdQbSR6H5_qz36YH2rCPqLxN3Js61qbQ5sGqygciXYDE2WsGYdBsqTePDhj5OCH1eHoEVLAR179WrxP4brWM4k-8OuFo8iDQ26Po9yyx138d0AyfDS2z4yiLOd225d3LbymXZE7brM6J6guT4TCOwuCkbzpr-vISqvUmlNaCVfg0Q--8ureSCF4Dz44AOh6FMQcMw5ngKi2bNDYemRjpgTWpYJi_UR7oBwCnWHdvMX40sIKO8l3H120IScaAkJ48oPl4PtVLkJcfAHhv-QDV75MKQcvpJhxN3pkcqYpBJSyDkgglsgitGCieQXvs2SrNIpWu__dyaFyZQsVildkjffW0pK_owsx4pNJhzu5b1vL1ndXG1ysGViU3IRBl-3azU7YLHRgJzmQti-e7wov1nFBo55k0wG9lhy5gxAgslkBYyyTAIBaSuX5SlX41AekAbUHu5jPxCUUxT1VzGIbh8gUxy_hYTNZ1uu0znoDJDyDtW9AJCfOPbUJMxJT8wYR9SUOmbJ7XdVbuuk1zngNd7rHb7aOrOzoaihEJGsPfbW-RfzXpS0nvDwXOp_dv5wrKajHNKX2MA2K0kXk-3pgTBR-gPhtdipW8lm4T-elmv90yztUVefTfIpNfSQM5u8BIzlSJ4pgEBbqzsTa3AAUThDe_5nTY5Uy6UT1-aKNzIWYbW72GmFSovtQpwgABoviKssDFISoBKOQS02mNCD6sHrTb4x9aC1tBkdD_QS3IsyVMH3n4oWjWFf_xrDUG1powInZHjMidliBruj84xbkQ7WiuUdyChPlv5zfsgRsDQ1q5Rc2qoTSdX92ikf7Yi8YS1iGJw== -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020, Jim Crist-Harif 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | 30 | *************************************************************************** 31 | * This software is a fork of the `pickle` module from Python 3.8.3rc1. As * 32 | * a derivative work, the original PSF license is included below. * 33 | *************************************************************************** 34 | 35 | 36 | PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 37 | -------------------------------------------- 38 | 39 | 1. This LICENSE AGREEMENT is between the Python Software Foundation 40 | ("PSF"), and the Individual or Organization ("Licensee") accessing and 41 | otherwise using this software ("Python") in source or binary form and 42 | its associated documentation. 43 | 44 | 2. Subject to the terms and conditions of this License Agreement, PSF hereby 45 | grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, 46 | analyze, test, perform and/or display publicly, prepare derivative works, 47 | distribute, and otherwise use Python alone or in any derivative version, 48 | provided, however, that PSF's License Agreement and PSF's notice of copyright, 49 | i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 50 | 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Python Software Foundation; 51 | All Rights Reserved" are retained in Python alone or in any derivative version 52 | prepared by Licensee. 53 | 54 | 3. In the event Licensee prepares a derivative work that is based on 55 | or incorporates Python or any part thereof, and wants to make 56 | the derivative work available to others as provided herein, then 57 | Licensee hereby agrees to include in any such work a brief summary of 58 | the changes made to Python. 59 | 60 | 4. PSF is making Python available to Licensee on an "AS IS" 61 | basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR 62 | IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND 63 | DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS 64 | FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT 65 | INFRINGE ANY THIRD PARTY RIGHTS. 66 | 67 | 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 68 | FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS 69 | A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, 70 | OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 71 | 72 | 6. This License Agreement will automatically terminate upon a material 73 | breach of its terms and conditions. 74 | 75 | 7. Nothing in this License Agreement shall be deemed to create any 76 | relationship of agency, partnership, or joint venture between PSF and 77 | Licensee. This License Agreement does not grant permission to use PSF 78 | trademarks or trade name in a trademark sense to endorse or promote 79 | products or services of Licensee, or any third party. 80 | 81 | 8. By copying, installing or otherwise using Python, Licensee 82 | agrees to be bound by the terms and conditions of this License 83 | Agreement. 84 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | notifications: 2 | email: false 3 | 4 | env: 5 | global: 6 | - secure: "C5o5hO1RDdoJCOR5h7hAdaeTt7ygQhQ7XrbFaYhicw7oOE0ezM8Qhd/t90I4exBu9r+FrqRZd6zgyDwknlMVzFGbRLg4DNNx3RHszHPHO1nZw4sTtOVC93ygsFDpQnzX81cYLPUb2JnIUz1WNeUYt8BBOGPgZGWV9kplksYtWEcYCrT681cB3DaJ0/SKfhYobTJU/D6wkRF5UxtpyDPS/1N4KN3rNf+BRR/j1X2IDEnDoCnVJxwTVTUK8GzcHd5EFLexGTDFb2x6qjk5cYPl21ic52KSw1u1pKT0nBzCBvHNBTRbdHq8S6qT8AH4MU1tKD6Clj+0Sa1AMokKDo9RmjIwKYIcgWliIEWpl+C72WJJ7P9vC8uAfQlRY/gm5ST3m4yFLDTZrvUCOBPJgaPwueCmzOl0Hw0cmh41bBWYXdJuxREfZgNt2h6xzP8JbDs+tBhjig1BUAWix7lZ6Phuk60TX2iSNYEqUZFjct/tqjJiGXP9F1OrK6Nja5okIuA6k138T8OlUfJ7brnMraHUmH6eVx2r600G1oqnjqEJRc7Kuy2o6lTfdoBdMJYCwT2sEd1UdaO2/NJasCdktv+yCFjsxFCg9lGZHPDG+bW9KMl++hqePJaOhynRMHJ8WleSxVjm2UF3rZCEp1U0tmCyspigmqdfWL34YbbZrzQuuLs=" 7 | - CIBW_BUILD="cp38-* cp39-*" 8 | - CIBW_TEST_REQUIRES=pytest 9 | - CIBW_TEST_COMMAND="pytest {project}/tests" 10 | - TWINE_USERNAME=__token__ 11 | - secure: "X3l7EJggx639ZKERCi4eYbr8Gl6DqKEWSlovnSqERCa+JQycGnovRmLNZvT1Rol4N+1blJHBcPPLr9HDqCQxLIMMyIKe9xE+MT3sKAw0OI01tfNULm+YcAPFja0giXZtp+7952wcbEokOF6TBR7gv/5AB3rvxLGnaSLW9LlR7rNyi1vPgFeh6KNjTrT8Lpsp8m9tbCWPzXL7O4PfLJORc1YrQD5Y6XR0U878jjLPxSOOlIw7Mwup6BKoXpZkWb3k3vKideckC8M9KArTr4vLJCQmFmRLGBg8mkYKmvoUoeQuwfqaBonyM+U1+l1I+SxnHIVe/rLXusuOdElKig3oFDNGTuA8gSHg1peuV5S3aGsshdmwFyhUdEdMJqQI00ZfuHmuYhc63L++h7iecxI5Dc9mL4T6pHZs11ptCUUpaRTLsxGg0HKMFvcRxuqG/3ZaFxJDknAJc6qzm2oydd3F8I3TAEIIlwkL5BrfTuDncHZH1YPo7Vkbpwsq4wBVXbQlTmEOuRQpRuygm2J+mwGGSEQfOh0n6H2YEXFl0yVBDEdbosJKFAO+vy1gwdn1JpQWxluil1Bcfbi/DE/3GFkn33XGg+l9XW7o1m9S3lIU6B9h8Oanx4VbXJn5mJmQEPHNm/xCquO5a4lxGz9udiizubn7yba6KBzilWJtEyBRg2c=" 12 | 13 | if: type != push OR branch = master OR branch =~ /\d+\.\d+(\.\d+)?(-\S*)?$/ 14 | 15 | stages: 16 | - test 17 | - name: deploy 18 | if: tag IS present AND repo = jcrist/quickle 19 | 20 | jobs: 21 | fast_finish: true 22 | 23 | include: 24 | - name: linux-tests 25 | stage: test 26 | language: python 27 | python: 28 | - "3.8" 29 | install: 30 | - pip install pytest black flake8 sphinx 31 | - pip install -e . 32 | script: 33 | - set -e 34 | - pytest -v 35 | - flake8 36 | - black --check . 37 | - | 38 | pushd docs 39 | make html 40 | popd 41 | if [[ "$TRAVIS_BRANCH" == "master" && "$TRAVIS_EVENT_TYPE" == "push" ]]; then 42 | pip install doctr 43 | doctr deploy . --built-docs docs/build/html/ 44 | fi 45 | - name: linux-tests-39 46 | stage: test 47 | language: python 48 | python: 49 | - "3.9-dev" 50 | install: 51 | - pip install pytest 52 | - pip install -e . 53 | script: 54 | - set -e 55 | - pytest -v 56 | - name: windows-tests 57 | stage: test 58 | os: windows 59 | language: shell 60 | before_install: 61 | - choco install python --version 3.8.0 62 | - export PATH="/c/Python38:/c/Python38/Scripts:$PATH" 63 | - ln -s /c/Python38/python.exe /c/Python38/python3.exe 64 | install: 65 | - python3 -m pip install -U pytest 66 | - python3 -m pip install -e . 67 | - pytest -v 68 | # Deploy source distribution 69 | - name: deploy-sdist 70 | stage: deploy 71 | language: python 72 | python: 73 | - "3.8" 74 | script: python3 setup.py sdist --formats=gztar 75 | after_success: | 76 | python3 -m pip install twine 77 | python3 -m twine upload --skip-existing dist/*.tar.gz 78 | # Deploy on linux 79 | - name: deploy-wheel-linux 80 | stage: deploy 81 | language: python 82 | python: 83 | - "3.8" 84 | services: docker 85 | install: python3 -m pip install cibuildwheel==1.6.3 86 | script: python3 -m cibuildwheel --output-dir wheelhouse 87 | after_success: | 88 | python3 -m pip install twine 89 | python3 -m twine upload --skip-existing wheelhouse/*.whl 90 | # Deploy on mac 91 | - name: deploy-wheel-macos 92 | stage: deploy 93 | os: osx 94 | language: shell 95 | install: python3 -m pip install cibuildwheel==1.6.3 96 | script: python3 -m cibuildwheel --output-dir wheelhouse 97 | after_success: | 98 | python3 -m pip install twine 99 | python3 -m twine upload --skip-existing wheelhouse/*.whl 100 | # Deploy on windows 101 | - name: deploy-wheel-windows 102 | stage: deploy 103 | os: windows 104 | language: shell 105 | before_install: 106 | - choco install python --version 3.8.0 107 | - export PATH="/c/Python38:/c/Python38/Scripts:$PATH" 108 | - ln -s /c/Python38/python.exe /c/Python38/python3.exe 109 | install: python3 -m pip install cibuildwheel==1.6.3 110 | script: python3 -m cibuildwheel --output-dir wheelhouse 111 | after_success: | 112 | python3 -m pip install twine 113 | python3 -m twine upload --skip-existing wheelhouse/*.whl 114 | -------------------------------------------------------------------------------- /docs/source/benchmarks.rst: -------------------------------------------------------------------------------- 1 | Benchmarks 2 | ========== 3 | 4 | .. note:: 5 | 6 | Benchmarks are *hard*. 7 | 8 | Repeatedly calling the same function in a tight loop will lead to the 9 | instruction cache staying hot and branches being highly predictable. That's 10 | not representative of real world access patterns. It's also hard to write a 11 | nonbiased benchmark. I wrote quickle, naturally whatever benchmark I 12 | publish it's going to perform well in. 13 | 14 | Even so, people like to see benchmarks. I've tried to be as nonbiased as I 15 | can be, and the results hopefully indicate a few tradeoffs you make when 16 | you choose different serialization formats. I encourage you to write your 17 | own benchmarks before making these decisions. 18 | 19 | Here we show a simple benchmark serializing some structured data. The data 20 | we're serializing has the following schema (defined here using `quickle.Struct` 21 | types): 22 | 23 | .. code-block:: python 24 | 25 | import quickle 26 | from typing import List, Optional 27 | 28 | class Address(quickle.Struct): 29 | street: str 30 | state: str 31 | zip: int 32 | 33 | class Person(quickle.Struct): 34 | first: str 35 | last: str 36 | age: int 37 | addresses: Optional[List[Address]] = None 38 | telephone: Optional[str] = None 39 | email: Optional[str] = None 40 | 41 | The libraries we're benchmarking are the following: 42 | 43 | - ``msgpack`` - msgpack_ with dict message types 44 | - ``orjson`` - orjson_ with dict message types 45 | - ``pyrobuf`` - pyrobuf_ with protobuf message types 46 | - ``pickle`` - pickle_ with dict message types 47 | - ``pickle tuples`` - pickle_ with `collections.namedtuple` message types 48 | - ``quickle`` - quickle_ with dict message types 49 | - ``quickle structs`` - quickle_ with `quickle.Struct` message types 50 | 51 | Each benchmark creates one or more instances of a ``Person`` message, and 52 | serializes it/deserializes it in a loop. The full benchmark code can be found 53 | `here `__. 54 | 55 | Benchmark - 1 Object 56 | -------------------- 57 | 58 | Some workflows involve sending around very small messages. Here the overhead 59 | per function call dominates (parsing of options, allocating temporary buffers, 60 | etc...). Libraries like ``quickle`` and ``msgpack``, where internal structures 61 | are allocated once and can be reused will generally perform better here than 62 | libraries like ``pickle``, where each call needs to allocate some temporary 63 | objects. 64 | 65 | .. raw:: html 66 | 67 |
68 | 69 | .. note:: 70 | 71 | You can use the radio buttons on the bottom to sort by total roundtrip 72 | time, dumps (serialization) time, loads (deserialization) time, or 73 | serialized message size. 74 | 75 | From the chart above, you can see that ``quickle structs`` is the fastest 76 | method for both serialization and deserialization. It also results in the 77 | second smallest message size (behind ``pyrobuf``). This makes sense, struct 78 | types don't need to serialize the fields in each message (things like 79 | ``first``, ``last``, ...), only the values, so there's less data to send 80 | around. Since python is dynamic, each object serialized requires a few pointer 81 | chases, so serializing fewer objects results in faster and smaller messages. 82 | 83 | I'm actually surprised at how much overhead ``pyrobuf`` has (the actual 84 | protobuf encoding should be pretty efficient), I suspect there's some 85 | optimizations that could still be done there. 86 | 87 | That said, all of these methods serialize/deserialize pretty quickly relative 88 | to other python operations, so unless you're counting every microsecond your 89 | choice here probably doesn't matter that much. 90 | 91 | 92 | Benchmark - 1000 Objects 93 | ------------------------ 94 | 95 | Here we serialize a list of 1000 ``Person`` objects. There's a lot more data 96 | here, so the per-call overhead will no longer dominate, and we're now measuring 97 | the efficiency of the encoding/decoding. 98 | 99 | .. raw:: html 100 | 101 |
102 | 103 | As with before ``quickle structs`` and ``quickle`` both perform well here. 104 | What's interesting is that ``msgpack`` and ``orjson`` have now moved to the 105 | back for deserialization time. 106 | 107 | The reason for this is *memoization*. Since each message here is structured 108 | (all dicts have the same keys), ``msgpack`` and ``orjson`` are serializing the 109 | same strings multiple times. In contrast, ``quickle`` and ``pickle`` both 110 | support memoization - identical objects in a message will only be serialized 111 | once, and then referenced later on. This results in smaller messages and faster 112 | deserialization times. For messages without repeat objects, memoization is an 113 | added cost you don't need. But as soon as you get more than a handful of 114 | repeat objects, the performance win becomes important. 115 | 116 | Note that ``quickle structs``, ``pickle tuples``, and ``pyrobuf`` don't require 117 | memoization to be efficient here, as the repeated field names aren't serialized 118 | as part of the message. 119 | 120 | 121 | Benchmark - 10,000 Objects 122 | -------------------------- 123 | 124 | Here we run the same benchmark as before, but 10,000 ``Person`` objects. 125 | 126 | .. raw:: html 127 | 128 |
129 | 130 | Like the 1000 object benchmark, the cost of serializing/deserializing repeated 131 | strings dominate for the ``orjson`` and ``msgpack`` benchmarks. 132 | 133 | 134 | .. raw:: html 135 | 136 | 137 | 138 | 149 | 150 | 151 | .. _msgpack: https://github.com/msgpack/msgpack-python 152 | .. _orjson: https://github.com/ijl/orjson 153 | .. _pyrobuf: https://github.com/appnexus/pyrobuf 154 | .. _pickle: https://docs.python.org/3/library/pickle.html 155 | .. _quickle: https://jcristharif.com/quickle/ 156 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Quickle 2 | ======= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Contents: 7 | 8 | 9 | ``quickle`` is a fast and small serialization format for a subset of Python 10 | types. It's based off of `Pickle 11 | `__, but includes several 12 | optimizations and extensions to provide improved performance and security. For 13 | supported types, serializing a message with ``quickle`` can be *~2-10x faster* 14 | than using ``pickle``. 15 | 16 | 17 | Highlights 18 | ---------- 19 | 20 | - Quickle is **fast**. :doc:`benchmarks` show it's among the fastest 21 | serialization methods for Python. 22 | - Quickle is **safe**. :ref:`Unlike pickle `, deserializing a 23 | user provided message doesn't allow for arbitrary code execution. 24 | - Quickle is **flexible**. Unlike ``msgpack`` or ``json``, Quickle natively 25 | supports a wide range of Python builtin types. 26 | - Quickle supports :ref:`"schema evolution" `. Messages can 27 | be sent between clients with different schemas without error. 28 | 29 | Installation 30 | ------------ 31 | 32 | Quickle can be installed via ``pip`` or ``conda``. Note that Python >= 3.8 is 33 | required. 34 | 35 | **pip** 36 | 37 | .. code-block:: shell 38 | 39 | pip install quickle 40 | 41 | **conda** 42 | 43 | .. code-block:: shell 44 | 45 | conda install -c conda-forge quickle 46 | 47 | 48 | .. currentmodule:: quickle 49 | 50 | 51 | Usage 52 | ----- 53 | 54 | Like ``pickle``, ``quickle`` exposes two functions `dumps` and `loads`, for 55 | serializing and deserializing objects respectively. 56 | 57 | .. code-block:: python 58 | 59 | >>> import quickle 60 | >>> data = quickle.dumps({"hello": "world"}) 61 | >>> quickle.loads(data) 62 | {'hello': 'world'} 63 | 64 | Note that if you're making multiple calls to `dumps` or `loads`, it's more 65 | efficient to create an `Encoder` and `Decoder` once, and then use 66 | `Encoder.dumps` and `Decoder.loads`. 67 | 68 | .. code-block:: python 69 | 70 | >>> enc = quickle.Encoder() 71 | >>> dec = quickle.Decoder() 72 | >>> data = enc.dumps({"hello": "world"}) 73 | >>> dec.loads(data) 74 | {'hello': 'world'} 75 | 76 | Supported Types 77 | ~~~~~~~~~~~~~~~ 78 | 79 | Quickle currently supports serializing the following types: 80 | 81 | - `None` 82 | - `bool` 83 | - `int` 84 | - `float` 85 | - `complex` 86 | - `str` 87 | - `bytes` 88 | - `bytearray` 89 | - `tuple` 90 | - `list` 91 | - `dict` 92 | - `set` 93 | - `frozenset` 94 | - `datetime.date` 95 | - `datetime.time` 96 | - `datetime.datetime` 97 | - `datetime.timedelta` 98 | - `datetime.timezone` 99 | - `zoneinfo.ZoneInfo` 100 | - `enum.Enum` 101 | - `quickle.PickleBuffer` 102 | - `quickle.Struct` 103 | 104 | 105 | Structs and Enums 106 | ~~~~~~~~~~~~~~~~~ 107 | 108 | Quickle can serialize most builtin types, but unlike pickle, it can't serialize 109 | arbitrary user classes. This is due to security concerns - deserializing 110 | arbitrary python objects requires executing arbitrary python code. 111 | Deserializing a malicious message could wipe your machine or steal secrets (see 112 | :ref:`why_not_pickle` for more information). 113 | 114 | Quickle does support serializing two non-builtin types: 115 | 116 | - `Struct` 117 | - `enum.Enum` 118 | 119 | Structs are useful for defining structured messages. Fields are defined using 120 | python type annotations (the type annotations themselves are ignored, only the 121 | field names are used). Defaults values can also be specified for any optional 122 | arguments. 123 | 124 | Here we define a struct representing a person, with two required fields and two 125 | optional fields. 126 | 127 | .. code-block:: python 128 | 129 | >>> class Person(quickle.Struct): 130 | ... """A struct describing a person""" 131 | ... first : str 132 | ... last : str 133 | ... address : str = "" 134 | ... phone : str = None 135 | 136 | Struct types automatically generate a few methods based on the provided type 137 | annotations: 138 | 139 | - ``__init__`` 140 | - ``__repr__`` 141 | - ``__copy__`` 142 | - ``__eq__`` & ``__ne__`` 143 | 144 | .. code-block:: python 145 | 146 | >>> harry = Person("Harry", "Potter", address="4 Privet Drive") 147 | >>> harry 148 | Person(first='Harry', last='Potter', address='4 Privet Drive', phone=None) 149 | >>> harry.first 150 | "Harry" 151 | >>> ron = Person("Ron", "Weasley", address="The Burrow") 152 | >>> ron == harry 153 | False 154 | 155 | It is forbidden to override ``__init__``/``__new__`` in a struct definition, 156 | but other methods can be overridden or added as needed. The struct fields are 157 | available via the ``__struct_fields__`` attribute (a tuple of the fields in 158 | argument order ) if you need them. Here we add a method for converting a struct 159 | to a dict. 160 | 161 | .. code-block:: python 162 | 163 | >>> class Point(quickle.Struct): 164 | ... """A point in 2D space""" 165 | ... x : float 166 | ... y : float 167 | ... 168 | ... def to_dict(self): 169 | ... return {f: getattr(self, f) for f in self.__struct_fields__} 170 | ... 171 | >>> p = Point(1.0, 2.0) 172 | >>> p.to_dict() 173 | {"x": 1.0, "y": 2.0} 174 | 175 | Struct types are written in C and are quite speedy and lightweight. They're 176 | great for defining structured messages both for serialization and for use in an 177 | application. 178 | 179 | To add serialization support for `Struct` types, you need to register them with 180 | an `Encoder` and `Decoder`. 181 | 182 | .. code-block:: python 183 | 184 | >>> enc = quickle.Encoder(registry=[Person, Point]) 185 | >>> dec = quickle.Decoder(registry=[Person, Point]) 186 | >>> data = enc.dumps(harry) 187 | >>> dec.loads(data) 188 | Person(first='Harry', last='Potter', address='4 Privet Drive', phone=None) 189 | 190 | Unregistered types will fail to serialize or deserialize. Note that for 191 | deserialization to be successful the registry of the `Decoder` must match that 192 | of the `Encoder`. 193 | 194 | Like `Struct` types, `enum.Enum` types also need to be registered before 195 | they can be serialized: 196 | 197 | .. code-block:: python 198 | 199 | >>> import enum 200 | >>> class Fruit(enum.IntEnum): 201 | ... APPLE = 1 202 | ... BANANA = 2 203 | ... ORANGE = 3 204 | ... 205 | >>> enc = quickle.Encoder(registry=[Fruit]) 206 | >>> dec = quickle.Decoder(registry=[Fruit]) 207 | >>> data = enc.dumps(Fruit.APPLE) 208 | >>> dec.loads(data) 209 | 210 | 211 | .. _schema-evolution: 212 | 213 | Schema Evolution 214 | ~~~~~~~~~~~~~~~~ 215 | 216 | Quickle includes support for "schema evolution", meaning that: 217 | 218 | - Messages serialized with an older version of a schema will be deserializable 219 | using a newer version of the schema. 220 | - Messages serialized with a newer version of the schema will be deserializable 221 | using an older version of the schema. 222 | 223 | This can be useful if, for example, you have clients and servers with 224 | mismatched versions. 225 | 226 | For schema evolution to work smoothly, you need to follow a few guidelines when 227 | defining and registering new `Struct` and `enum.Enum` types: 228 | 229 | 1. Any new fields on a struct must be added to the end of the struct 230 | definition, and must contain default values. *Do not reorder fields in a 231 | struct.* 232 | 2. Any new `Struct` or `enum.Enum` types must be appended to the end of the 233 | registry. *Do not reorder types in the registry.* 234 | 235 | For example, suppose we wanted to add a new ``email`` field to our ``Person`` 236 | struct. To do so, we add it at the end of the definition, with a default value. 237 | 238 | .. code-block:: python 239 | 240 | >>> class Person2(quickle.Struct): 241 | ... """A struct describing a person""" 242 | ... first : str 243 | ... last : str 244 | ... address : str = "" 245 | ... phone : str = None 246 | ... email : str = None # added at the end, with a default 247 | ... 248 | >>> vernon = Person2("Vernon", "Dursley", address="4 Privet Drive", email="vernon@grunnings.com") 249 | 250 | Messages serialized using the new and old schemas can still be exchanged 251 | without error. 252 | 253 | .. code-block:: python 254 | 255 | >>> old_enc = quickle.Encoder(registry=[Person]) 256 | >>> old_dec = quickle.Decoder(registry=[Person]) 257 | >>> new_enc = quickle.Encoder(registry=[Person2]) 258 | >>> new_dec = quickle.Decoder(registry=[Person2]) 259 | 260 | >>> new_msg = new_enc.dumps(vernon) 261 | >>> old_dec.loads(new_msg) # deserializing a new msg with an older decoder 262 | Person(first="Vernon", last="Dursley", address="4 Privet Drive", phone=None) 263 | 264 | >>> old_msg = old_enc.dumps(harry) 265 | >>> new_dec.loads(old_msg) # deserializing an old msg with a new decoder 266 | Person2(first='Harry', last='Potter', address='4 Privet Drive', phone=None, email=None) 267 | 268 | 269 | .. toctree:: 270 | :hidden: 271 | :maxdepth: 2 272 | 273 | faq.rst 274 | benchmarks.rst 275 | api.rst 276 | -------------------------------------------------------------------------------- /docs/source/_static/bench-1.json: -------------------------------------------------------------------------------- 1 | {"target_id": null, "root_id": "1092", "doc": {"roots": {"references": [{"attributes": {"factors": ["pickle tuples", "pyrobuf", "pickle", "orjson", "msgpack", "quickle", "quickle structs"], "range_padding": 0.1}, "id": "1002", "type": "FactorRange"}, {"attributes": {"axis": {"id": "1075"}, "grid_line_color": null, "ticker": null}, "id": "1077", "type": "Grid"}, {"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1025"}, "hover_glyph": null, "muted_glyph": null, "name": "dumps_labels", "nonselection_glyph": {"id": "1026"}, "selection_glyph": null, "view": {"id": "1028"}}, "id": "1027", "type": "GlyphRenderer"}, {"attributes": {"axis_label": "Size (B)", "formatter": {"id": "1102"}, "minor_tick_line_color": null, "ticker": {"id": "1079"}}, "id": "1078", "type": "LinearAxis"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#c9d9d3"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#c9d9d3"}, "top": {"field": "dumps"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1023"}}}, "id": "1026", "type": "VBar"}, {"attributes": {}, "id": "1073", "type": "LinearScale"}, {"attributes": {"axis": {"id": "1078"}, "dimension": 1, "grid_line_color": null, "ticker": null}, "id": "1081", "type": "Grid"}, {"attributes": {}, "id": "1034", "type": "UnionRenderers"}, {"attributes": {}, "id": "1079", "type": "BasicTicker"}, {"attributes": {}, "id": "1035", "type": "Selection"}, {"attributes": {"source": {"id": "1001"}}, "id": "1028", "type": "CDSView"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#1f77b4"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#1f77b4"}, "top": {"field": "size"}, "width": {"value": 0.9}, "x": {"field": "benchmark"}}, "id": "1087", "type": "VBar"}, {"attributes": {"callback": null, "tooltips": [["size", "@size_labels"]]}, "id": "1082", "type": "HoverTool"}, {"attributes": {"active_drag": "auto", "active_inspect": "auto", "active_multi": null, "active_scroll": "auto", "active_tap": "auto", "tools": [{"id": "1082"}]}, "id": "1083", "type": "Toolbar"}, {"attributes": {"label": {"value": "total"}, "renderers": [{"id": "1056"}]}, "id": "1065", "type": "LegendItem"}, {"attributes": {}, "id": "1009", "type": "CategoricalScale"}, {"attributes": {"children": [{"id": "1003"}, {"id": "1066"}, {"id": "1090"}], "sizing_mode": "scale_width"}, "id": "1092", "type": "Column"}, {"attributes": {"fill_color": {"value": "#e84d60"}, "line_color": {"value": "#e84d60"}, "top": {"field": "total"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1052"}}}, "id": "1054", "type": "VBar"}, {"attributes": {"fill_color": {"value": "#c9d9d3"}, "line_color": {"value": "#c9d9d3"}, "top": {"field": "dumps"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1023"}}}, "id": "1025", "type": "VBar"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#e84d60"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#e84d60"}, "top": {"field": "total"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1052"}}}, "id": "1055", "type": "VBar"}, {"attributes": {}, "id": "1071", "type": "CategoricalScale"}, {"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1054"}, "hover_glyph": null, "muted_glyph": null, "name": "total_labels", "nonselection_glyph": {"id": "1055"}, "selection_glyph": null, "view": {"id": "1057"}}, "id": "1056", "type": "GlyphRenderer"}, {"attributes": {}, "id": "1007", "type": "DataRange1d"}, {"attributes": {"source": {"id": "1001"}}, "id": "1057", "type": "CDSView"}, {"attributes": {}, "id": "1100", "type": "CategoricalTickFormatter"}, {"attributes": {"fill_color": {"value": "#1f77b4"}, "line_color": {"value": "#1f77b4"}, "top": {"field": "size"}, "width": {"value": 0.9}, "x": {"field": "benchmark"}}, "id": "1086", "type": "VBar"}, {"attributes": {}, "id": "1102", "type": "BasicTickFormatter"}, {"attributes": {}, "id": "1011", "type": "LinearScale"}, {"attributes": {"label": {"value": "dumps"}, "renderers": [{"id": "1027"}]}, "id": "1037", "type": "LegendItem"}, {"attributes": {"text": "Benchmark - 1 object"}, "id": "1004", "type": "Title"}, {"attributes": {"active": 0, "css_classes": ["centered-radio"], "inline": true, "js_property_callbacks": {"change:active": [{"id": "1091"}]}, "labels": ["total", "dumps", "loads", "size"], "sizing_mode": "scale_width"}, "id": "1090", "type": "RadioGroup"}, {"attributes": {"data": {"benchmark": ["orjson", "msgpack", "pyrobuf", "pickle", "pickle tuples", "quickle", "quickle structs"], "dumps": [0.641565942, 0.43604192399999997, 0.9311699740000003, 0.8349872180000002, 2.271539950000001, 0.36569584199999916, 0.16995488149999982], "dumps_labels": ["0.64 us", "0.44 us", "0.93 us", "0.83 us", "2.27 us", "0.37 us", "0.17 us"], "loads": [1.0379782899999994, 0.8793490019999997, 1.1759347899999995, 0.9196570699999995, 1.5309923349999988, 0.7760646359999992, 0.31514420699999945], "loads_labels": ["1.04 us", "0.88 us", "1.18 us", "0.92 us", "1.53 us", "0.78 us", "0.32 us"], "size": [135.0, 109.0, 71.0, 145.0, 120.0, 130.0, 77.0], "size_labels": ["135 B", "109 B", "71 B", "145 B", "120 B", "130 B", "77 B"], "total": [1.6795442319999994, 1.3153909259999996, 2.107104764, 1.7546442879999997, 3.802532285, 1.1417604779999984, 0.4850990884999992], "total_labels": ["1.68 us", "1.32 us", "2.11 us", "1.75 us", "3.80 us", "1.14 us", "0.49 us"]}, "selected": {"id": "1035"}, "selection_policy": {"id": "1034"}}, "id": "1001", "type": "ColumnDataSource"}, {"attributes": {"axis": {"id": "1013"}, "grid_line_color": null, "ticker": null}, "id": "1015", "type": "Grid"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#718dbf"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#718dbf"}, "top": {"field": "loads"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1038"}}}, "id": "1041", "type": "VBar"}, {"attributes": {"below": [{"id": "1013"}], "center": [{"id": "1015"}, {"id": "1019"}, {"id": "1036"}], "left": [{"id": "1016"}], "plot_height": 250, "plot_width": 660, "renderers": [{"id": "1027"}, {"id": "1042"}, {"id": "1056"}], "sizing_mode": "scale_width", "title": {"id": "1004"}, "toolbar": {"id": "1021"}, "toolbar_location": null, "x_range": {"id": "1002"}, "x_scale": {"id": "1009"}, "y_range": {"id": "1007"}, "y_scale": {"id": "1011"}}, "id": "1003", "subtype": "Figure", "type": "Plot"}, {"attributes": {}, "id": "1014", "type": "CategoricalTicker"}, {"attributes": {"label": {"value": "loads"}, "renderers": [{"id": "1042"}]}, "id": "1051", "type": "LegendItem"}, {"attributes": {"source": {"id": "1001"}}, "id": "1089", "type": "CDSView"}, {"attributes": {"fill_color": {"value": "#718dbf"}, "line_color": {"value": "#718dbf"}, "top": {"field": "loads"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1038"}}}, "id": "1040", "type": "VBar"}, {"attributes": {}, "id": "1031", "type": "CategoricalTickFormatter"}, {"attributes": {"formatter": {"id": "1031"}, "ticker": {"id": "1014"}, "visible": false}, "id": "1013", "type": "CategoricalAxis"}, {"attributes": {"range": {"id": "1002"}, "value": -0.25}, "id": "1023", "type": "Dodge"}, {"attributes": {"axis_label": "Time (us)", "formatter": {"id": "1033"}, "minor_tick_line_color": null, "ticker": {"id": "1017"}}, "id": "1016", "type": "LinearAxis"}, {"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1086"}, "hover_glyph": null, "muted_glyph": null, "nonselection_glyph": {"id": "1087"}, "selection_glyph": null, "view": {"id": "1089"}}, "id": "1088", "type": "GlyphRenderer"}, {"attributes": {"args": {"x_range": {"id": "1002"}}, "code": "\n var lookup = [[\"pickle tuples\", \"pyrobuf\", \"pickle\", \"orjson\", \"msgpack\", \"quickle\", \"quickle structs\"], [\"pickle tuples\", \"pyrobuf\", \"pickle\", \"orjson\", \"msgpack\", \"quickle\", \"quickle structs\"], [\"pickle tuples\", \"pyrobuf\", \"orjson\", \"pickle\", \"msgpack\", \"quickle\", \"quickle structs\"], [\"pickle\", \"orjson\", \"quickle\", \"pickle tuples\", \"msgpack\", \"quickle structs\", \"pyrobuf\"]];\n x_range.factors = lookup[this.active];\n x_range.change.emit();\n "}, "id": "1091", "type": "CustomJS"}, {"attributes": {"axis": {"id": "1016"}, "dimension": 1, "grid_line_color": null, "ticker": null}, "id": "1019", "type": "Grid"}, {"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1040"}, "hover_glyph": null, "muted_glyph": null, "name": "loads_labels", "nonselection_glyph": {"id": "1041"}, "selection_glyph": null, "view": {"id": "1043"}}, "id": "1042", "type": "GlyphRenderer"}, {"attributes": {"below": [{"id": "1075"}], "center": [{"id": "1077"}, {"id": "1081"}], "left": [{"id": "1078"}], "plot_height": 150, "plot_width": 660, "renderers": [{"id": "1088"}], "sizing_mode": "scale_width", "title": null, "toolbar": {"id": "1083"}, "toolbar_location": null, "x_range": {"id": "1002"}, "x_scale": {"id": "1071"}, "y_range": {"id": "1069"}, "y_scale": {"id": "1073"}}, "id": "1066", "subtype": "Figure", "type": "Plot"}, {"attributes": {}, "id": "1017", "type": "BasicTicker"}, {"attributes": {"source": {"id": "1001"}}, "id": "1043", "type": "CDSView"}, {"attributes": {}, "id": "1076", "type": "CategoricalTicker"}, {"attributes": {"callback": null, "tooltips": [["time", "@$name"]]}, "id": "1020", "type": "HoverTool"}, {"attributes": {"range": {"id": "1002"}, "value": 0.25}, "id": "1052", "type": "Dodge"}, {"attributes": {"formatter": {"id": "1100"}, "ticker": {"id": "1076"}}, "id": "1075", "type": "CategoricalAxis"}, {"attributes": {}, "id": "1033", "type": "BasicTickFormatter"}, {"attributes": {"range": {"id": "1002"}}, "id": "1038", "type": "Dodge"}, {"attributes": {"active_drag": "auto", "active_inspect": "auto", "active_multi": null, "active_scroll": "auto", "active_tap": "auto", "tools": [{"id": "1020"}]}, "id": "1021", "type": "Toolbar"}, {"attributes": {"items": [{"id": "1037"}, {"id": "1051"}, {"id": "1065"}], "orientation": "horizontal"}, "id": "1036", "type": "Legend"}, {"attributes": {"start": 0}, "id": "1069", "type": "DataRange1d"}], "root_ids": ["1092"]}, "title": "", "version": "2.2.3"}} -------------------------------------------------------------------------------- /docs/source/_static/bench-10k.json: -------------------------------------------------------------------------------- 1 | {"target_id": null, "root_id": "1092", "doc": {"roots": {"references": [{"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1025"}, "hover_glyph": null, "muted_glyph": null, "name": "dumps_labels", "nonselection_glyph": {"id": "1026"}, "selection_glyph": null, "view": {"id": "1028"}}, "id": "1027", "type": "GlyphRenderer"}, {"attributes": {"start": 0}, "id": "1069", "type": "DataRange1d"}, {"attributes": {"axis": {"id": "1075"}, "grid_line_color": null, "ticker": null}, "id": "1077", "type": "Grid"}, {"attributes": {}, "id": "1033", "type": "BasicTickFormatter"}, {"attributes": {"fill_color": {"value": "#1f77b4"}, "line_color": {"value": "#1f77b4"}, "top": {"field": "size"}, "width": {"value": 0.9}, "x": {"field": "benchmark"}}, "id": "1086", "type": "VBar"}, {"attributes": {}, "id": "1076", "type": "CategoricalTicker"}, {"attributes": {"text": "Benchmark - 10,000 objects"}, "id": "1004", "type": "Title"}, {"attributes": {"axis_label": "Size (MiB)", "formatter": {"id": "1102"}, "minor_tick_line_color": null, "ticker": {"id": "1079"}}, "id": "1078", "type": "LinearAxis"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#c9d9d3"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#c9d9d3"}, "top": {"field": "dumps"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1023"}}}, "id": "1026", "type": "VBar"}, {"attributes": {}, "id": "1073", "type": "LinearScale"}, {"attributes": {}, "id": "1035", "type": "UnionRenderers"}, {"attributes": {"axis": {"id": "1078"}, "dimension": 1, "grid_line_color": null, "ticker": null}, "id": "1081", "type": "Grid"}, {"attributes": {}, "id": "1034", "type": "Selection"}, {"attributes": {}, "id": "1079", "type": "BasicTicker"}, {"attributes": {"factors": ["pickle tuples", "pyrobuf", "orjson", "msgpack", "pickle", "quickle", "quickle structs"], "range_padding": 0.1}, "id": "1002", "type": "FactorRange"}, {"attributes": {"args": {"x_range": {"id": "1002"}}, "code": "\n var lookup = [[\"pickle tuples\", \"pyrobuf\", \"orjson\", \"msgpack\", \"pickle\", \"quickle\", \"quickle structs\"], [\"pickle tuples\", \"pyrobuf\", \"pickle\", \"msgpack\", \"orjson\", \"quickle\", \"quickle structs\"], [\"orjson\", \"msgpack\", \"pyrobuf\", \"pickle tuples\", \"pickle\", \"quickle\", \"quickle structs\"], [\"orjson\", \"msgpack\", \"pickle\", \"quickle\", \"pickle tuples\", \"quickle structs\", \"pyrobuf\"]];\n x_range.factors = lookup[this.active];\n x_range.change.emit();\n "}, "id": "1091", "type": "CustomJS"}, {"attributes": {"data": {"benchmark": ["orjson", "msgpack", "pyrobuf", "pickle", "pickle tuples", "quickle", "quickle structs"], "dumps": [5.89301764, 8.444870419999999, 16.096714150000004, 11.014746549999987, 33.69571799999997, 5.076112960000003, 3.1366877199999976], "dumps_labels": ["5.89 ms", "8.44 ms", "16.10 ms", "11.01 ms", "33.70 ms", "5.08 ms", "3.14 ms"], "loads": [24.646502000000005, 21.8029665, 18.16096775, 14.647304000000005, 15.026853750000013, 12.014572000000001, 8.535809180000022], "loads_labels": ["24.65 ms", "21.80 ms", "18.16 ms", "14.65 ms", "15.03 ms", "12.01 ms", "8.54 ms"], "size": [2.095222, 1.642463, 1.058312, 1.390125, 1.243305, 1.288968, 1.133393], "size_labels": ["2.0 MiB", "1.6 MiB", "1.0 MiB", "1.3 MiB", "1.2 MiB", "1.2 MiB", "1.1 MiB"], "total": [30.539519640000005, 30.24783692, 34.2576819, 25.662050549999993, 48.722571749999986, 17.090684960000004, 11.67249690000002], "total_labels": ["30.54 ms", "30.25 ms", "34.26 ms", "25.66 ms", "48.72 ms", "17.09 ms", "11.67 ms"]}, "selected": {"id": "1034"}, "selection_policy": {"id": "1035"}}, "id": "1001", "type": "ColumnDataSource"}, {"attributes": {"callback": null, "tooltips": [["size", "@size_labels"]]}, "id": "1082", "type": "HoverTool"}, {"attributes": {"source": {"id": "1001"}}, "id": "1057", "type": "CDSView"}, {"attributes": {"active_drag": "auto", "active_inspect": "auto", "active_multi": null, "active_scroll": "auto", "active_tap": "auto", "tools": [{"id": "1082"}]}, "id": "1083", "type": "Toolbar"}, {"attributes": {"label": {"value": "total"}, "renderers": [{"id": "1056"}]}, "id": "1065", "type": "LegendItem"}, {"attributes": {}, "id": "1100", "type": "CategoricalTickFormatter"}, {"attributes": {"fill_color": {"value": "#e84d60"}, "line_color": {"value": "#e84d60"}, "top": {"field": "total"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1052"}}}, "id": "1054", "type": "VBar"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#e84d60"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#e84d60"}, "top": {"field": "total"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1052"}}}, "id": "1055", "type": "VBar"}, {"attributes": {"range": {"id": "1002"}, "value": 0.25}, "id": "1052", "type": "Dodge"}, {"attributes": {}, "id": "1071", "type": "CategoricalScale"}, {"attributes": {}, "id": "1031", "type": "CategoricalTickFormatter"}, {"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1054"}, "hover_glyph": null, "muted_glyph": null, "name": "total_labels", "nonselection_glyph": {"id": "1055"}, "selection_glyph": null, "view": {"id": "1057"}}, "id": "1056", "type": "GlyphRenderer"}, {"attributes": {}, "id": "1011", "type": "LinearScale"}, {"attributes": {"source": {"id": "1001"}}, "id": "1089", "type": "CDSView"}, {"attributes": {}, "id": "1102", "type": "BasicTickFormatter"}, {"attributes": {"active": 0, "css_classes": ["centered-radio"], "inline": true, "js_property_callbacks": {"change:active": [{"id": "1091"}]}, "labels": ["total", "dumps", "loads", "size"], "sizing_mode": "scale_width"}, "id": "1090", "type": "RadioGroup"}, {"attributes": {"below": [{"id": "1013"}], "center": [{"id": "1015"}, {"id": "1019"}, {"id": "1036"}], "left": [{"id": "1016"}], "plot_height": 250, "plot_width": 660, "renderers": [{"id": "1027"}, {"id": "1042"}, {"id": "1056"}], "sizing_mode": "scale_width", "title": {"id": "1004"}, "toolbar": {"id": "1021"}, "toolbar_location": null, "x_range": {"id": "1002"}, "x_scale": {"id": "1009"}, "y_range": {"id": "1007"}, "y_scale": {"id": "1011"}}, "id": "1003", "subtype": "Figure", "type": "Plot"}, {"attributes": {"range": {"id": "1002"}, "value": -0.25}, "id": "1023", "type": "Dodge"}, {"attributes": {}, "id": "1007", "type": "DataRange1d"}, {"attributes": {"source": {"id": "1001"}}, "id": "1043", "type": "CDSView"}, {"attributes": {}, "id": "1009", "type": "CategoricalScale"}, {"attributes": {"children": [{"id": "1003"}, {"id": "1066"}, {"id": "1090"}], "sizing_mode": "scale_width"}, "id": "1092", "type": "Column"}, {"attributes": {"label": {"value": "dumps"}, "renderers": [{"id": "1027"}]}, "id": "1037", "type": "LegendItem"}, {"attributes": {"source": {"id": "1001"}}, "id": "1028", "type": "CDSView"}, {"attributes": {"axis": {"id": "1013"}, "grid_line_color": null, "ticker": null}, "id": "1015", "type": "Grid"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#718dbf"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#718dbf"}, "top": {"field": "loads"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1038"}}}, "id": "1041", "type": "VBar"}, {"attributes": {}, "id": "1014", "type": "CategoricalTicker"}, {"attributes": {"label": {"value": "loads"}, "renderers": [{"id": "1042"}]}, "id": "1051", "type": "LegendItem"}, {"attributes": {"fill_color": {"value": "#718dbf"}, "line_color": {"value": "#718dbf"}, "top": {"field": "loads"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1038"}}}, "id": "1040", "type": "VBar"}, {"attributes": {"formatter": {"id": "1031"}, "ticker": {"id": "1014"}, "visible": false}, "id": "1013", "type": "CategoricalAxis"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#1f77b4"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#1f77b4"}, "top": {"field": "size"}, "width": {"value": 0.9}, "x": {"field": "benchmark"}}, "id": "1087", "type": "VBar"}, {"attributes": {"axis_label": "Time (ms)", "formatter": {"id": "1033"}, "minor_tick_line_color": null, "ticker": {"id": "1017"}}, "id": "1016", "type": "LinearAxis"}, {"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1086"}, "hover_glyph": null, "muted_glyph": null, "nonselection_glyph": {"id": "1087"}, "selection_glyph": null, "view": {"id": "1089"}}, "id": "1088", "type": "GlyphRenderer"}, {"attributes": {"fill_color": {"value": "#c9d9d3"}, "line_color": {"value": "#c9d9d3"}, "top": {"field": "dumps"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1023"}}}, "id": "1025", "type": "VBar"}, {"attributes": {"axis": {"id": "1016"}, "dimension": 1, "grid_line_color": null, "ticker": null}, "id": "1019", "type": "Grid"}, {"attributes": {"below": [{"id": "1075"}], "center": [{"id": "1077"}, {"id": "1081"}], "left": [{"id": "1078"}], "plot_height": 150, "plot_width": 660, "renderers": [{"id": "1088"}], "sizing_mode": "scale_width", "title": null, "toolbar": {"id": "1083"}, "toolbar_location": null, "x_range": {"id": "1002"}, "x_scale": {"id": "1071"}, "y_range": {"id": "1069"}, "y_scale": {"id": "1073"}}, "id": "1066", "subtype": "Figure", "type": "Plot"}, {"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1040"}, "hover_glyph": null, "muted_glyph": null, "name": "loads_labels", "nonselection_glyph": {"id": "1041"}, "selection_glyph": null, "view": {"id": "1043"}}, "id": "1042", "type": "GlyphRenderer"}, {"attributes": {}, "id": "1017", "type": "BasicTicker"}, {"attributes": {"callback": null, "tooltips": [["time", "@$name"]]}, "id": "1020", "type": "HoverTool"}, {"attributes": {"formatter": {"id": "1100"}, "ticker": {"id": "1076"}}, "id": "1075", "type": "CategoricalAxis"}, {"attributes": {"range": {"id": "1002"}}, "id": "1038", "type": "Dodge"}, {"attributes": {"active_drag": "auto", "active_inspect": "auto", "active_multi": null, "active_scroll": "auto", "active_tap": "auto", "tools": [{"id": "1020"}]}, "id": "1021", "type": "Toolbar"}, {"attributes": {"items": [{"id": "1037"}, {"id": "1051"}, {"id": "1065"}], "orientation": "horizontal"}, "id": "1036", "type": "Legend"}], "root_ids": ["1092"]}, "title": "", "version": "2.2.3"}} -------------------------------------------------------------------------------- /docs/source/_static/bench-1k.json: -------------------------------------------------------------------------------- 1 | {"target_id": null, "root_id": "1092", "doc": {"roots": {"references": [{"attributes": {"axis": {"id": "1075"}, "grid_line_color": null, "ticker": null}, "id": "1077", "type": "Grid"}, {"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1025"}, "hover_glyph": null, "muted_glyph": null, "name": "dumps_labels", "nonselection_glyph": {"id": "1026"}, "selection_glyph": null, "view": {"id": "1028"}}, "id": "1027", "type": "GlyphRenderer"}, {"attributes": {"range": {"id": "1002"}}, "id": "1038", "type": "Dodge"}, {"attributes": {"axis_label": "Size (KiB)", "formatter": {"id": "1100"}, "minor_tick_line_color": null, "ticker": {"id": "1079"}}, "id": "1078", "type": "LinearAxis"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#c9d9d3"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#c9d9d3"}, "top": {"field": "dumps"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1023"}}}, "id": "1026", "type": "VBar"}, {"attributes": {}, "id": "1073", "type": "LinearScale"}, {"attributes": {"axis": {"id": "1078"}, "dimension": 1, "grid_line_color": null, "ticker": null}, "id": "1081", "type": "Grid"}, {"attributes": {}, "id": "1079", "type": "BasicTicker"}, {"attributes": {}, "id": "1033", "type": "CategoricalTickFormatter"}, {"attributes": {"factors": ["pickle tuples", "pyrobuf", "msgpack", "orjson", "pickle", "quickle", "quickle structs"], "range_padding": 0.1}, "id": "1002", "type": "FactorRange"}, {"attributes": {}, "id": "1100", "type": "BasicTickFormatter"}, {"attributes": {"callback": null, "tooltips": [["size", "@size_labels"]]}, "id": "1082", "type": "HoverTool"}, {"attributes": {"source": {"id": "1001"}}, "id": "1028", "type": "CDSView"}, {"attributes": {}, "id": "1035", "type": "Selection"}, {"attributes": {"active_drag": "auto", "active_inspect": "auto", "active_multi": null, "active_scroll": "auto", "active_tap": "auto", "tools": [{"id": "1082"}]}, "id": "1083", "type": "Toolbar"}, {"attributes": {"label": {"value": "total"}, "renderers": [{"id": "1056"}]}, "id": "1065", "type": "LegendItem"}, {"attributes": {}, "id": "1034", "type": "UnionRenderers"}, {"attributes": {"fill_color": {"value": "#e84d60"}, "line_color": {"value": "#e84d60"}, "top": {"field": "total"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1052"}}}, "id": "1054", "type": "VBar"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#e84d60"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#e84d60"}, "top": {"field": "total"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1052"}}}, "id": "1055", "type": "VBar"}, {"attributes": {}, "id": "1102", "type": "CategoricalTickFormatter"}, {"attributes": {}, "id": "1071", "type": "CategoricalScale"}, {"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1054"}, "hover_glyph": null, "muted_glyph": null, "name": "total_labels", "nonselection_glyph": {"id": "1055"}, "selection_glyph": null, "view": {"id": "1057"}}, "id": "1056", "type": "GlyphRenderer"}, {"attributes": {"source": {"id": "1001"}}, "id": "1057", "type": "CDSView"}, {"attributes": {"active": 0, "css_classes": ["centered-radio"], "inline": true, "js_property_callbacks": {"change:active": [{"id": "1091"}]}, "labels": ["total", "dumps", "loads", "size"], "sizing_mode": "scale_width"}, "id": "1090", "type": "RadioGroup"}, {"attributes": {}, "id": "1009", "type": "CategoricalScale"}, {"attributes": {"text": "Benchmark - 1000 objects"}, "id": "1004", "type": "Title"}, {"attributes": {"data": {"benchmark": ["orjson", "msgpack", "pyrobuf", "pickle", "pickle tuples", "quickle", "quickle structs"], "dumps": [0.4710531440000001, 0.7835906839999995, 1.4878739149999998, 0.9637191079999994, 2.5734439200000025, 0.46218468400000035, 0.29741671000000025], "dumps_labels": ["471.05 us", "783.59 us", "1.49 ms", "963.72 us", "2.57 ms", "462.18 us", "297.42 us"], "loads": [2.013785874999999, 1.7191356899999999, 1.8370722849999988, 1.0295863999999977, 1.2527082099999998, 0.9193165660000027, 0.7351074959999977], "loads_labels": ["2.01 ms", "1.72 ms", "1.84 ms", "1.03 ms", "1.25 ms", "919.32 us", "735.11 us"], "size": [208.371, 163.343, 105.262, 138.292, 123.642, 128.377, 112.695], "size_labels": ["203.5 KiB", "159.5 KiB", "102.8 KiB", "135.1 KiB", "120.7 KiB", "125.4 KiB", "110.1 KiB"], "total": [2.4848390189999994, 2.5027263739999994, 3.324946199999999, 1.993305507999997, 3.8261521300000023, 1.3815012500000032, 1.032524205999998], "total_labels": ["2.48 ms", "2.50 ms", "3.32 ms", "1.99 ms", "3.83 ms", "1.38 ms", "1.03 ms"]}, "selected": {"id": "1035"}, "selection_policy": {"id": "1034"}}, "id": "1001", "type": "ColumnDataSource"}, {"attributes": {}, "id": "1011", "type": "LinearScale"}, {"attributes": {"fill_color": {"value": "#c9d9d3"}, "line_color": {"value": "#c9d9d3"}, "top": {"field": "dumps"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1023"}}}, "id": "1025", "type": "VBar"}, {"attributes": {}, "id": "1031", "type": "BasicTickFormatter"}, {"attributes": {"range": {"id": "1002"}, "value": -0.25}, "id": "1023", "type": "Dodge"}, {"attributes": {"label": {"value": "dumps"}, "renderers": [{"id": "1027"}]}, "id": "1037", "type": "LegendItem"}, {"attributes": {}, "id": "1007", "type": "DataRange1d"}, {"attributes": {"axis": {"id": "1013"}, "grid_line_color": null, "ticker": null}, "id": "1015", "type": "Grid"}, {"attributes": {"below": [{"id": "1013"}], "center": [{"id": "1015"}, {"id": "1019"}, {"id": "1036"}], "left": [{"id": "1016"}], "plot_height": 250, "plot_width": 660, "renderers": [{"id": "1027"}, {"id": "1042"}, {"id": "1056"}], "sizing_mode": "scale_width", "title": {"id": "1004"}, "toolbar": {"id": "1021"}, "toolbar_location": null, "x_range": {"id": "1002"}, "x_scale": {"id": "1009"}, "y_range": {"id": "1007"}, "y_scale": {"id": "1011"}}, "id": "1003", "subtype": "Figure", "type": "Plot"}, {"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1086"}, "hover_glyph": null, "muted_glyph": null, "nonselection_glyph": {"id": "1087"}, "selection_glyph": null, "view": {"id": "1089"}}, "id": "1088", "type": "GlyphRenderer"}, {"attributes": {"args": {"x_range": {"id": "1002"}}, "code": "\n var lookup = [[\"pickle tuples\", \"pyrobuf\", \"msgpack\", \"orjson\", \"pickle\", \"quickle\", \"quickle structs\"], [\"pickle tuples\", \"pyrobuf\", \"pickle\", \"msgpack\", \"orjson\", \"quickle\", \"quickle structs\"], [\"orjson\", \"pyrobuf\", \"msgpack\", \"pickle tuples\", \"pickle\", \"quickle\", \"quickle structs\"], [\"orjson\", \"msgpack\", \"pickle\", \"quickle\", \"pickle tuples\", \"quickle structs\", \"pyrobuf\"]];\n x_range.factors = lookup[this.active];\n x_range.change.emit();\n "}, "id": "1091", "type": "CustomJS"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#718dbf"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#718dbf"}, "top": {"field": "loads"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1038"}}}, "id": "1041", "type": "VBar"}, {"attributes": {"fill_color": {"value": "#718dbf"}, "line_color": {"value": "#718dbf"}, "top": {"field": "loads"}, "width": {"value": 0.2}, "x": {"field": "benchmark", "transform": {"id": "1038"}}}, "id": "1040", "type": "VBar"}, {"attributes": {"label": {"value": "loads"}, "renderers": [{"id": "1042"}]}, "id": "1051", "type": "LegendItem"}, {"attributes": {}, "id": "1014", "type": "CategoricalTicker"}, {"attributes": {"formatter": {"id": "1033"}, "ticker": {"id": "1014"}, "visible": false}, "id": "1013", "type": "CategoricalAxis"}, {"attributes": {"fill_color": {"value": "#1f77b4"}, "line_color": {"value": "#1f77b4"}, "top": {"field": "size"}, "width": {"value": 0.9}, "x": {"field": "benchmark"}}, "id": "1086", "type": "VBar"}, {"attributes": {"source": {"id": "1001"}}, "id": "1089", "type": "CDSView"}, {"attributes": {"axis_label": "Time (ms)", "formatter": {"id": "1031"}, "minor_tick_line_color": null, "ticker": {"id": "1017"}}, "id": "1016", "type": "LinearAxis"}, {"attributes": {"children": [{"id": "1003"}, {"id": "1066"}, {"id": "1090"}], "sizing_mode": "scale_width"}, "id": "1092", "type": "Column"}, {"attributes": {"data_source": {"id": "1001"}, "glyph": {"id": "1040"}, "hover_glyph": null, "muted_glyph": null, "name": "loads_labels", "nonselection_glyph": {"id": "1041"}, "selection_glyph": null, "view": {"id": "1043"}}, "id": "1042", "type": "GlyphRenderer"}, {"attributes": {"axis": {"id": "1016"}, "dimension": 1, "grid_line_color": null, "ticker": null}, "id": "1019", "type": "Grid"}, {"attributes": {"below": [{"id": "1075"}], "center": [{"id": "1077"}, {"id": "1081"}], "left": [{"id": "1078"}], "plot_height": 150, "plot_width": 660, "renderers": [{"id": "1088"}], "sizing_mode": "scale_width", "title": null, "toolbar": {"id": "1083"}, "toolbar_location": null, "x_range": {"id": "1002"}, "x_scale": {"id": "1071"}, "y_range": {"id": "1069"}, "y_scale": {"id": "1073"}}, "id": "1066", "subtype": "Figure", "type": "Plot"}, {"attributes": {"source": {"id": "1001"}}, "id": "1043", "type": "CDSView"}, {"attributes": {}, "id": "1017", "type": "BasicTicker"}, {"attributes": {"callback": null, "tooltips": [["time", "@$name"]]}, "id": "1020", "type": "HoverTool"}, {"attributes": {"range": {"id": "1002"}, "value": 0.25}, "id": "1052", "type": "Dodge"}, {"attributes": {"formatter": {"id": "1102"}, "ticker": {"id": "1076"}}, "id": "1075", "type": "CategoricalAxis"}, {"attributes": {}, "id": "1076", "type": "CategoricalTicker"}, {"attributes": {"fill_alpha": {"value": 0.1}, "fill_color": {"value": "#1f77b4"}, "line_alpha": {"value": 0.1}, "line_color": {"value": "#1f77b4"}, "top": {"field": "size"}, "width": {"value": 0.9}, "x": {"field": "benchmark"}}, "id": "1087", "type": "VBar"}, {"attributes": {"active_drag": "auto", "active_inspect": "auto", "active_multi": null, "active_scroll": "auto", "active_tap": "auto", "tools": [{"id": "1020"}]}, "id": "1021", "type": "Toolbar"}, {"attributes": {"items": [{"id": "1037"}, {"id": "1051"}, {"id": "1065"}], "orientation": "horizontal"}, "id": "1036", "type": "Legend"}, {"attributes": {"start": 0}, "id": "1069", "type": "DataRange1d"}], "root_ids": ["1092"]}, "title": "", "version": "2.2.3"}} -------------------------------------------------------------------------------- /benchmarks/bench.py: -------------------------------------------------------------------------------- 1 | import random 2 | import string 3 | import timeit 4 | from typing import List, Optional, NamedTuple 5 | 6 | import gc 7 | import msgpack 8 | import orjson 9 | import pickle 10 | import quickle 11 | import proto_bench 12 | 13 | 14 | # Define struct schemas for use with Quickle 15 | class Address(quickle.Struct): 16 | street: str 17 | state: str 18 | zip: int 19 | 20 | 21 | class Person(quickle.Struct): 22 | first: str 23 | last: str 24 | age: int 25 | addresses: Optional[List[Address]] = None 26 | telephone: Optional[str] = None 27 | email: Optional[str] = None 28 | 29 | 30 | # Define named tuple schemas for use with pickle 31 | class AddressTuple(NamedTuple): 32 | street: str 33 | state: str 34 | zip: int 35 | 36 | 37 | class PersonTuple(NamedTuple): 38 | first: str 39 | last: str 40 | age: int 41 | addresses: Optional[List[Address]] = None 42 | telephone: Optional[str] = None 43 | email: Optional[str] = None 44 | 45 | 46 | states = [ 47 | "AL", 48 | "AK", 49 | "AZ", 50 | "AR", 51 | "CA", 52 | "CO", 53 | "CT", 54 | "DE", 55 | "FL", 56 | "GA", 57 | "HI", 58 | "ID", 59 | "IL", 60 | "IN", 61 | "IA", 62 | "KS", 63 | "KY", 64 | "LA", 65 | "ME", 66 | "MD", 67 | "MA", 68 | "MI", 69 | "MN", 70 | "MS", 71 | "MO", 72 | "MT", 73 | "NE", 74 | "NV", 75 | "NH", 76 | "NJ", 77 | "NM", 78 | "NY", 79 | "NC", 80 | "ND", 81 | "OH", 82 | "OK", 83 | "OR", 84 | "PA", 85 | "RI", 86 | "SC", 87 | "SD", 88 | "TN", 89 | "TX", 90 | "UT", 91 | "VT", 92 | "VA", 93 | "WA", 94 | "WV", 95 | "WI", 96 | "WY", 97 | ] 98 | 99 | 100 | def randstr(random, min=None, max=None): 101 | if max is not None: 102 | min = random.randint(min, max) 103 | return "".join(random.choices(string.ascii_letters, k=min)) 104 | 105 | 106 | def make_person(rand=random): 107 | n_addresses = rand.choice([0, 1, 1, 2, 2, 3]) 108 | has_phone = rand.choice([True, True, False]) 109 | has_email = rand.choice([True, True, False]) 110 | 111 | addresses = [ 112 | { 113 | "street": randstr(rand, 10, 40), 114 | "state": rand.choice(states), 115 | "zip": rand.randint(10000, 99999), 116 | } 117 | for _ in range(n_addresses) 118 | ] 119 | 120 | return { 121 | "first": randstr(rand, 3, 15), 122 | "last": randstr(rand, 3, 15), 123 | "age": rand.randint(0, 99), 124 | "addresses": addresses if addresses else None, 125 | "telephone": randstr(rand, 9) if has_phone else None, 126 | "email": randstr(rand, 15, 30) if has_email else None, 127 | } 128 | 129 | 130 | def make_people(n, seed=42): 131 | rand = random.Random(seed) 132 | if n > 1: 133 | return [make_person(rand) for _ in range(n)] 134 | else: 135 | person = make_person(rand) 136 | person["addresses"] = None 137 | return person 138 | 139 | 140 | def bench(dumps, loads, ndata, convert=None, no_gc=False): 141 | setup = "" if no_gc else "gc.enable()" 142 | data = make_people(ndata) 143 | if convert: 144 | data = convert(data) 145 | gc.collect() 146 | timer = timeit.Timer( 147 | "func(data)", setup=setup, globals={"func": dumps, "data": data, "gc": gc} 148 | ) 149 | n, t = timer.autorange() 150 | dumps_time = t / n 151 | 152 | data = dumps(data) 153 | msg_size = len(data) 154 | gc.collect() 155 | timer = timeit.Timer( 156 | "func(data)", setup=setup, globals={"func": loads, "data": data, "gc": gc} 157 | ) 158 | n, t = timer.autorange() 159 | loads_time = t / n 160 | return dumps_time, loads_time, msg_size 161 | 162 | 163 | def bench_msgpack(n, no_gc): 164 | packer = msgpack.Packer() 165 | return bench(packer.pack, msgpack.loads, n, no_gc=no_gc) 166 | 167 | 168 | def bench_orjson(n, no_gc): 169 | return bench(orjson.dumps, orjson.loads, n, no_gc=no_gc) 170 | 171 | 172 | def bench_pyrobuf(n, no_gc): 173 | def convert_one(addresses=None, email=None, telephone=None, **kwargs): 174 | p = proto_bench.Person() 175 | p.ParseFromDict(kwargs) 176 | if addresses: 177 | for a in addresses: 178 | p.addresses.append(proto_bench.Address(**a)) 179 | if telephone: 180 | p.telephone = telephone 181 | if email: 182 | p.email = email 183 | return p 184 | 185 | def convert(data): 186 | if isinstance(data, list): 187 | data = proto_bench.People(people=[convert_one(**d) for d in data]) 188 | else: 189 | data = convert_one(**data) 190 | return data 191 | 192 | if n > 1: 193 | loads = proto_bench.People.FromString 194 | else: 195 | loads = proto_bench.Person.FromString 196 | 197 | def dumps(p): 198 | return p.SerializeToString() 199 | 200 | return bench(dumps, loads, n, convert, no_gc=no_gc) 201 | 202 | 203 | def bench_pickle(n, no_gc): 204 | return bench(pickle.dumps, pickle.loads, n, no_gc=no_gc) 205 | 206 | 207 | def bench_pickle_namedtuple(n, no_gc): 208 | def convert_one(addresses=None, **kwargs): 209 | addrs = [AddressTuple(**a) for a in addresses] if addresses else None 210 | return PersonTuple(addresses=addrs, **kwargs) 211 | 212 | def convert(data): 213 | return ( 214 | [convert_one(**d) for d in data] 215 | if isinstance(data, list) 216 | else convert_one(**data) 217 | ) 218 | 219 | return bench(pickle.dumps, pickle.loads, n, convert, no_gc=no_gc) 220 | 221 | 222 | def bench_quickle(n, no_gc): 223 | enc = quickle.Encoder() 224 | dec = quickle.Decoder() 225 | return bench(enc.dumps, dec.loads, n, no_gc=no_gc) 226 | 227 | 228 | def bench_quickle_structs(n, no_gc): 229 | enc = quickle.Encoder(registry=[Person, Address], memoize=False) 230 | dec = quickle.Decoder(registry=[Person, Address]) 231 | 232 | def convert_one(addresses=None, **kwargs): 233 | addrs = [Address(**a) for a in addresses] if addresses else None 234 | return Person(addresses=addrs, **kwargs) 235 | 236 | def convert(data): 237 | return ( 238 | [convert_one(**d) for d in data] 239 | if isinstance(data, list) 240 | else convert_one(**data) 241 | ) 242 | 243 | return bench(enc.dumps, dec.loads, n, convert, no_gc=no_gc) 244 | 245 | 246 | BENCHMARKS = [ 247 | ("orjson", bench_orjson), 248 | ("msgpack", bench_msgpack), 249 | ("pyrobuf", bench_pyrobuf), 250 | ("pickle", bench_pickle), 251 | ("pickle tuples", bench_pickle_namedtuple), 252 | ("quickle", bench_quickle), 253 | ("quickle structs", bench_quickle_structs), 254 | ] 255 | 256 | 257 | def format_time(n): 258 | if n >= 1: 259 | return "%.2f s" % n 260 | if n >= 1e-3: 261 | return "%.2f ms" % (n * 1e3) 262 | return "%.2f us" % (n * 1e6) 263 | 264 | 265 | def format_bytes(n): 266 | if n >= 2 ** 30: 267 | return "%.1f GiB" % (n / (2 ** 30)) 268 | elif n >= 2 ** 20: 269 | return "%.1f MiB" % (n / (2 ** 20)) 270 | elif n >= 2 ** 10: 271 | return "%.1f KiB" % (n / (2 ** 10)) 272 | return "%s B" % n 273 | 274 | 275 | def preprocess_results(results): 276 | data = dict(zip(["benchmark", "dumps", "loads", "size"], map(list, zip(*results)))) 277 | data["total"] = [d + l for d, l in zip(data["dumps"], data["loads"])] 278 | 279 | max_time = max(data["total"]) 280 | if max_time < 1e-6: 281 | time_unit = "ns" 282 | scale = 1e9 283 | elif max_time < 1e-3: 284 | time_unit = "us" 285 | scale = 1e6 286 | else: 287 | time_unit = "ms" 288 | scale = 1e3 289 | 290 | for k in ["dumps", "loads", "total"]: 291 | data[f"{k}_labels"] = [format_time(t) for t in data[k]] 292 | data[k] = [scale * t for t in data[k]] 293 | 294 | max_size = max(data["size"]) 295 | if max_size < 1e3: 296 | size_unit = "B" 297 | scale = 1 298 | elif max_size < 1e6: 299 | size_unit = "KiB" 300 | scale = 1e3 301 | elif max_size < 1e9: 302 | size_unit = "MiB" 303 | scale = 1e6 304 | 305 | data["size_labels"] = [format_bytes(s) for s in data["size"]] 306 | data["size"] = [s / scale for s in data["size"]] 307 | 308 | return data, time_unit, size_unit 309 | 310 | 311 | def make_plot(results, title): 312 | import json 313 | import bokeh.plotting as bp 314 | from bokeh.transform import dodge 315 | from bokeh.layouts import column 316 | from bokeh.models import CustomJS, RadioGroup, FactorRange 317 | 318 | data, time_unit, size_unit = preprocess_results(results) 319 | 320 | sort_options = ["total", "dumps", "loads", "size"] 321 | sort_orders = [ 322 | list(zip(*sorted(zip(data[order], data["benchmark"]), reverse=True)))[1] 323 | for order in sort_options 324 | ] 325 | 326 | source = bp.ColumnDataSource(data=data) 327 | tooltips = [("time", "@$name")] 328 | 329 | x_range = FactorRange(*sort_orders[0]) 330 | 331 | p = bp.figure( 332 | x_range=x_range, 333 | plot_height=250, 334 | plot_width=660, 335 | title=title, 336 | toolbar_location=None, 337 | tools="", 338 | tooltips=tooltips, 339 | sizing_mode="scale_width", 340 | ) 341 | 342 | p.vbar( 343 | x=dodge("benchmark", -0.25, range=p.x_range), 344 | top="dumps", 345 | width=0.2, 346 | source=source, 347 | color="#c9d9d3", 348 | legend_label="dumps", 349 | name="dumps_labels", 350 | ) 351 | p.vbar( 352 | x=dodge("benchmark", 0.0, range=p.x_range), 353 | top="loads", 354 | width=0.2, 355 | source=source, 356 | color="#718dbf", 357 | legend_label="loads", 358 | name="loads_labels", 359 | ) 360 | p.vbar( 361 | x=dodge("benchmark", 0.25, range=p.x_range), 362 | top="total", 363 | width=0.2, 364 | source=source, 365 | color="#e84d60", 366 | legend_label="total", 367 | name="total_labels", 368 | ) 369 | 370 | p.x_range.range_padding = 0.1 371 | p.xaxis.visible = False 372 | p.xgrid.grid_line_color = None 373 | p.ygrid.grid_line_color = None 374 | p.yaxis.axis_label = f"Time ({time_unit})" 375 | p.yaxis.minor_tick_line_color = None 376 | p.legend.location = "top_right" 377 | p.legend.orientation = "horizontal" 378 | tooltips = [("size", "@size_labels")] 379 | 380 | size_plot = bp.figure( 381 | x_range=x_range, 382 | plot_height=150, 383 | plot_width=660, 384 | title=None, 385 | toolbar_location=None, 386 | tools="hover", 387 | tooltips=tooltips, 388 | sizing_mode="scale_width", 389 | ) 390 | size_plot.vbar(x="benchmark", top="size", width=0.9, source=source) 391 | 392 | size_plot.y_range.start = 0 393 | size_plot.yaxis.axis_label = f"Size ({size_unit})" 394 | size_plot.yaxis.minor_tick_line_color = None 395 | size_plot.x_range.range_padding = 0.1 396 | size_plot.xgrid.grid_line_color = None 397 | size_plot.ygrid.grid_line_color = None 398 | 399 | # Setup widget 400 | select = RadioGroup( 401 | labels=sort_options, active=0, inline=True, css_classes=["centered-radio"] 402 | ) 403 | callback = CustomJS( 404 | args=dict(x_range=x_range), 405 | code=""" 406 | var lookup = {lookup_table}; 407 | x_range.factors = lookup[this.active]; 408 | x_range.change.emit(); 409 | """.format( 410 | lookup_table=json.dumps(sort_orders) 411 | ), 412 | ) 413 | select.js_on_click(callback) 414 | out = column(p, size_plot, select, sizing_mode="scale_width") 415 | return out 416 | 417 | 418 | def run(n, plot_title, plot_name, save_plot=False, save_json=False, no_gc=False): 419 | results = [] 420 | for name, func in BENCHMARKS: 421 | print(f"- {name}...") 422 | dumps_time, loads_time, msg_size = func(n, no_gc) 423 | print(f" dumps: {dumps_time * 1e6:.2f} us") 424 | print(f" loads: {loads_time * 1e6:.2f} us") 425 | print(f" size: {msg_size} bytes") 426 | results.append((name, dumps_time, loads_time, msg_size)) 427 | if save_plot or save_json: 428 | import json 429 | from bokeh.resources import CDN 430 | from bokeh.embed import file_html, json_item 431 | 432 | plot = make_plot(results, plot_title) 433 | if save_plot: 434 | with open(f"{plot_name}.html", "w") as f: 435 | html = file_html(plot, CDN, "Benchmarks") 436 | f.write(html) 437 | if save_json: 438 | with open(f"{plot_name}.json", "w") as f: 439 | data = json.dumps(json_item(plot)) 440 | f.write(data) 441 | 442 | 443 | def run_1(save_plot=False, save_json=False, no_gc=False): 444 | print("Benchmark - 1 object") 445 | run(1, "Benchmark - 1 object", "bench-1", save_plot, save_json, no_gc) 446 | 447 | 448 | def run_1k(save_plot=False, save_json=False, no_gc=False): 449 | print("Benchmark - 1k objects") 450 | run(1000, "Benchmark - 1000 objects", "bench-1k", save_plot, save_json, no_gc) 451 | 452 | 453 | def run_10k(save_plot=False, save_json=False, no_gc=False): 454 | print("Benchmark - 10k objects") 455 | run(10000, "Benchmark - 10,000 objects", "bench-10k", save_plot, save_json, no_gc) 456 | 457 | 458 | def run_all(save_plot=False, save_json=False, no_gc=False): 459 | for runner in [run_1, run_1k, run_10k]: 460 | runner(save_plot, save_json, no_gc) 461 | 462 | 463 | benchmarks = {"all": run_all, "1": run_1, "1k": run_1k, "10k": run_10k} 464 | 465 | 466 | def main(): 467 | import argparse 468 | 469 | parser = argparse.ArgumentParser( 470 | description="Benchmark different python serializers" 471 | ) 472 | parser.add_argument( 473 | "--benchmark", 474 | default="all", 475 | choices=list(benchmarks), 476 | help="which benchmark to run, defaults to 'all'", 477 | ) 478 | parser.add_argument( 479 | "--plot", 480 | action="store_true", 481 | help="whether to plot the results", 482 | ) 483 | parser.add_argument( 484 | "--json", 485 | action="store_true", 486 | help="whether to output json representations of each plot", 487 | ) 488 | parser.add_argument( 489 | "--no-gc", 490 | action="store_true", 491 | help="whether to disable the gc during benchmarking", 492 | ) 493 | args = parser.parse_args() 494 | benchmarks[args.benchmark](args.plot, args.json, args.no_gc) 495 | 496 | 497 | if __name__ == "__main__": 498 | main() 499 | -------------------------------------------------------------------------------- /tests/test_struct.py: -------------------------------------------------------------------------------- 1 | import enum 2 | import copy 3 | import datetime 4 | import gc 5 | import inspect 6 | import pickle 7 | import sys 8 | 9 | import pytest 10 | 11 | import quickle 12 | from quickle import Struct, PickleBuffer 13 | 14 | 15 | class Fruit(enum.IntEnum): 16 | APPLE = 1 17 | BANANA = 2 18 | 19 | 20 | def as_tuple(x): 21 | return tuple(getattr(x, f) for f in x.__struct_fields__) 22 | 23 | 24 | def test_struct_class_attributes(): 25 | assert Struct.__struct_fields__ == () 26 | assert Struct.__struct_defaults__ == () 27 | assert Struct.__slots__ == () 28 | assert Struct.__module__ == "quickle" 29 | 30 | 31 | def test_struct_instance_attributes(): 32 | class Test(Struct): 33 | c: int 34 | b: float 35 | a: str = "hello" 36 | 37 | x = Test(1, 2.0, a="goodbye") 38 | 39 | assert x.__struct_fields__ == ("c", "b", "a") 40 | assert x.__struct_defaults__ == ("hello",) 41 | assert x.__slots__ == ("a", "b", "c") 42 | 43 | assert x.c == 1 44 | assert x.b == 2.0 45 | assert x.a == "goodbye" 46 | 47 | 48 | def test_struct_subclass_forbids_init_new_slots(): 49 | with pytest.raises(TypeError, match="__init__"): 50 | 51 | class Test1(Struct): 52 | a: int 53 | 54 | def __init__(self, a): 55 | pass 56 | 57 | with pytest.raises(TypeError, match="__new__"): 58 | 59 | class Test2(Struct): 60 | a: int 61 | 62 | def __new__(self, a): 63 | pass 64 | 65 | with pytest.raises(TypeError, match="__slots__"): 66 | 67 | class Test3(Struct): 68 | __slots__ = ("a",) 69 | a: int 70 | 71 | 72 | def test_struct_subclass_forbids_non_struct_bases(): 73 | class Mixin(object): 74 | def method(self): 75 | pass 76 | 77 | with pytest.raises(TypeError, match="All base classes must be"): 78 | 79 | class Test(Struct, Mixin): 80 | a: int 81 | 82 | 83 | def test_struct_subclass_forbids_mixed_layouts(): 84 | class A(Struct): 85 | a: int 86 | b: int 87 | 88 | class B(Struct): 89 | c: int 90 | d: int 91 | 92 | # This error is raised by cpython 93 | with pytest.raises(TypeError, match="lay-out conflict"): 94 | 95 | class C(A, B): 96 | pass 97 | 98 | 99 | def test_structmeta_no_args(): 100 | class Test(Struct): 101 | pass 102 | 103 | assert Test.__struct_fields__ == () 104 | assert Test.__struct_defaults__ == () 105 | assert Test.__slots__ == () 106 | 107 | sig = inspect.Signature(parameters=[]) 108 | assert Test.__signature__ == sig 109 | 110 | 111 | def test_structmeta_positional_only(): 112 | class Test(Struct): 113 | y: float 114 | x: int 115 | 116 | assert Test.__struct_fields__ == ("y", "x") 117 | assert Test.__struct_defaults__ == () 118 | assert Test.__slots__ == ("x", "y") 119 | 120 | sig = inspect.Signature( 121 | parameters=[ 122 | inspect.Parameter( 123 | "y", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=float 124 | ), 125 | inspect.Parameter( 126 | "x", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=int 127 | ), 128 | ] 129 | ) 130 | assert Test.__signature__ == sig 131 | 132 | 133 | def test_structmeta_positional_and_keyword(): 134 | class Test(Struct): 135 | c: int 136 | d: int = 1 137 | b: float 138 | a: float = 2.0 139 | 140 | assert Test.__struct_fields__ == ("c", "b", "d", "a") 141 | assert Test.__struct_defaults__ == (1, 2.0) 142 | assert Test.__slots__ == ("a", "b", "c", "d") 143 | 144 | sig = inspect.Signature( 145 | parameters=[ 146 | inspect.Parameter( 147 | "c", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=int 148 | ), 149 | inspect.Parameter( 150 | "b", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=float 151 | ), 152 | inspect.Parameter( 153 | "d", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=int, default=1 154 | ), 155 | inspect.Parameter( 156 | "a", 157 | inspect.Parameter.POSITIONAL_OR_KEYWORD, 158 | annotation=float, 159 | default=2.0, 160 | ), 161 | ] 162 | ) 163 | assert Test.__signature__ == sig 164 | 165 | 166 | def test_structmeta_keyword_only(): 167 | class Test(Struct): 168 | y: int = 1 169 | x: float = 2.0 170 | 171 | assert Test.__struct_fields__ == ("y", "x") 172 | assert Test.__struct_defaults__ == (1, 2.0) 173 | assert Test.__slots__ == ("x", "y") 174 | 175 | sig = inspect.Signature( 176 | parameters=[ 177 | inspect.Parameter( 178 | "y", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=int, default=1 179 | ), 180 | inspect.Parameter( 181 | "x", 182 | inspect.Parameter.POSITIONAL_OR_KEYWORD, 183 | annotation=float, 184 | default=2.0, 185 | ), 186 | ] 187 | ) 188 | assert Test.__signature__ == sig 189 | 190 | 191 | def test_structmeta_subclass_no_change(): 192 | class Test(Struct): 193 | y: float 194 | x: int 195 | 196 | class Test2(Test): 197 | pass 198 | 199 | assert Test2.__struct_fields__ == ("y", "x") 200 | assert Test2.__struct_defaults__ == () 201 | assert Test2.__slots__ == () 202 | 203 | sig = inspect.Signature( 204 | parameters=[ 205 | inspect.Parameter( 206 | "y", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=float 207 | ), 208 | inspect.Parameter( 209 | "x", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=int 210 | ), 211 | ] 212 | ) 213 | assert Test2.__signature__ == sig 214 | 215 | assert as_tuple(Test2(1, 2)) == (1, 2) 216 | assert as_tuple(Test2(y=1, x=2)) == (1, 2) 217 | 218 | 219 | def test_structmeta_subclass_extends(): 220 | class Test(Struct): 221 | c: int 222 | d: int = 1 223 | b: float 224 | a: float = 2.0 225 | 226 | class Test2(Test): 227 | e: str 228 | f: float = 3.0 229 | 230 | assert Test2.__struct_fields__ == ("c", "b", "e", "d", "a", "f") 231 | assert Test2.__struct_defaults__ == (1, 2.0, 3.0) 232 | assert Test2.__slots__ == ("e", "f") 233 | 234 | sig = inspect.Signature( 235 | parameters=[ 236 | inspect.Parameter( 237 | "c", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=int 238 | ), 239 | inspect.Parameter( 240 | "b", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=float 241 | ), 242 | inspect.Parameter( 243 | "e", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=str 244 | ), 245 | inspect.Parameter( 246 | "d", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=int, default=1 247 | ), 248 | inspect.Parameter( 249 | "a", 250 | inspect.Parameter.POSITIONAL_OR_KEYWORD, 251 | annotation=float, 252 | default=2.0, 253 | ), 254 | inspect.Parameter( 255 | "f", 256 | inspect.Parameter.POSITIONAL_OR_KEYWORD, 257 | annotation=float, 258 | default=3.0, 259 | ), 260 | ] 261 | ) 262 | assert Test2.__signature__ == sig 263 | 264 | assert as_tuple(Test2(1, 2, 3, 4, 5, 6)) == (1, 2, 3, 4, 5, 6) 265 | assert as_tuple(Test2(4, 5, 6)) == (4, 5, 6, 1, 2.0, 3.0) 266 | 267 | 268 | def test_structmeta_subclass_overrides(): 269 | class Test(Struct): 270 | c: int 271 | d: int = 1 272 | b: float 273 | a: float = 2.0 274 | 275 | class Test2(Test): 276 | d: int = 2 # change default 277 | c: int = 3 # switch to keyword 278 | a: float # switch to positional 279 | 280 | assert Test2.__struct_fields__ == ("b", "a", "d", "c") 281 | assert Test2.__struct_defaults__ == (2, 3) 282 | assert Test2.__slots__ == () 283 | 284 | sig = inspect.Signature( 285 | parameters=[ 286 | inspect.Parameter( 287 | "b", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=float 288 | ), 289 | inspect.Parameter( 290 | "a", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=float 291 | ), 292 | inspect.Parameter( 293 | "d", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=int, default=2 294 | ), 295 | inspect.Parameter( 296 | "c", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=int, default=3 297 | ), 298 | ] 299 | ) 300 | assert Test2.__signature__ == sig 301 | 302 | assert as_tuple(Test2(1, 2, 3, 4)) == (1, 2, 3, 4) 303 | assert as_tuple(Test2(4, 5)) == (4, 5, 2, 3) 304 | 305 | 306 | def test_structmeta_subclass_mixin_struct_base(): 307 | class A(Struct): 308 | b: int 309 | a: float = 1.0 310 | 311 | class Mixin(Struct): 312 | def as_dict(self): 313 | return {f: getattr(self, f) for f in self.__struct_fields__} 314 | 315 | class B(A, Mixin): 316 | a: float = 2.0 317 | 318 | assert B.__struct_fields__ == ("b", "a") 319 | assert B.__struct_defaults__ == (2.0,) 320 | assert B.__slots__ == () 321 | 322 | sig = inspect.Signature( 323 | parameters=[ 324 | inspect.Parameter( 325 | "b", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=int 326 | ), 327 | inspect.Parameter( 328 | "a", 329 | inspect.Parameter.POSITIONAL_OR_KEYWORD, 330 | annotation=float, 331 | default=2.0, 332 | ), 333 | ] 334 | ) 335 | assert B.__signature__ == sig 336 | 337 | b = B(1) 338 | assert b.as_dict() == {"b": 1, "a": 2.0} 339 | 340 | 341 | def test_struct_init(): 342 | class Test(Struct): 343 | a: int 344 | b: float 345 | c: int = 3 346 | d: float = 4.0 347 | 348 | assert as_tuple(Test(1, 2.0)) == (1, 2.0, 3, 4.0) 349 | assert as_tuple(Test(1, b=2.0)) == (1, 2.0, 3, 4.0) 350 | assert as_tuple(Test(a=1, b=2.0)) == (1, 2.0, 3, 4.0) 351 | assert as_tuple(Test(1, b=2.0, c=5)) == (1, 2.0, 5, 4.0) 352 | assert as_tuple(Test(1, b=2.0, d=5.0)) == (1, 2.0, 3, 5.0) 353 | assert as_tuple(Test(1, 2.0, 5)) == (1, 2.0, 5, 4.0) 354 | assert as_tuple(Test(1, 2.0, 5, 6.0)) == (1, 2.0, 5, 6.0) 355 | 356 | with pytest.raises(TypeError, match="Missing required argument 'a'"): 357 | Test() 358 | 359 | with pytest.raises(TypeError, match="Missing required argument 'b'"): 360 | Test(1) 361 | 362 | with pytest.raises(TypeError, match="Extra positional arguments provided"): 363 | Test(1, 2, 3, 4, 5) 364 | 365 | with pytest.raises(TypeError, match="Argument 'a' given by name and position"): 366 | Test(1, 2, a=3) 367 | 368 | with pytest.raises(TypeError, match="Extra keyword arguments provided"): 369 | Test(1, 2, e=5) 370 | 371 | 372 | def test_struct_repr(): 373 | assert repr(Struct()) == "Struct()" 374 | 375 | class Test(Struct): 376 | pass 377 | 378 | assert repr(Test()) == "Test()" 379 | 380 | class Test(Struct): 381 | a: int 382 | b: str 383 | 384 | assert repr(Test(1, "hello")) == "Test(a=1, b='hello')" 385 | 386 | 387 | def test_struct_repr_errors(): 388 | msg = "Oh no!" 389 | 390 | class Bad: 391 | def __repr__(self): 392 | raise ValueError(msg) 393 | 394 | class Test(Struct): 395 | a: object 396 | b: object 397 | 398 | t = Test(1, Bad()) 399 | 400 | with pytest.raises(ValueError, match=msg): 401 | repr(t) 402 | 403 | 404 | def test_struct_copy(): 405 | x = copy.copy(Struct()) 406 | assert type(x) is Struct 407 | 408 | class Test(Struct): 409 | b: int 410 | a: int 411 | 412 | x = copy.copy(Test(1, 2)) 413 | assert type(x) is Test 414 | assert x.b == 1 415 | assert x.a == 2 416 | 417 | 418 | def test_struct_compare(): 419 | def assert_eq(a, b): 420 | assert a == b 421 | assert not a != b 422 | 423 | def assert_neq(a, b): 424 | assert a != b 425 | assert not a == b 426 | 427 | class Test(Struct): 428 | a: int 429 | b: int 430 | 431 | class Test2(Test): 432 | pass 433 | 434 | x = Struct() 435 | 436 | assert_eq(x, Struct()) 437 | assert_neq(x, None) 438 | 439 | x = Test(1, 2) 440 | assert_eq(x, Test(1, 2)) 441 | assert_neq(x, None) 442 | assert_neq(x, Test(1, 3)) 443 | assert_neq(x, Test(2, 2)) 444 | assert_neq(x, Test2(1, 2)) 445 | 446 | 447 | def test_struct_compare_errors(): 448 | msg = "Oh no!" 449 | 450 | class Bad: 451 | def __eq__(self, other): 452 | raise ValueError(msg) 453 | 454 | __ne__ = __eq__ 455 | 456 | class Test(Struct): 457 | a: object 458 | b: object 459 | 460 | t = Test(1, Bad()) 461 | t2 = Test(1, 2) 462 | 463 | with pytest.raises(ValueError, match=msg): 464 | t == t2 465 | with pytest.raises(ValueError, match=msg): 466 | t != t2 467 | with pytest.raises(ValueError, match=msg): 468 | t2 == t 469 | with pytest.raises(ValueError, match=msg): 470 | t2 != t 471 | 472 | 473 | @pytest.mark.parametrize( 474 | "default", 475 | [ 476 | None, 477 | False, 478 | True, 479 | 1, 480 | 2.0, 481 | 1.5 + 2.32j, 482 | b"test", 483 | "test", 484 | bytearray(b"test"), 485 | PickleBuffer(b"test"), 486 | (), 487 | frozenset(), 488 | Fruit.APPLE, 489 | datetime.time(1), 490 | datetime.date.today(), 491 | datetime.timedelta(seconds=2), 492 | datetime.datetime.now(), 493 | ], 494 | ) 495 | def test_struct_immutable_defaults_use_instance(default): 496 | class Test(Struct): 497 | value: object = default 498 | 499 | t = Test() 500 | assert t.value is default 501 | 502 | 503 | @pytest.mark.parametrize("default", [[], {}, set()]) 504 | def test_struct_empty_mutable_defaults_fast_copy(default): 505 | class Test(Struct): 506 | value: object = default 507 | 508 | t = Test() 509 | assert t.value == default 510 | assert t.value is not default 511 | 512 | 513 | class Point(Struct): 514 | x: int 515 | y: int 516 | 517 | 518 | @pytest.mark.parametrize( 519 | "default", 520 | [ 521 | (Point(1, 2),), 522 | [Point(1, 2)], 523 | {frozenset("a"): None}, 524 | set([frozenset("a")]), 525 | frozenset([frozenset("a")]), 526 | ], 527 | ) 528 | def test_struct_mutable_defaults_deep_copy(default): 529 | class Test(Struct): 530 | value: object = default 531 | 532 | t = Test() 533 | assert t.value == default 534 | assert t.value is not default 535 | for x, y in zip(t.value, default): 536 | assert x == y 537 | assert x is not y 538 | 539 | 540 | def test_struct_reference_counting(): 541 | """Test that struct operations that access fields properly decref""" 542 | 543 | class Test(Struct): 544 | value: list 545 | 546 | data = [1, 2, 3] 547 | 548 | t = Test(data) 549 | assert sys.getrefcount(data) == 3 550 | 551 | repr(t) 552 | assert sys.getrefcount(data) == 3 553 | 554 | t2 = t.__copy__() 555 | assert sys.getrefcount(data) == 4 556 | 557 | assert t == t2 558 | assert sys.getrefcount(data) == 4 559 | 560 | quickle.dumps(t, registry=[Test]) 561 | assert sys.getrefcount(data) == 4 562 | 563 | 564 | def test_struct_gc_not_added_if_not_needed(): 565 | """Structs aren't tracked by GC until/unless they reference a container type""" 566 | 567 | class Test(Struct): 568 | x: object 569 | y: object 570 | 571 | assert not gc.is_tracked(Test(1, 2)) 572 | assert not gc.is_tracked(Test("hello", "world")) 573 | assert gc.is_tracked(Test([1, 2, 3], 1)) 574 | assert gc.is_tracked(Test(1, [1, 2, 3])) 575 | # Tuples are all tracked on creation, but through GC passes eventually 576 | # become untracked if they don't contain tracked types 577 | untracked_tuple = (1, 2, 3) 578 | for i in range(5): 579 | gc.collect() 580 | if not gc.is_tracked(untracked_tuple): 581 | break 582 | else: 583 | assert False, "something has changed with Python's GC, investigate" 584 | assert not gc.is_tracked(Test(1, untracked_tuple)) 585 | tracked_tuple = ([],) 586 | assert gc.is_tracked(Test(1, tracked_tuple)) 587 | 588 | # On mutation, if a tracked objected is stored on a struct, an untracked 589 | # struct will become tracked 590 | t = Test(1, 2) 591 | assert not gc.is_tracked(t) 592 | t.x = 3 593 | assert not gc.is_tracked(t) 594 | t.x = untracked_tuple 595 | assert not gc.is_tracked(t) 596 | t.x = [] 597 | assert gc.is_tracked(t) 598 | 599 | # An error in setattr doesn't change tracked status 600 | t = Test(1, 2) 601 | assert not gc.is_tracked(t) 602 | with pytest.raises(AttributeError): 603 | t.z = [] 604 | assert not gc.is_tracked(t) 605 | 606 | 607 | def test_struct_gc_set_on_unpickle(): 608 | """Unpickling doesn't go through the struct constructor""" 609 | 610 | class Test(quickle.Struct): 611 | x: object 612 | y: object 613 | 614 | ts = [Test(1, 2), Test(3, "hello"), Test([], ()), Test((), ())] 615 | a, b, c, d = quickle.loads(quickle.dumps(ts, registry=[Test]), registry=[Test]) 616 | assert not gc.is_tracked(a) 617 | assert not gc.is_tracked(b) 618 | assert gc.is_tracked(c) 619 | assert not gc.is_tracked(d) 620 | 621 | 622 | def test_struct_gc_set_on_copy(): 623 | """Copying doesn't go through the struct constructor""" 624 | 625 | class Test(quickle.Struct): 626 | x: object 627 | y: object 628 | 629 | assert not gc.is_tracked(copy.copy(Test(1, 2))) 630 | assert not gc.is_tracked(copy.copy(Test(1, ()))) 631 | assert gc.is_tracked(copy.copy(Test(1, []))) 632 | 633 | 634 | class MyStruct(Struct): 635 | x: int 636 | y: int 637 | z: str = "default" 638 | 639 | 640 | def test_structs_are_pickleable(): 641 | """While designed for use with quickle, they should still work with pickle""" 642 | t = MyStruct(1, 2, "hello") 643 | t2 = MyStruct(3, 4) 644 | 645 | assert pickle.loads(pickle.dumps(t)) == t 646 | assert pickle.loads(pickle.dumps(t2)) == t2 647 | 648 | 649 | def test_struct_handles_missing_attributes(): 650 | """If an attribute is unset, raise an AttributeError appropriately""" 651 | t = MyStruct(1, 2) 652 | del t.y 653 | t2 = MyStruct(1, 2) 654 | 655 | match = "Struct field 'y' is unset" 656 | 657 | with pytest.raises(AttributeError, match=match): 658 | repr(t) 659 | 660 | with pytest.raises(AttributeError, match=match): 661 | copy.copy(t) 662 | 663 | with pytest.raises(AttributeError, match=match): 664 | t == t2 665 | 666 | with pytest.raises(AttributeError, match=match): 667 | t2 == t 668 | 669 | with pytest.raises(AttributeError, match=match): 670 | pickle.dumps(t) 671 | 672 | with pytest.raises(AttributeError, match=match): 673 | quickle.dumps(t, registry=[MyStruct]) 674 | -------------------------------------------------------------------------------- /tests/test_quickle.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import enum 3 | import gc 4 | import itertools 5 | import pickle 6 | import pickletools 7 | import string 8 | import sys 9 | import uuid 10 | from distutils.version import StrictVersion 11 | 12 | import pytest 13 | 14 | import quickle 15 | 16 | 17 | BATCHSIZE = 1000 18 | 19 | 20 | def test_picklebuffer_is_shared(): 21 | assert pickle.PickleBuffer is quickle.PickleBuffer 22 | 23 | 24 | def test_module_version(): 25 | StrictVersion(quickle.__version__) 26 | 27 | 28 | def check(obj, sol=None): 29 | if sol is None: 30 | sol = obj 31 | 32 | quick_res = quickle.dumps(obj) 33 | obj2 = quickle.loads(quick_res) 34 | assert obj2 == sol 35 | assert type(obj2) is type(sol) 36 | 37 | obj3 = pickle.loads(quick_res) 38 | assert obj3 == sol 39 | assert type(obj3) is type(sol) 40 | 41 | pickle_res = pickle.dumps(obj, protocol=5) 42 | obj4 = quickle.loads(pickle_res) 43 | assert obj4 == sol 44 | assert type(obj4) is type(sol) 45 | 46 | 47 | def test_pickle_none(): 48 | check(None) 49 | 50 | 51 | @pytest.mark.parametrize("value", [True, False]) 52 | def test_pickle_bool(value): 53 | check(value) 54 | 55 | 56 | @pytest.mark.parametrize("nbytes", [1, 2, 4, 8, 254, 255, 256, 257]) 57 | @pytest.mark.parametrize("negative", [False, True]) 58 | def test_pickle_int(nbytes, negative): 59 | value = 2 ** (nbytes * 8 - 6) 60 | if negative: 61 | value *= -1 62 | 63 | check(value) 64 | 65 | 66 | @pytest.mark.parametrize( 67 | "value", 68 | [ 69 | 0.0, 70 | 4.94e-324, 71 | 1e-310, 72 | 7e-308, 73 | 6.626e-34, 74 | 0.1, 75 | 0.5, 76 | 3.14, 77 | 263.44582062374053, 78 | 6.022e23, 79 | 1e30, 80 | ], 81 | ) 82 | @pytest.mark.parametrize("negative", [False, True]) 83 | def test_pickle_float(value, negative): 84 | if negative: 85 | value *= -1 86 | 87 | check(value) 88 | 89 | 90 | @pytest.mark.parametrize("nbytes", [0, 10, 512]) 91 | def test_pickle_bytes(nbytes): 92 | value = b"y" * nbytes 93 | check(value) 94 | 95 | 96 | @pytest.mark.parametrize("nbytes", [0, 10, 512]) 97 | def test_pickle_bytearray(nbytes): 98 | value = bytearray(b"y" * nbytes) 99 | check(value) 100 | 101 | 102 | @pytest.mark.parametrize("nbytes", [0, 10, 512]) 103 | def test_pickle_unicode(nbytes): 104 | value = "y" * nbytes 105 | check(value) 106 | 107 | 108 | @pytest.mark.parametrize( 109 | "value", 110 | ["<\\u>", "<\\\u1234>", "<\n>", "<\\>", "\U00012345", "<\\\U00012345>", "<\udc80>"], 111 | ) 112 | def test_pickle_unicode_edgecases(value): 113 | check(value) 114 | 115 | 116 | @pytest.mark.parametrize("n", [0, 1, 5, 100, BATCHSIZE + 10]) 117 | def test_pickle_set(n): 118 | check(set(range(n))) 119 | 120 | 121 | @pytest.mark.parametrize("n", [0, 1, 5, 100, BATCHSIZE + 10]) 122 | def test_pickle_frozenset(n): 123 | check(frozenset(range(n))) 124 | 125 | 126 | @pytest.mark.parametrize("n", [0, 1, 2, 3, 100, BATCHSIZE + 10]) 127 | def test_pickle_tuple(n): 128 | check(tuple(range(n))) 129 | 130 | 131 | def test_pickle_recursive_tuple(): 132 | obj = ([None],) 133 | obj[0][0] = obj 134 | 135 | quick_res = quickle.dumps(obj) 136 | for loads in [quickle.loads, pickle.loads]: 137 | obj2 = loads(quick_res) 138 | assert isinstance(obj2, tuple) 139 | assert obj2[0][0] is obj2 140 | # Fix the cycle so `==` works, then test 141 | obj2[0][0] = None 142 | assert obj2 == ([None],) 143 | 144 | 145 | @pytest.mark.parametrize("n", [0, 1, 5, 100, BATCHSIZE + 10]) 146 | def test_pickle_list(n): 147 | check(list(range(n))) 148 | 149 | 150 | def test_pickle_recursive_list(): 151 | # self referential 152 | obj = [] 153 | obj.append(obj) 154 | 155 | quick_res = quickle.dumps(obj) 156 | for loads in [quickle.loads, pickle.loads]: 157 | obj2 = loads(quick_res) 158 | assert isinstance(obj2, list) 159 | assert obj2[0] is obj2 160 | assert len(obj2) == 1 161 | 162 | # one level removed 163 | obj = [[None]] 164 | obj[0][0] = obj 165 | 166 | quick_res = quickle.dumps(obj) 167 | for loads in [quickle.loads, pickle.loads]: 168 | obj2 = loads(quick_res) 169 | assert isinstance(obj2, list) 170 | assert obj2[0][0] is obj2 171 | # Fix the cycle so `==` works, then test 172 | obj2[0][0] = None 173 | assert obj2 == [[None]] 174 | 175 | 176 | @pytest.mark.parametrize("n", [0, 1, 5, 100, BATCHSIZE + 10]) 177 | def test_pickle_dict(n): 178 | value = dict( 179 | zip(itertools.product(string.ascii_letters, string.ascii_letters), range(n)) 180 | ) 181 | check(value) 182 | 183 | 184 | def test_pickle_recursive_dict(): 185 | # self referential 186 | obj = {} 187 | obj[0] = obj 188 | 189 | quick_res = quickle.dumps(obj) 190 | for loads in [quickle.loads, pickle.loads]: 191 | obj2 = loads(quick_res) 192 | assert isinstance(obj2, dict) 193 | assert obj2[0] is obj2 194 | assert len(obj2) == 1 195 | 196 | # one level removed 197 | obj = {0: []} 198 | obj[0].append(obj) 199 | 200 | quick_res = quickle.dumps(obj) 201 | for loads in [quickle.loads, pickle.loads]: 202 | obj2 = loads(quick_res) 203 | assert isinstance(obj2, dict) 204 | assert obj2[0][0] is obj2 205 | # Fix the cycle so `==` works, then test 206 | obj2[0].pop() 207 | assert obj2 == {0: []} 208 | 209 | 210 | def test_pickle_highly_nested_list(): 211 | obj = [] 212 | for _ in range(66): 213 | obj = [obj] 214 | check(obj) 215 | 216 | 217 | def test_pickle_large_memo(): 218 | obj = [[1, 2, 3] for _ in range(2000)] 219 | check(obj) 220 | 221 | 222 | def test_pickle_a_little_bit_of_everything(): 223 | obj = [ 224 | 1, 225 | 1.5, 226 | True, 227 | False, 228 | None, 229 | "hello", 230 | b"hello", 231 | bytearray(b"hello"), 232 | (1, 2, 3), 233 | [1, 2, 3], 234 | {"hello": "world"}, 235 | {1, 2, 3}, 236 | frozenset([1, 2, 3]), 237 | ] 238 | check(obj) 239 | 240 | 241 | def opcode_in_pickle(code, pickle): 242 | for op, _, _ in pickletools.genops(pickle): 243 | if op.code == code.decode("latin-1"): 244 | return True 245 | return False 246 | 247 | 248 | @pytest.mark.parametrize("memoize", [True, False]) 249 | def test_pickle_memoize_class_setting(memoize): 250 | obj = [[1], [2]] 251 | 252 | enc = quickle.Encoder(memoize=memoize) 253 | assert enc.memoize == memoize 254 | 255 | # immutable 256 | with pytest.raises(AttributeError): 257 | enc.memoize = not memoize 258 | assert enc.memoize == memoize 259 | 260 | # default taken from class 261 | res = enc.dumps(obj) 262 | assert opcode_in_pickle(pickle.MEMOIZE, res) == memoize 263 | assert enc.memoize == memoize 264 | 265 | # specify None, no change 266 | res = enc.dumps(obj, memoize=None) 267 | assert opcode_in_pickle(pickle.MEMOIZE, res) == memoize 268 | assert enc.memoize == memoize 269 | 270 | # specify same, no change 271 | res = enc.dumps(obj, memoize=memoize) 272 | assert opcode_in_pickle(pickle.MEMOIZE, res) == memoize 273 | assert enc.memoize == memoize 274 | 275 | # overridden by opposite value 276 | res = enc.dumps(obj, memoize=(not memoize)) 277 | assert opcode_in_pickle(pickle.MEMOIZE, res) != memoize 278 | assert enc.memoize == memoize 279 | 280 | 281 | @pytest.mark.parametrize("memoize", [True, False]) 282 | def test_pickle_memoize_function_settings(memoize): 283 | obj = [[1], [2]] 284 | 285 | res = quickle.dumps(obj, memoize=memoize) 286 | assert opcode_in_pickle(pickle.MEMOIZE, res) == memoize 287 | obj2 = quickle.loads(res) 288 | assert obj == obj2 289 | 290 | obj = [[]] * 2 291 | res = quickle.dumps(obj, memoize=memoize) 292 | assert opcode_in_pickle(pickle.MEMOIZE, res) == memoize 293 | obj2 = quickle.loads(res) 294 | assert obj == obj2 295 | assert (obj2[0] is not obj2[1]) == (not memoize) 296 | 297 | 298 | def test_pickle_memoize_false_recursion_error(): 299 | obj = [] 300 | obj.append(obj) 301 | with pytest.raises(RecursionError): 302 | quickle.dumps(obj, memoize=False) 303 | 304 | 305 | @pytest.mark.parametrize("cls", [bytes, bytearray]) 306 | def test_pickle_picklebuffer_no_callback(cls): 307 | sol = cls(b"hello") 308 | obj = quickle.PickleBuffer(sol) 309 | check(obj, sol) 310 | 311 | 312 | @pytest.mark.parametrize("cls", [bytes, bytearray]) 313 | def test_pickler_collect_buffers_true(cls): 314 | data = cls(b"hello") 315 | pbuf = quickle.PickleBuffer(data) 316 | 317 | enc = quickle.Encoder(collect_buffers=True) 318 | assert enc.collect_buffers 319 | 320 | with pytest.raises(AttributeError): 321 | enc.collect_buffers = False 322 | 323 | # No buffers present returns None 324 | res, buffers = enc.dumps(data) 325 | assert buffers is None 326 | assert quickle.loads(res) == data 327 | 328 | # Buffers are collected and returned 329 | res, buffers = enc.dumps(pbuf) 330 | assert buffers == [pbuf] 331 | assert quickle.loads(res, buffers=buffers) is pbuf 332 | 333 | # Override None uses default 334 | res, buffers = enc.dumps(pbuf, collect_buffers=None) 335 | assert buffers == [pbuf] 336 | assert quickle.loads(res, buffers=buffers) is pbuf 337 | 338 | # Override True is same as default 339 | res, buffers = enc.dumps(pbuf, collect_buffers=True) 340 | assert buffers == [pbuf] 341 | assert quickle.loads(res, buffers=buffers) is pbuf 342 | 343 | # Override False disables buffer collecting 344 | res = enc.dumps(pbuf, collect_buffers=False) 345 | assert quickle.loads(res) == data 346 | 347 | # Override doesn't persist 348 | res, buffers = enc.dumps(pbuf) 349 | assert buffers == [pbuf] 350 | assert quickle.loads(res, buffers=buffers) is pbuf 351 | 352 | 353 | @pytest.mark.parametrize("cls", [bytes, bytearray]) 354 | def test_pickler_collect_buffers_false(cls): 355 | data = cls(b"hello") 356 | pbuf = quickle.PickleBuffer(data) 357 | 358 | enc = quickle.Encoder(collect_buffers=False) 359 | assert not enc.collect_buffers 360 | 361 | with pytest.raises(AttributeError): 362 | enc.collect_buffers = True 363 | 364 | # By default buffers are serialized in-band 365 | res = enc.dumps(pbuf) 366 | assert quickle.loads(res) == data 367 | 368 | # Override None uses default 369 | res = enc.dumps(pbuf, collect_buffers=None) 370 | assert quickle.loads(res) == data 371 | 372 | # Override False is the same as default 373 | res = enc.dumps(pbuf, collect_buffers=False) 374 | assert quickle.loads(res) == data 375 | 376 | # Override True works 377 | res, buffers = enc.dumps(pbuf, collect_buffers=True) 378 | assert buffers == [pbuf] 379 | assert quickle.loads(res, buffers=buffers) is pbuf 380 | 381 | # If no buffers present, output is None 382 | res, buffers = enc.dumps(data, collect_buffers=True) 383 | assert buffers is None 384 | assert quickle.loads(res, buffers=buffers) == data 385 | 386 | # Override doesn't persist 387 | res = enc.dumps(pbuf) 388 | assert quickle.loads(res) == data 389 | 390 | 391 | @pytest.mark.parametrize("cls", [bytes, bytearray]) 392 | def test_quickle_pickle_collect_buffers_true_compatibility(cls): 393 | data = cls(b"hello") 394 | pbuf = quickle.PickleBuffer(data) 395 | 396 | # quickle -> pickle 397 | quick_res, quick_buffers = quickle.dumps(pbuf, collect_buffers=True) 398 | obj = pickle.loads(quick_res, buffers=quick_buffers) 399 | assert obj is pbuf 400 | 401 | # pickle -> quickle 402 | pickle_buffers = [] 403 | pickle_res = pickle.dumps(pbuf, buffer_callback=pickle_buffers.append, protocol=5) 404 | obj = quickle.loads(pickle_res, buffers=pickle_buffers) 405 | assert obj is pbuf 406 | 407 | 408 | @pytest.mark.parametrize("cls", [bytes, bytearray]) 409 | def test_quickle_pickle_collect_buffers_false_compatibility(cls): 410 | data = cls(b"hello") 411 | pbuf = quickle.PickleBuffer(data) 412 | 413 | # quickle -> pickle 414 | quick_res = quickle.dumps(pbuf) 415 | obj = pickle.loads(quick_res) 416 | assert obj == data 417 | 418 | # pickle -> quickle 419 | pickle_res = pickle.dumps(pbuf, protocol=5) 420 | obj = quickle.loads(pickle_res) 421 | assert obj == data 422 | 423 | 424 | def test_loads_buffers_errors(): 425 | obj = quickle.PickleBuffer(b"hello") 426 | res, _ = quickle.dumps(obj, collect_buffers=True) 427 | 428 | with pytest.raises(TypeError): 429 | quickle.loads(res, buffers=object()) 430 | 431 | with pytest.raises(quickle.DecodingError): 432 | quickle.loads(res, buffers=[]) 433 | 434 | 435 | @pytest.mark.parametrize("value", [object(), object, sum, itertools.count]) 436 | def test_dumps_and_loads_unpickleable_types(value): 437 | with pytest.raises(TypeError): 438 | quickle.dumps(value) 439 | 440 | o = pickle.dumps(value, protocol=5) 441 | 442 | with pytest.raises(quickle.DecodingError): 443 | quickle.loads(o) 444 | 445 | 446 | def test_loads_truncated_input(): 447 | data = quickle.dumps([1, 2, 3]) 448 | with pytest.raises(quickle.DecodingError): 449 | quickle.loads(data[:-2]) 450 | 451 | 452 | def test_loads_bad_pickle(): 453 | with pytest.raises(quickle.DecodingError): 454 | quickle.loads(b"this isn't valid at all") 455 | 456 | 457 | def test_getsizeof(): 458 | a = sys.getsizeof(quickle.Encoder(write_buffer_size=64)) 459 | b = sys.getsizeof(quickle.Encoder(write_buffer_size=128)) 460 | assert b > a 461 | # Smoketest 462 | sys.getsizeof(quickle.Decoder()) 463 | 464 | 465 | @pytest.mark.parametrize( 466 | "enc", 467 | [ 468 | # bad stacks 469 | b".", # STOP 470 | b"0", # POP 471 | b"1", # POP_MARK 472 | b"a", # APPEND 473 | b"Na", 474 | b"e", # APPENDS 475 | b"(e", 476 | b"s", # SETITEM 477 | b"Ns", 478 | b"NNs", 479 | b"t", # TUPLE 480 | b"u", # SETITEMS 481 | b"(u", 482 | b"}(Nu", 483 | b"\x85", # TUPLE1 484 | b"\x86", # TUPLE2 485 | b"N\x86", 486 | b"\x87", # TUPLE3 487 | b"N\x87", 488 | b"NN\x87", 489 | b"\x90", # ADDITEMS 490 | b"(\x90", 491 | b"\x91", # FROZENSET 492 | b"\x94", # MEMOIZE 493 | # bad marks 494 | b"N(.", # STOP 495 | b"]N(a", # APPEND 496 | b"}NN(s", # SETITEM 497 | b"}N(Ns", 498 | b"}(NNs", 499 | b"}((u", # SETITEMS 500 | b"N(\x85", # TUPLE1 501 | b"NN(\x86", # TUPLE2 502 | b"N(N\x86", 503 | b"NNN(\x87", # TUPLE3 504 | b"NN(N\x87", 505 | b"N(NN\x87", 506 | b"]((\x90", # ADDITEMS 507 | b"N(\x94", # MEMOIZE 508 | ], 509 | ) 510 | def test_bad_stack_or_mark(enc): 511 | with pytest.raises(quickle.DecodingError): 512 | quickle.loads(enc) 513 | 514 | 515 | @pytest.mark.parametrize( 516 | "enc", 517 | [ 518 | b"B", # BINBYTES 519 | b"B\x03\x00\x00", 520 | b"B\x03\x00\x00\x00", 521 | b"B\x03\x00\x00\x00ab", 522 | b"C", # SHORT_BINBYTES 523 | b"C\x03", 524 | b"C\x03ab", 525 | b"G", # BINFLOAT 526 | b"G\x00\x00\x00\x00\x00\x00\x00", 527 | b"J", # BININT 528 | b"J\x00\x00\x00", 529 | b"K", # BININT1 530 | b"M", # BININT2 531 | b"M\x00", 532 | b"T", # BINSTRING 533 | b"T\x03\x00\x00", 534 | b"T\x03\x00\x00\x00", 535 | b"T\x03\x00\x00\x00ab", 536 | b"U", # SHORT_BINSTRING 537 | b"U\x03", 538 | b"U\x03ab", 539 | b"X", # BINUNICODE 540 | b"X\x03\x00\x00", 541 | b"X\x03\x00\x00\x00", 542 | b"X\x03\x00\x00\x00ab", 543 | b"Nh", # BINGET 544 | b"Nj", # LONG_BINGET 545 | b"Nj\x00\x00\x00", 546 | b"Nr\x00\x00\x00", 547 | b"\x80", # PROTO 548 | b"\x8a", # LONG1 549 | b"\x8b", # LONG4 550 | b"\x8b\x00\x00\x00", 551 | b"\x8c", # SHORT_BINUNICODE 552 | b"\x8c\x03", 553 | b"\x8c\x03ab", 554 | b"\x8d", # BINUNICODE8 555 | b"\x8d\x03\x00\x00\x00\x00\x00\x00", 556 | b"\x8d\x03\x00\x00\x00\x00\x00\x00\x00", 557 | b"\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab", 558 | b"\x8e", # BINBYTES8 559 | b"\x8e\x03\x00\x00\x00\x00\x00\x00", 560 | b"\x8e\x03\x00\x00\x00\x00\x00\x00\x00", 561 | b"\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab", 562 | b"\x96", # BYTEARRAY8 563 | b"\x96\x03\x00\x00\x00\x00\x00\x00", 564 | b"\x96\x03\x00\x00\x00\x00\x00\x00\x00", 565 | b"\x96\x03\x00\x00\x00\x00\x00\x00\x00ab", 566 | b"\x95", # FRAME 567 | b"\x95\x02\x00\x00\x00\x00\x00\x00", 568 | b"\x95\x02\x00\x00\x00\x00\x00\x00\x00", 569 | b"\x95\x02\x00\x00\x00\x00\x00\x00\x00N", 570 | ], 571 | ) 572 | def test_truncated_data(enc): 573 | with pytest.raises(quickle.DecodingError): 574 | quickle.loads(enc) 575 | 576 | 577 | class MyStruct(quickle.Struct): 578 | x: object 579 | y: object 580 | 581 | 582 | class MyStruct2(quickle.Struct): 583 | x: object 584 | y: object = 1 585 | z: object = [] 586 | z2: object = 3 587 | 588 | 589 | class MyStruct3(quickle.Struct): 590 | x: object 591 | y: object 592 | z: object 593 | 594 | 595 | def test_pickler_unpickler_registry_kwarg_errors(): 596 | with pytest.raises(TypeError, match="registry must be a list or a dict"): 597 | quickle.Encoder(registry="bad") 598 | 599 | with pytest.raises(TypeError, match="an integer is required"): 600 | quickle.Encoder(registry={MyStruct: 1.0}) 601 | 602 | with pytest.raises(ValueError, match="registry values must be between"): 603 | quickle.Encoder(registry={MyStruct: -1}) 604 | 605 | with pytest.raises(TypeError, match="registry must be a list or a dict"): 606 | quickle.Decoder(registry="bad") 607 | 608 | 609 | @pytest.mark.parametrize("registry_type", ["list", "dict"]) 610 | @pytest.mark.parametrize("use_functions", [True, False]) 611 | def test_pickle_struct(registry_type, use_functions): 612 | if registry_type == "list": 613 | p_registry = u_registry = [MyStruct] 614 | else: 615 | p_registry = {MyStruct: 0} 616 | u_registry = {0: MyStruct} 617 | 618 | x = MyStruct(1, 2) 619 | 620 | if use_functions: 621 | s = quickle.dumps(x, registry=p_registry) 622 | x2 = quickle.loads(s, registry=u_registry) 623 | else: 624 | enc = quickle.Encoder(registry=p_registry) 625 | dec = quickle.Decoder(registry=u_registry) 626 | s = enc.dumps(x) 627 | x2 = dec.loads(s) 628 | 629 | assert x == x2 630 | 631 | 632 | @pytest.mark.parametrize("code", [0, 2 ** 8 - 1, 2 ** 16 - 1, 2 ** 31 - 1]) 633 | def test_pickle_struct_codes(code): 634 | x = MyStruct(1, 2) 635 | 636 | p_registry = {MyStruct: code} 637 | u_registry = {code: MyStruct} 638 | 639 | s = quickle.dumps(x, registry=p_registry) 640 | x2 = quickle.loads(s, registry=u_registry) 641 | 642 | assert x2 == x 643 | 644 | 645 | def test_pickle_struct_code_out_of_range(): 646 | x = MyStruct(1, 2) 647 | with pytest.raises(Exception) as exc: 648 | quickle.dumps(x, registry={MyStruct: 2 ** 32}) 649 | if isinstance(exc.value, ValueError): 650 | assert "registry values must be between" in str(exc.value) 651 | else: 652 | assert isinstance(exc.value, OverflowError) 653 | 654 | 655 | def test_pickle_struct_recursive(): 656 | x = MyStruct(1, None) 657 | x.y = x 658 | s = quickle.dumps(x, registry=[MyStruct]) 659 | x2 = quickle.loads(s, registry=[MyStruct]) 660 | assert x2.x == 1 661 | assert x2.y is x2 662 | assert type(x) is MyStruct 663 | 664 | 665 | @pytest.mark.parametrize("registry", ["missing", None, [], {}, {1: MyStruct}]) 666 | def test_pickle_errors_struct_missing_from_registry(registry): 667 | x = MyStruct(1, 2) 668 | s = quickle.dumps(x, registry=[MyStruct]) 669 | kwargs = {} if registry == "missing" else {"registry": registry} 670 | with pytest.raises(ValueError, match="Typecode"): 671 | quickle.loads(s, **kwargs) 672 | 673 | 674 | @pytest.mark.parametrize( 675 | "registry", ["missing", None, [], [MyStruct2], {}, {MyStruct2: 0}] 676 | ) 677 | def test_unpickle_errors_struct_typecode_missing_from_registry(registry): 678 | kwargs = {} if registry == "missing" else {"registry": registry} 679 | x = MyStruct(1, 2) 680 | with pytest.raises(TypeError, match="Type MyStruct isn't in type registry"): 681 | quickle.dumps(x, **kwargs) 682 | 683 | 684 | def test_unpickle_errors_obj_in_registry_is_not_struct_type(): 685 | class Foo(object): 686 | pass 687 | 688 | x = MyStruct(1, 2) 689 | s = quickle.dumps(x, registry=[MyStruct]) 690 | with pytest.raises(TypeError, match="Value for typecode"): 691 | quickle.loads(s, registry=[Foo]) 692 | 693 | 694 | def test_unpickle_errors_buildstruct_on_non_struct_object(): 695 | s = b"\x80\x05K\x00\x94(K\x01K\x02\xb0." 696 | with pytest.raises(quickle.DecodingError, match="BUILDSTRUCT"): 697 | quickle.loads(s, registry=[MyStruct]) 698 | 699 | 700 | def test_struct_registry_mismatch_fewer_args_no_defaults_errors(): 701 | x = MyStruct(1, 2) 702 | s = quickle.dumps(x, registry=[MyStruct]) 703 | with pytest.raises(TypeError, match="Missing required argument 'z'"): 704 | quickle.loads(s, registry=[MyStruct3]) 705 | 706 | 707 | def test_struct_registry_mismatch_fewer_args_default_parameters_respected(): 708 | """Unpickling a struct with a newer version that has additional default 709 | parameters at the end works (the defaults are used).""" 710 | x = MyStruct(1, 2) 711 | s = quickle.dumps(x, registry=[MyStruct]) 712 | x2 = quickle.loads(s, registry=[MyStruct2]) 713 | assert isinstance(x2, MyStruct2) 714 | assert x2.x == x.x 715 | assert x2.y == x.y 716 | assert x2.z == [] 717 | assert x2.z2 == 3 718 | 719 | 720 | def test_struct_registry_mismatch_extra_args_are_ignored(): 721 | """Unpickling a struct with an older version that has fewer parameters 722 | works (the extra args are ignored).""" 723 | x = MyStruct2(1, 2) 724 | s = quickle.dumps(x, registry=[MyStruct2]) 725 | x2 = quickle.loads(s, registry=[MyStruct]) 726 | assert x2.x == 1 727 | assert x2.y == 2 728 | 729 | 730 | class Fruit(enum.IntEnum): 731 | APPLE = 1 732 | BANANA = 2 733 | ORANGE = 3 734 | 735 | 736 | class PyObjects(enum.Enum): 737 | LIST = [] 738 | STRING = "" 739 | OBJECT = object() 740 | 741 | 742 | @pytest.mark.parametrize("x", list(Fruit)) 743 | def test_pickle_intenum(x): 744 | s = quickle.dumps(x, registry=[Fruit]) 745 | x2 = quickle.loads(s, registry=[Fruit]) 746 | assert x2 == x 747 | 748 | 749 | @pytest.mark.parametrize("x", list(PyObjects)) 750 | def test_pickle_enum(x): 751 | s = quickle.dumps(x, registry=[PyObjects]) 752 | assert x.name.encode() in s 753 | x2 = quickle.loads(s, registry=[PyObjects]) 754 | assert x2 == x 755 | 756 | 757 | @pytest.mark.parametrize("code", [0, 2 ** 8 - 1, 2 ** 16 - 1, 2 ** 31 - 1]) 758 | def test_pickle_enum_codes(code): 759 | p_registry = {Fruit: code} 760 | u_registry = {code: Fruit} 761 | 762 | s = quickle.dumps(Fruit.APPLE, registry=p_registry) 763 | x2 = quickle.loads(s, registry=u_registry) 764 | 765 | assert x2 == Fruit.APPLE 766 | 767 | 768 | def test_pickle_enum_code_out_of_range(): 769 | class Fruit(enum.IntEnum): 770 | APPLE = 1 771 | 772 | with pytest.raises(Exception) as exc: 773 | quickle.dumps(Fruit.APPLE, registry={Fruit: 2 ** 32}) 774 | if isinstance(exc.value, ValueError): 775 | assert "registry values must be between" in str(exc.value) 776 | else: 777 | assert isinstance(exc.value, OverflowError) 778 | 779 | 780 | @pytest.mark.parametrize("registry", [None, [], {1: PyObjects}]) 781 | def test_pickle_errors_enum_missing_from_registry(registry): 782 | s = quickle.dumps(Fruit.APPLE, registry=[Fruit]) 783 | with pytest.raises(ValueError, match="Typecode"): 784 | quickle.loads(s, registry=registry) 785 | 786 | 787 | @pytest.mark.parametrize("registry", [None, [PyObjects], {PyObjects: 1}]) 788 | def test_unpickle_errors_enum_typecode_missing_from_registry(registry): 789 | with pytest.raises(TypeError, match="Type Fruit isn't in type registry"): 790 | quickle.dumps(Fruit.APPLE, registry=registry) 791 | 792 | 793 | def test_unpickle_errors_obj_in_registry_is_not_enum_type(): 794 | s = quickle.dumps(Fruit.APPLE, registry=[Fruit]) 795 | with pytest.raises(TypeError, match="Value for typecode"): 796 | quickle.loads(s, registry=[MyStruct]) 797 | 798 | 799 | def test_unpickle_errors_intenum_missing_value(): 800 | class Fruit2(enum.IntEnum): 801 | APPLE = 1 802 | 803 | s = quickle.dumps(Fruit.ORANGE, registry=[Fruit]) 804 | with pytest.raises(ValueError, match="Fruit2"): 805 | quickle.loads(s, registry=[Fruit2]) 806 | 807 | 808 | def test_unpickle_errors_enum_missing_attribute(): 809 | class PyObjects2(enum.Enum): 810 | LIST = [] 811 | 812 | s = quickle.dumps(PyObjects.OBJECT, registry=[PyObjects]) 813 | with pytest.raises(AttributeError, match="OBJECT"): 814 | quickle.loads(s, registry=[PyObjects2]) 815 | 816 | 817 | @pytest.mark.parametrize("x", [0j, 1j, 1 + 0j, 1 + 1j, 1e-9 - 2.5e9j]) 818 | def test_pickle_complex(x): 819 | s = quickle.dumps(x) 820 | x2 = quickle.loads(s) 821 | assert x == x2 822 | 823 | 824 | TIMEDELTA_MAX_DAYS = 999999999 825 | 826 | 827 | @pytest.mark.parametrize( 828 | "x", 829 | [ 830 | datetime.timedelta(), 831 | datetime.timedelta(days=TIMEDELTA_MAX_DAYS), 832 | datetime.timedelta(days=-TIMEDELTA_MAX_DAYS), 833 | datetime.timedelta(days=1234, seconds=56, microseconds=78), 834 | datetime.timedelta(seconds=24 * 3600 - 1), 835 | datetime.timedelta(microseconds=1000000 - 1), 836 | ], 837 | ) 838 | def test_timedelta(x): 839 | s = quickle.dumps(x) 840 | x2 = quickle.loads(s) 841 | assert x == x2 842 | 843 | 844 | @pytest.mark.parametrize("positive", [True, False]) 845 | def test_loads_timedelta_out_of_range(positive): 846 | s = quickle.dumps(datetime.timedelta(days=1234)) 847 | days = TIMEDELTA_MAX_DAYS + 1 848 | if not positive: 849 | days = -days 850 | key = (1234).to_bytes(4, "little", signed=True) 851 | bad = s.replace(key, days.to_bytes(4, "little", signed=True)) 852 | with pytest.raises(OverflowError): 853 | quickle.loads(bad) 854 | 855 | 856 | @pytest.mark.parametrize("x", [datetime.date(2020, 1, 1), datetime.date(9999, 12, 31)]) 857 | def test_date(x): 858 | s = quickle.dumps(x) 859 | x2 = quickle.loads(s) 860 | assert x == x2 861 | 862 | 863 | def test_loads_date_out_of_range(): 864 | s = quickle.dumps(datetime.date(9999, 12, 31)) 865 | bad = s.replace((9999).to_bytes(2, "little"), (10000).to_bytes(2, "little")) 866 | with pytest.raises(ValueError): 867 | quickle.loads(bad) 868 | 869 | 870 | @pytest.mark.parametrize( 871 | "x", 872 | [ 873 | datetime.time(hour=5, minute=30, second=25), 874 | datetime.time(hour=23, minute=59, second=59, microsecond=999999, fold=0), 875 | datetime.time(hour=23, minute=59, second=59, microsecond=999999, fold=1), 876 | datetime.time(hour=5, tzinfo=datetime.timezone.utc), 877 | datetime.time(hour=5, tzinfo=datetime.timezone(datetime.timedelta(0, 1, 2))), 878 | ], 879 | ) 880 | def test_time(x): 881 | s = quickle.dumps(x) 882 | x2 = quickle.loads(s) 883 | assert x == x2 884 | 885 | 886 | @pytest.mark.parametrize( 887 | "x", 888 | [ 889 | datetime.datetime.now(), 890 | datetime.datetime( 891 | year=9999, 892 | month=12, 893 | day=31, 894 | hour=23, 895 | minute=59, 896 | second=59, 897 | microsecond=999999, 898 | fold=0, 899 | ), 900 | datetime.datetime( 901 | year=9999, 902 | month=12, 903 | day=31, 904 | hour=23, 905 | minute=59, 906 | second=59, 907 | microsecond=999999, 908 | fold=1, 909 | ), 910 | datetime.datetime.now(datetime.timezone.utc), 911 | datetime.datetime.now(datetime.timezone(datetime.timedelta(0, 1, 2))), 912 | ], 913 | ) 914 | def test_datetime(x): 915 | s = quickle.dumps(x) 916 | x2 = quickle.loads(s) 917 | assert x == x2 918 | 919 | 920 | def test_timezone_utc(): 921 | s = quickle.dumps(datetime.timezone.utc) 922 | x = quickle.loads(s) 923 | assert x == datetime.timezone.utc 924 | 925 | 926 | @pytest.mark.parametrize( 927 | "offset", 928 | [ 929 | datetime.timedelta(hours=23, minutes=59, seconds=59, microseconds=999999), 930 | datetime.timedelta(hours=1, minutes=2, seconds=3, microseconds=4), 931 | datetime.timedelta(microseconds=1), 932 | datetime.timedelta(microseconds=-1), 933 | datetime.timedelta(hours=-1, minutes=-2, seconds=-3, microseconds=-4), 934 | datetime.timedelta(hours=-23, minutes=-59, seconds=-59, microseconds=-999999), 935 | ], 936 | ) 937 | def test_timezone(offset): 938 | x = datetime.timezone(offset) 939 | s = quickle.dumps(x) 940 | x2 = quickle.loads(s) 941 | assert x == x2 942 | 943 | 944 | @pytest.fixture 945 | def zoneinfo_parts(): 946 | zoneinfo = pytest.importorskip("zoneinfo") 947 | a, b = sorted(zoneinfo.available_timezones())[:2] 948 | za = zoneinfo.ZoneInfo(a) 949 | zb = zoneinfo.ZoneInfo(b) 950 | objs = [ 951 | za, 952 | zb, 953 | datetime.datetime.now(za), 954 | datetime.datetime.now(zb), 955 | datetime.time(hour=4, tzinfo=za), 956 | datetime.time(hour=5, tzinfo=zb), 957 | ] 958 | return objs 959 | 960 | 961 | @pytest.mark.parametrize("ind", range(6)) 962 | def test_zoneinfo(zoneinfo_parts, ind): 963 | x = zoneinfo_parts[ind] 964 | s = quickle.dumps(x) 965 | x2 = quickle.loads(s) 966 | assert x == x2 967 | 968 | 969 | def test_zoneinfo_not_found(): 970 | try: 971 | import zoneinfo # noqa 972 | 973 | pytest.skip("zoneinfo successfully imported") 974 | except ImportError: 975 | pass 976 | 977 | with pytest.raises(quickle.DecodingError, match="zoneinfo"): 978 | quickle.loads(b"\x8c\x0fAmerica/Chicago\xc0.") 979 | 980 | 981 | def test_objects_with_only_one_refcount_arent_memoized(): 982 | class Test(quickle.Struct): 983 | x: list 984 | y: str 985 | 986 | def rstr(): 987 | return str(uuid.uuid4().hex) 988 | 989 | data = [ 990 | (rstr(),), 991 | (rstr(), rstr(), rstr(), rstr(), rstr()), 992 | ([[[rstr()]]],), 993 | [rstr()], 994 | {rstr()}, 995 | frozenset([rstr()]), 996 | {rstr(): rstr()}, 997 | rstr(), 998 | rstr().encode(), 999 | bytearray(rstr().encode()), 1000 | Test([rstr()], rstr()), 1001 | ] 1002 | 1003 | s = quickle.dumps(data, registry=[Test]) 1004 | # only initial arg is memoized, since its refcnt is 2 1005 | assert s.count(pickle.MEMOIZE) == 1 1006 | 1007 | # Grab a reference to a tuple containing only non-container types 1008 | a = data[1] 1009 | s = quickle.dumps(data, registry=[Test]) 1010 | # 2 memoize codes, 1 for data and 1 for the tuple 1011 | assert s.count(pickle.MEMOIZE) == 2 1012 | del a 1013 | 1014 | # Grab a reference to a tuple containing container types 1015 | a = data[2] 1016 | s = quickle.dumps(data, registry=[Test]) 1017 | # 5 memoize codes, 1 for data and 1 for the tuple, 1 for each list 1018 | assert s.count(pickle.MEMOIZE) == 5 1019 | del a 1020 | 1021 | 1022 | @pytest.mark.parametrize("use_decoder", [True, False]) 1023 | def test_loads_errors_wrong_type(use_decoder): 1024 | """Previously this would segfault on GC collect if using `quickle.loads` on 1025 | the wrong type""" 1026 | if use_decoder: 1027 | dec = quickle.Decoder() 1028 | else: 1029 | dec = quickle 1030 | 1031 | for i in range(3): 1032 | with pytest.raises(TypeError): 1033 | dec.loads(1) 1034 | gc.collect() 1035 | --------------------------------------------------------------------------------