├── .github ├── ISSUE_TEMPLATE │ ├── 1-feature-request.yml │ ├── 2-bug.yml │ └── 3-release.md └── workflows │ └── tests.yml ├── .gitignore ├── LICENSE ├── README.md ├── pyproject.toml ├── requirements-tests.txt ├── requirements.txt ├── structured ├── __init__.py ├── base_types.py ├── hint_types │ ├── __init__.py │ ├── arrays.py │ ├── basic_types.py │ ├── condition.py │ ├── serialize_as.py │ ├── strings.py │ └── tuples.py ├── py.typed ├── serializers │ ├── __init__.py │ ├── api.py │ ├── arrays.py │ ├── conditional.py │ ├── self.py │ ├── strings.py │ ├── structs.py │ ├── structured.py │ ├── tuples.py │ └── unions.py ├── structured.py ├── type_checking.py └── utils.py └── tests ├── __init__.py ├── test_annotations.py ├── test_arrays.py ├── test_base_types.py ├── test_conditional.py ├── test_generics.py ├── test_self.py ├── test_serializers.py ├── test_strings.py ├── test_structured.py ├── test_tuples.py ├── test_unions.py └── test_utils.py /.github/ISSUE_TEMPLATE/1-feature-request.yml: -------------------------------------------------------------------------------- 1 | name: Feature Request 2 | description: Request new functionality. 3 | title: "[REQUEST]: " 4 | labels: ["enhancement"] 5 | assignees: 6 | - lojack5 7 | body: 8 | - type: markdown 9 | attributes: 10 | value: | 11 | Not quite everything you need in `structured`? Tell us how we can improve it! 12 | - type: textarea 13 | id: feature 14 | attributes: 15 | label: Description 16 | description: Let us know the feature you're looking for, and how it'd work. 17 | validations: 18 | required: true 19 | - type: textarea 20 | id: usecase 21 | attributes: 22 | label: Use 23 | description: Let us know what the feature would be useful for. This could be your specific use case, or even other ways it could be used. 24 | - type: textarea 25 | id: more 26 | attributes: 27 | label: Anything else? 28 | description: If there are any other details, feel free to let us know. 29 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/2-bug.yml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: File a bug report. 3 | title: "[BUG]: " 4 | labels: ["bug"] 5 | assignees: 6 | - lojack5 7 | body: 8 | - type: markdown 9 | attributes: 10 | value: | 11 | So you found something wrong with `structured`. Some information will help us diagnose and fix it. 12 | - type: textarea 13 | id: reproducer 14 | attributes: 15 | label: Reproducer 16 | description: Give a detailed explanation on how to reproduce the bug, as well as expected behavior and actual behavior. 17 | validations: 18 | required: true 19 | - type: dropdown 20 | id: os 21 | attributes: 22 | label: Operating System 23 | description: What operation system(s) do you observe this bug on? 24 | multiple: true 25 | options: 26 | - Windows 27 | - Linux 28 | - iOS 29 | - Other (specify) 30 | validations: 31 | required: true 32 | - type: input 33 | id: os-text 34 | attributes: 35 | label: Other 36 | description: If your operating system isn't listed, or a more specific operating system if the bug doesn't happen on all versions. 37 | placeholder: ex. Ubunto Linux 24.04.1 LTS 38 | - type: dropdown 39 | id: python 40 | attributes: 41 | label: Python Version 42 | multiple: true 43 | options: 44 | - Python 3.9 45 | - Python 3.10 46 | - Python 3.11 47 | - Python 3.12 48 | - Other (specify) 49 | validations: 50 | required: true 51 | - type: input 52 | id: python-text 53 | attributes: 54 | label: Other 55 | description: If your Python version isn't listed, or a more specific version (for example, only on Python 3.10.4). 56 | placeholder: ex. Pypy 3.10 57 | - type: input 58 | id: structured 59 | attributes: 60 | label: structured version 61 | description: What version of structured does the error happen on? Note if it doesn't happen on the latest release, we may not address the bug. 62 | placeholder: 3.1.0 63 | validations: 64 | required: true 65 | - type: textarea 66 | id: more-info 67 | attributes: 68 | label: Additional Information 69 | description: Provide any additional information you feel is needed to understand this bug. 70 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/3-release.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Prepare a Release 3 | about: For developers, to aid in preparing a new release. 4 | title: "[RELEASE]: " 5 | --- 6 | 7 | A checklist of things that need to be done for a release: 8 | - [ ] Create a release branch `release-` 9 | - [ ] Update documentation if needed 10 | - [ ] Update the version string in `structured/__init__.py` 11 | - [ ] Check for any needed updates from dependencies and update: 12 | - [ ] `pyproject.toml` 13 | - [ ] `requirements.txt` 14 | - [ ] `requirements-tests.txt` 15 | - [ ] Update PyPi information in `pyproject.toml` with other needed changes (ex: supported Python versions) 16 | - [ ] Create a Pull Request for all of these updates. 17 | - [ ] Verify all tests pass. 18 | - [ ] Merge the changes into `main`. 19 | - [ ] [Create the distributables](https://packaging.python.org/en/latest/tutorials/packaging-projects/#generating-distribution-archives): `py -m build` 20 | - [ ] [Upload the distributables to PyPi](https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives) `py -m twine upload` 21 | - [ ] Create a release on Github from the head of `main`. 22 | - [ ] Close this issue. 23 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: tests 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | branches: [ main ] 11 | 12 | jobs: 13 | build: 14 | 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | python-version: ["3.9", "3.10", "3.11", "3.12"] 19 | os: [ubuntu-latest, windows-latest, macos-latest] 20 | 21 | runs-on: ${{ matrix.os }} 22 | 23 | steps: 24 | - name: Checkout code 25 | uses: actions/checkout@v3 26 | with: 27 | fetch-depth: 1 28 | 29 | - name: Set up Python ${{ matrix.python-version }} 30 | uses: actions/setup-python@v4 31 | with: 32 | python-version: ${{ matrix.python-version }} 33 | cache: pip 34 | cache-dependency-path: requirements-tests.txt 35 | 36 | - name: Install dependencies 37 | run: pip install -r requirements-tests.txt 38 | 39 | - name: Test with pytest 40 | run: pytest 41 | 42 | - name: Check formatting with black 43 | uses: psf/black@stable 44 | with: 45 | src: "./structured" 46 | 47 | - name: Check formatting with isort 48 | run: isort structured 49 | 50 | - name: Lint with flake8 51 | run: | 52 | # Use the flake8-pyproject entry point to pull in config from 53 | # pyproject.toml 54 | flake8p structured 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv* 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | # VSCode 163 | .vscode/ 164 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2022, Lojack 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.coverage.run] 2 | branch = true 3 | source = ['structured'] 4 | omit = ['structured/type_checking.py'] 5 | 6 | [tool.coverage.report] 7 | show_missing = true 8 | skip_covered = true 9 | exclude_lines = [ 10 | 'pragma: no cover', 11 | 'raise NotImplementedError', 12 | 'if (typing.)?TYPE_CHECKING:', 13 | 'if __name__ == .__main__.:', 14 | '\.\.\.', 15 | ] 16 | 17 | [tool.pytest.ini_options] 18 | testpaths = ['tests'] 19 | addopts = "--cov --cov-report term --cov-report=html --cov-report=xml" 20 | 21 | [build-system] 22 | requires = ['setuptools>=61.0'] 23 | build-backend = 'setuptools.build_meta' 24 | 25 | [tool.setuptools] 26 | package-dir = {"structured" = "structured"} 27 | 28 | [tool.setuptools.dynamic] 29 | version = {attr = 'structured.__version__'} 30 | 31 | [project] 32 | name = 'structured_classes' 33 | dynamic = ['version'] 34 | authors = [ 35 | { name = 'lojack5' }, 36 | ] 37 | description = 'Annotated classes that pack and unpack C structures.' 38 | readme = 'README.md' 39 | license = { text = 'BSD 3-Clause' } 40 | requires-python = '>=3.9' 41 | dependencies = [ 42 | 'typing_extensions~=4.4.0; python_version < "3.11"', 43 | ] 44 | classifiers = [ 45 | 'Intended Audience :: Developers', 46 | 'Programming Language :: Python :: 3 :: Only', 47 | 'Programming Language :: Python :: 3.9', 48 | 'Programming Language :: Python :: 3.10', 49 | 'Programming Language :: Python :: 3.11', 50 | 'Programming Language :: Python :: 3.12', 51 | 'License :: OSI Approved :: BSD License', 52 | 'Operating System :: OS Independent', 53 | 'Typing :: Typed', 54 | ] 55 | 56 | [project.urls] 57 | 'Homepage' = 'https://github.com/lojack5/structured' 58 | 'Bug Tracker' = 'https://github.com/lojack5/structured/issues' 59 | 60 | [tool.black] 61 | skip-string-normalization = true 62 | 63 | [tool.flake8] 64 | max-line-length = 88 65 | extend-ignore = ['E203'] 66 | per-file-ignores = [ 67 | # F401: Imported but unused 68 | # F403: Star import used 69 | 'type_checking.py: F401', 70 | '__init__.py: F403, F401', 71 | ] 72 | 73 | [tool.isort] 74 | profile = 'black' 75 | -------------------------------------------------------------------------------- /requirements-tests.txt: -------------------------------------------------------------------------------- 1 | # Base requirements 2 | -r requirements.txt 3 | 4 | # Tests / linting / formatters 5 | pytest~=8.1.1 6 | pytest-cov~=4.1.0 7 | black~=24.2.0 8 | isort~=5.13.2 9 | flake8~=7.0.0 10 | Flake8-pyproject~=1.2.3 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | typing_extensions~=4.4.0; python_version < "3.11" 2 | -------------------------------------------------------------------------------- /structured/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'lojack5' 2 | __version__ = '3.1.0' 3 | 4 | from .base_types import * 5 | from .hint_types import * 6 | from .serializers import * 7 | from .structured import * 8 | -------------------------------------------------------------------------------- /structured/base_types.py: -------------------------------------------------------------------------------- 1 | """ 2 | A few types with no other obvious place to put them. 3 | """ 4 | 5 | __all__ = [ 6 | 'ByteOrder', 7 | 'ByteOrderMode', 8 | ] 9 | 10 | from enum import Enum 11 | 12 | from .type_checking import Any, annotated, safe_issubclass 13 | 14 | 15 | class ByteOrder(str, Enum): 16 | """Byte order specifiers for passing to the struct module. See the stdlib 17 | documentation for details on what each means. 18 | """ 19 | 20 | DEFAULT = '' 21 | LITTLE_ENDIAN = '<' 22 | LE = LITTLE_ENDIAN 23 | BIG_ENDIAN = '>' 24 | BE = BIG_ENDIAN 25 | NATIVE_STANDARD = '=' 26 | NATIVE_NATIVE = '@' 27 | NETWORK = '!' 28 | 29 | 30 | class ByteOrderMode(str, Enum): 31 | """How derived classes with conflicting byte order markings should function.""" 32 | 33 | OVERRIDE = 'override' 34 | STRICT = 'strict' 35 | 36 | 37 | class requires_indexing: 38 | """Marker base class to indicate a class must be indexed in order to get a 39 | true Serializer. 40 | """ 41 | 42 | @staticmethod 43 | def _transform(base_type: Any, hint: Any) -> Any: 44 | for a in (base_type, hint): 45 | if safe_issubclass(a, requires_indexing): 46 | raise TypeError(f'{a.__name__} must be indexed.') 47 | 48 | 49 | annotated.register_transform(requires_indexing._transform) 50 | -------------------------------------------------------------------------------- /structured/hint_types/__init__.py: -------------------------------------------------------------------------------- 1 | from .arrays import * 2 | from .basic_types import * 3 | from .condition import * 4 | from .serialize_as import * 5 | from .strings import * 6 | from .tuples import * 7 | -------------------------------------------------------------------------------- /structured/hint_types/arrays.py: -------------------------------------------------------------------------------- 1 | """ 2 | Array typhinting classes to return serializers. 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | __all__ = [ 8 | 'array', 9 | 'Header', 10 | ] 11 | 12 | from functools import cache 13 | 14 | from ..base_types import requires_indexing 15 | from ..serializers import ( 16 | ArraySerializer, 17 | DynamicStructArraySerializer, 18 | NullSerializer, 19 | Serializer, 20 | StaticStructArraySerializer, 21 | StructSerializer, 22 | ) 23 | from ..type_checking import Annotated, Generic, Optional, S, Self, T, TypeVar, annotated 24 | from ..utils import StructuredAlias 25 | from .basic_types import _SizeTypes 26 | 27 | 28 | class Header: 29 | """Dispatching class for creating header serializers.""" 30 | 31 | def __init__( 32 | self, 33 | count: int | StructSerializer[int], 34 | data_size: Optional[StructSerializer[int]], 35 | ) -> None: 36 | self.count = count 37 | self.data_size = data_size 38 | 39 | @classmethod 40 | def __class_getitem__(cls, args) -> Self: 41 | # Unpack arguments 42 | if not isinstance(args, tuple): 43 | # Length argument only 44 | length_kind = args 45 | size_kind = None 46 | elif len(args) == 2: 47 | # Length and size check arguments 48 | length_kind, size_kind = args 49 | else: 50 | raise TypeError(f'{cls.__name__}[] expected 1 or 2 arguments, got {args}') 51 | # Indirection so we can cache on the full arguments 52 | return cls._create(length_kind, size_kind) 53 | 54 | @classmethod 55 | @cache 56 | def _create( 57 | cls, 58 | length_kind: int | TypeVar | StructSerializer[int], 59 | size_kind: None | TypeVar | StructSerializer[int], 60 | ) -> Self: 61 | # TypeVar check 62 | if isinstance(length_kind, TypeVar) or isinstance(size_kind, TypeVar): 63 | return StructuredAlias(cls, (length_kind, size_kind)) # type: ignore 64 | # Check length argument 65 | if length_kind in _SizeTypes: 66 | length_kind = annotated.transform(length_kind) 67 | elif isinstance(length_kind, int): 68 | if length_kind < 0: 69 | raise ValueError( 70 | f'array length must be non-negative, got {length_kind}' 71 | ) 72 | else: 73 | raise TypeError(f'invalid array length type: {length_kind!r}') 74 | # Check size argument 75 | if size_kind is not None and size_kind not in _SizeTypes: 76 | raise TypeError(f'invalid array size check type: {size_kind!r}') 77 | else: 78 | size_kind = annotated.transform(size_kind) 79 | # All good 80 | return cls(length_kind, size_kind) 81 | 82 | 83 | class array(Generic[S, T], list[T], requires_indexing): 84 | """Dispatching class used for typehinting to create ArraySerializers""" 85 | 86 | @classmethod 87 | @cache 88 | def __class_getitem__(cls, args) -> type[list[T]]: 89 | if not isinstance(args, tuple) or len(args) != 2: 90 | raise TypeError(f'{cls.__name__}[] expected 2 arguments, got {args!r}') 91 | header, item_type = args 92 | # TypeVar checks 93 | if isinstance(header, StructuredAlias) or isinstance(item_type, TypeVar): 94 | return StructuredAlias(cls, (header, item_type)) # type: ignore 95 | elif not isinstance(header, Header): 96 | raise TypeError(f'invalid array header type: {header!r}') 97 | # Item type checks 98 | item_serializer = annotated.transform(item_type) 99 | if not isinstance(item_serializer, Serializer): 100 | raise TypeError(f'invalid array item type: {item_type!r}') 101 | elif item_serializer.is_final(): 102 | raise TypeError( 103 | f'invalid array item type: {item_type!r} contains final ' 104 | f'{item_serializer.get_final()}. Final serializers cannot be used.' 105 | ) 106 | # All good, check for specializations for struct.Struct unpackable 107 | if ( 108 | isinstance(item_serializer, StructSerializer) 109 | and item_serializer.num_values == 1 110 | ): 111 | if isinstance(header.count, int): 112 | return Annotated[ 113 | list[T], StaticStructArraySerializer(header.count, item_serializer) 114 | ] 115 | else: 116 | return Annotated[ 117 | list[T], DynamicStructArraySerializer(header.count, item_serializer) 118 | ] 119 | # General array serializer 120 | if isinstance(header.count, int): 121 | # Static length 122 | static_length = header.count 123 | if header.data_size is None: 124 | header_serializer = NullSerializer() # no size check 125 | else: 126 | header_serializer = header.data_size # with size check 127 | else: 128 | # Dynamic length 129 | static_length = -1 130 | if header.data_size is None: 131 | header_serializer = header.count # no size check 132 | else: 133 | header_serializer = header.count + header.data_size 134 | return Annotated[ 135 | list[T], 136 | ArraySerializer[T](header_serializer, item_serializer, static_length), 137 | ] 138 | -------------------------------------------------------------------------------- /structured/hint_types/basic_types.py: -------------------------------------------------------------------------------- 1 | """ 2 | Basic typhints for types with direct struct serialization. 3 | """ 4 | 5 | __all__ = [ 6 | 'bool8', 7 | 'int8', 8 | 'uint8', 9 | 'int16', 10 | 'uint16', 11 | 'int32', 12 | 'uint32', 13 | 'int64', 14 | 'uint64', 15 | 'float16', 16 | 'float32', 17 | 'float64', 18 | 'pad', 19 | 'pascal', 20 | ] 21 | 22 | from ..base_types import requires_indexing 23 | from ..serializers import StructSerializer 24 | from ..type_checking import Annotated, ClassVar, TypeVar 25 | 26 | bool8 = Annotated[bool, StructSerializer[bool]('?')] 27 | int8 = Annotated[int, StructSerializer[int]('b')] 28 | uint8 = Annotated[int, StructSerializer[int]('B')] 29 | int16 = Annotated[int, StructSerializer[int]('h')] 30 | uint16 = Annotated[int, StructSerializer[int]('H')] 31 | int32 = Annotated[int, StructSerializer[int]('i')] 32 | uint32 = Annotated[int, StructSerializer[int]('I')] 33 | int64 = Annotated[int, StructSerializer[int]('q')] 34 | uint64 = Annotated[int, StructSerializer[int]('Q')] 35 | float16 = Annotated[float, StructSerializer[float]('e')] 36 | float32 = Annotated[float, StructSerializer[float]('f')] 37 | float64 = Annotated[float, StructSerializer[float]('d')] 38 | 39 | 40 | _SizeTypes = (uint8, uint16, uint32, uint64) 41 | _TSize = TypeVar('_TSize', uint8, uint16, uint32, uint64) 42 | 43 | 44 | class counted(requires_indexing): 45 | """Base class for simple StructSerializers which have a count argument 46 | before the format specifier. For example `char[10]` and `pad[13]`. 47 | """ 48 | 49 | serializer: ClassVar[StructSerializer] 50 | value_type: ClassVar[type] 51 | 52 | def __class_getitem__(cls, count: int) -> Annotated: 53 | # Use matrix multiplication operator, to fold in strings, 54 | # ie 's' @ 2 -> '2s', whereas 's' * 2 -> 'ss' 55 | return Annotated[cls.value_type, cls.serializer @ count] # type: ignore 56 | 57 | 58 | class pad(counted): 59 | """Represents one (or more, via pad[x]) padding bytes in the format string. 60 | Padding bytes are discarded when read, and are written zeroed out. 61 | """ 62 | 63 | serializer = StructSerializer('x') 64 | value_type = type(None) 65 | 66 | 67 | class pascal(bytes, counted): 68 | """String format specifier (bytes in Python). See 'p' in the stdlib struct 69 | documentation for specific details. 70 | """ 71 | 72 | serializer = StructSerializer[bytes]('p') 73 | value_type = bytes 74 | -------------------------------------------------------------------------------- /structured/hint_types/condition.py: -------------------------------------------------------------------------------- 1 | """ 2 | Class to use with Annotated to specify a custom type for serialization. 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | __all__ = [ 8 | 'Condition', 9 | ] 10 | 11 | from ..serializers import ConditionalSerializer, Serializer 12 | from ..type_checking import ( 13 | TYPE_CHECKING, 14 | Any, 15 | Callable, 16 | Generic, 17 | S, 18 | Ts, 19 | TypeVar, 20 | Unpack, 21 | annotated, 22 | ) 23 | 24 | if TYPE_CHECKING: 25 | from ..structured import Structured 26 | 27 | TStructured = TypeVar('TStructured', bound=Structured) 28 | else: 29 | TStructured = 'Structured' 30 | 31 | 32 | class Condition(Generic[S, Unpack[Ts]]): 33 | def __init__( 34 | self, condition: Callable[[TStructured], bool], *defaults: tuple[Unpack[Ts]] 35 | ) -> None: 36 | self.condition = condition 37 | self.defaults = defaults 38 | 39 | @staticmethod 40 | def _transform(base_type: Any, hint: Any) -> Any: 41 | if isinstance(hint, Condition): 42 | if not isinstance(base_type, Serializer): 43 | raise TypeError( 44 | 'Condition must be paired with a serialized type, got ' 45 | f'{base_type}' 46 | ) 47 | return ConditionalSerializer(base_type, hint.condition, hint.defaults) 48 | 49 | 50 | annotated.register_transform(Condition._transform) 51 | -------------------------------------------------------------------------------- /structured/hint_types/serialize_as.py: -------------------------------------------------------------------------------- 1 | """ 2 | Class to use with Annotated to specify a custom type for serialization. 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | __all__ = [ 8 | 'SerializeAs', 9 | ] 10 | 11 | from ..serializers import StructActionSerializer, StructSerializer 12 | from ..type_checking import Any, Callable, Generic, S, Self, T, annotated 13 | 14 | 15 | class SerializeAs(Generic[S, T]): 16 | __slots__ = ('serializer',) 17 | serializer: StructSerializer 18 | 19 | def __init__(self, hint: S) -> None: 20 | serializer = annotated.transform(hint) 21 | if not isinstance(serializer, StructSerializer): 22 | raise TypeError(f'SerializeAs requires a basic type, got {hint}') 23 | elif serializer.num_values != 1: 24 | raise TypeError( 25 | f'SerializeAs requires a basic type with one value, got {serializer}' 26 | ) 27 | self.serializer = serializer 28 | 29 | def with_factory(self, action: Callable[[S], T]) -> Self: 30 | """Specify a factory method for creating your type from the unpacked type.""" 31 | st = self.serializer 32 | return type(self)(StructActionSerializer(st.format, actions=(action,))) 33 | 34 | @staticmethod 35 | def _transform(base_type: Any, hint: Any) -> Any: 36 | if isinstance(hint, SerializeAs): 37 | st = hint.serializer 38 | if isinstance(st, StructActionSerializer): 39 | return st 40 | else: 41 | return StructActionSerializer(st.format, actions=(base_type,)) 42 | 43 | 44 | annotated.register_transform(SerializeAs._transform) 45 | -------------------------------------------------------------------------------- /structured/hint_types/strings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Char and Unicode typhint classes, which dispatch to the appropriate serializer 3 | based on provided specialization args. 4 | """ 5 | 6 | __all__ = [ 7 | 'EncoderDecoder', 8 | 'unicode', 9 | 'char', 10 | 'null_unicode', 11 | 'null_char', 12 | 'NET', 13 | ] 14 | 15 | import math 16 | from functools import cache, partial 17 | 18 | from ..base_types import requires_indexing 19 | from ..serializers import ( 20 | ConsumingCharSerializer, 21 | DynamicCharSerializer, 22 | NETCharSerializer, 23 | Serializer, 24 | TCharSerializer, 25 | TerminatedCharSerializer, 26 | UnicodeSerializer, 27 | static_char_serializer, 28 | ) 29 | from ..type_checking import Annotated, TypeVar, Union, annotated, cast 30 | from ..utils import StructuredAlias 31 | from .basic_types import _SizeTypes, _TSize 32 | 33 | 34 | class NET: 35 | """Marker class for denoting .NET strings.""" 36 | 37 | 38 | class char(bytes, requires_indexing): 39 | """A bytestring, with three ways of denoting length. If size is an integer, 40 | it is a static size. If a uint* type is specified, it is prefixed with 41 | a packed value of that type which holds the length. If the NET type is 42 | specified, uses the variable (1-2 bytes) .NET string size marker. 43 | 44 | char[3] - statically sized. 45 | char[uint32] - dynamically sized. 46 | char[NET] - dynamically sized. 47 | 48 | :param size: The size of the bytestring. 49 | :type size: Union[int, 50 | type[Union[uint8, uint16, uint32, uint64]], 51 | type[NET]] 52 | """ 53 | 54 | def __class_getitem__(cls, args) -> TCharSerializer: 55 | """Create a char specialization.""" 56 | if not isinstance(args, tuple): 57 | args = (args,) 58 | return cls._create(*args) 59 | 60 | @classmethod 61 | @cache 62 | def _create( 63 | cls, 64 | count: Union[int, type[_TSize], type[NET]], 65 | ) -> TCharSerializer: 66 | if count in _SizeTypes: 67 | count = annotated.transform(count) 68 | serializer = DynamicCharSerializer(count) 69 | elif isinstance(count, int): 70 | serializer = static_char_serializer(count) 71 | elif count is NET: 72 | serializer = NETCharSerializer() 73 | elif isinstance(count, TypeVar): 74 | return StructuredAlias(cls, (count,)) # type: ignore 75 | elif isinstance(count, bytes): 76 | serializer = TerminatedCharSerializer(count) 77 | elif math.isinf(count) and count > 0: 78 | serializer = ConsumingCharSerializer() 79 | else: 80 | raise TypeError( 81 | f'{cls.__qualname__}[] count must be an int, NET, terminator ' 82 | f'byte, or uint* type, got {count!r}' 83 | ) 84 | return Annotated[bytes, serializer] # type: ignore 85 | 86 | 87 | class EncoderDecoder: 88 | """Base class for creating custom encoding/decoding methods for strings. 89 | Subclass and implement encode for encoding a string to bytes, and decode 90 | for decoding a string from bytes. 91 | """ 92 | 93 | @classmethod 94 | def encode(cls, strng: str) -> bytes: 95 | """Encode `strng`. 96 | 97 | :param strng: String to encode. 98 | :return: The encoded bytestring. 99 | """ 100 | raise NotImplementedError 101 | 102 | @classmethod 103 | def decode(cls, byts: bytes) -> str: 104 | """Decode `byts`. 105 | 106 | :param byts: The bytestring to decode. 107 | :return: The decoded string. 108 | :rtype: str 109 | """ 110 | raise NotImplementedError 111 | 112 | 113 | class unicode(str, requires_indexing): 114 | """A char-like type which is automatically encoded when packing and decoded 115 | when unpacking. Arguments are the same as for char, with an additional 116 | optional argument `encoding`. If encoding is a string, it is the name of 117 | a python standard codec to use. Otherwise, it must be a subclass of 118 | `EncoderDecoder` to provide the encoding and decoding methods. 119 | 120 | :param size: The size of the *encoded* string. 121 | :type size: Union[int, 122 | Union[type[uint8, uint16, uint32, uint64]], 123 | type[NET]] 124 | :param encoding: Encoding method to use. 125 | :type encoding: Union[str, type[EncoderDecoder]] 126 | """ 127 | 128 | @classmethod 129 | def __class_getitem__(cls, args) -> Serializer[str]: 130 | """Create the specialization.""" 131 | if not isinstance(args, tuple): 132 | args = (args,) 133 | return cls.create(*args) 134 | 135 | @classmethod 136 | def create( 137 | cls, 138 | count: Union[int, type[_TSize], type[NET]], 139 | encoding: Union[str, type[EncoderDecoder]] = 'utf8', 140 | ) -> Serializer[str]: 141 | # Double indirection so @cache can see the default args 142 | # as actual args. 143 | return cls._create(count, encoding) 144 | 145 | @classmethod 146 | @cache 147 | def _create( 148 | cls, 149 | count: Union[int, type[_TSize], type[NET]], 150 | encoding: Union[str, type[EncoderDecoder]], 151 | ) -> Serializer[str]: 152 | """Create the specialization. 153 | 154 | :param count: Size of the *encoded* string. 155 | :param encoding: Encoding method to use. 156 | :return: The specialized class. 157 | """ 158 | # Encoding/Decoding method 159 | if isinstance(count, TypeVar): 160 | return StructuredAlias(cls, (count, encoding)) # type: ignore 161 | if isinstance(encoding, str): 162 | encoder = partial(str.encode, encoding=encoding) 163 | decoder = partial(bytes.decode, encoding=encoding) 164 | elif ( 165 | isinstance(encoding, type) 166 | and issubclass(encoding, EncoderDecoder) 167 | or isinstance(encoding, EncoderDecoder) 168 | ): 169 | encoder = encoding.encode 170 | decoder = encoding.decode 171 | else: 172 | raise TypeError('An encoding or an EncoderDecoder must be specified.') 173 | serializer = annotated.transform(char[count]) 174 | serializer = cast(TCharSerializer, serializer) # definitely is at this point 175 | return Annotated[ 176 | str, UnicodeSerializer(serializer, encoder, decoder) 177 | ] # type: ignore 178 | 179 | 180 | null_char = char[b'\0'] 181 | null_unicode = unicode[b'\0'] 182 | -------------------------------------------------------------------------------- /structured/hint_types/tuples.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provides transformations for tuple type hints into serializers. 3 | """ 4 | 5 | from ..serializers import Serializer, TupleSerializer 6 | from ..type_checking import Any, Tuple, TypeVar, annotated, get_tuple_args, istuple 7 | from ..utils import StructuredAlias 8 | 9 | 10 | def transform_tuple(base_type: Any, hint: Any) -> Any: 11 | if istuple(base_type): 12 | if tuple_args := get_tuple_args(base_type): 13 | if any(isinstance(arg, (TypeVar, StructuredAlias)) for arg in tuple_args): 14 | return StructuredAlias(Tuple, tuple_args) 15 | serializers = [annotated.transform(x) for x in tuple_args] 16 | if all(isinstance(x, Serializer) for x in serializers): 17 | return TupleSerializer(serializers) 18 | 19 | 20 | annotated.register_transform(transform_tuple) 21 | -------------------------------------------------------------------------------- /structured/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lojack5/structured/00e5a465932182783a101c7bbc0b2280ced7a25e/structured/py.typed -------------------------------------------------------------------------------- /structured/serializers/__init__.py: -------------------------------------------------------------------------------- 1 | from .api import * 2 | from .arrays import * 3 | from .conditional import * 4 | from .self import * 5 | from .strings import * 6 | from .structs import * 7 | from .structured import * 8 | from .tuples import * 9 | from .unions import * 10 | -------------------------------------------------------------------------------- /structured/serializers/api.py: -------------------------------------------------------------------------------- 1 | """Defines the base Serializer API. 2 | 3 | The Serializer API is almost identical to struct.Struct, with a few additions 4 | and one alteration: 5 | - New methods 'prepack' and 'preunpack'. 6 | - New attribute `num_values`. 7 | - New unpacking method `unpack_read`. 8 | - New packing method `pack_read`. 9 | - New configuration method `with_byte_order`. 10 | - Modified packing method `pack` 11 | - All unpacking methods may return an iterable of values instead of a tuple. 12 | For more details, check the docstrings on each method or attribute. 13 | 14 | A note on "container" serializers (for example, CompoundSerializer and 15 | ArraySerializer): Due to the posibility of recursive nesting via the 16 | `typing.Self` type-hint as a serializable type, care must be taken with 17 | delegating to sub-serializers. In particular, only updating `self.size` at the 18 | *end* of a pack/unpack operation ensures that nested usages of the same 19 | serializer won't overwrite intermediate values. 20 | 21 | Similarly (although this is true regardless of nesting), you almost always want 22 | a custom `prepack` and `preunpack` method, to pass that information along to 23 | the nested serializers. 24 | """ 25 | 26 | from __future__ import annotations 27 | 28 | __all__ = [ 29 | 'Serializer', 30 | 'NullSerializer', 31 | 'CompoundSerializer', 32 | ] 33 | 34 | from io import BytesIO 35 | 36 | from ..base_types import ByteOrder 37 | from ..type_checking import ( 38 | Any, 39 | BinaryIO, 40 | ClassVar, 41 | Generic, 42 | Iterable, 43 | Optional, 44 | ReadableBuffer, 45 | Self, 46 | Ss, 47 | Ts, 48 | TypeVar, 49 | Unpack, 50 | WritableBuffer, 51 | ) 52 | 53 | 54 | class Serializer(Generic[Unpack[Ts]]): 55 | size: int 56 | """A possibly dynamic attribute indicating the size in bytes for this 57 | Serializer to pack or unpack. Due to serializers dealing with possibly 58 | dynamic data, this is only guaranteed to be up to date with the most 59 | recently called `pack*` or `unpack*` method. Also note, serializers are 60 | shared between classes, so you really must access `size` *immediately* after 61 | one of these calls to ensure it's accurate. 62 | """ 63 | num_values: int 64 | """Indicates the number of variables returned from an unpack operation, and 65 | the number of varialbes required for a pack operation. 66 | """ 67 | 68 | def prepack(self, partial_object: Any) -> Serializer: 69 | """Perform any state logic needed just prior to a pack operation on 70 | `partial_object`. The object will be a fully initialized instance 71 | for pack operations, but only a proxy object for unpack operations. 72 | Durin unpacking, only the attributes unpacked before this serializer are 73 | set on the object. 74 | 75 | :param partial_object: The object being packed or unpacked. 76 | :return: A serializer appropriate for unpacking the next attribute(s). 77 | """ 78 | return self 79 | 80 | def preunpack(self, partial_object: Any) -> Serializer: 81 | """Perform any state logic needed just prior to an unpack operation 82 | on `partial_object`. The object will be a fully initialized instance 83 | for pack operations, but only a proxy object for unpack operations. 84 | Durin unpacking, only the attributes unpacked before this serializer are 85 | set on the object. 86 | 87 | :param partial_object: The object being packed or unpacked. 88 | :return: A serializer appropriate for unpacking the next attribute(s). 89 | """ 90 | return self 91 | 92 | def pack(self, *values: Unpack[Ts]) -> bytes: 93 | """Pack the given values according to this Serializer's logic, returning 94 | the packed bytes. 95 | 96 | :return: The packed bytes version of the values. 97 | """ 98 | raise NotImplementedError 99 | 100 | def pack_into( 101 | self, 102 | buffer: WritableBuffer, 103 | offset: int, 104 | *values: Unpack[Ts], 105 | ) -> None: 106 | """Pack the given values according to this Serializer's logic, placing 107 | them into a buffer supporting the Buffer Protocol. 108 | 109 | :param buffer: An object supporting the Buffer Protocol. 110 | :param offset: Location in the buffer to place the packed bytes. 111 | """ 112 | raise NotImplementedError 113 | 114 | def pack_write(self, writable: BinaryIO, *values: Unpack[Ts]) -> None: 115 | """Pack the given values according to this Serializer's logic, placing 116 | them into a writable file-like object. 117 | 118 | :param writable: A writable file-like object. 119 | """ 120 | raise NotImplementedError 121 | 122 | def unpack(self, buffer: ReadableBuffer) -> Iterable: 123 | """Unpack values from a bytes-like buffer, returning the values in a 124 | tuple. Unlike `struct.pack`, the Serializer must accept a buffer that 125 | is larger than the needed number of bytes for unpacking. 126 | 127 | :param buffer: A readable bytes-like object. 128 | :return: The unpacked values in a tuple. 129 | """ 130 | raise NotImplementedError 131 | 132 | def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> Iterable: 133 | """Unpack values from a buffer supporting the Buffer Protocol, returning 134 | the values in a tuple. 135 | 136 | :param buffer: A readable object supporing the Buffer Protocol. 137 | :param offset: Location in the buffer to draw data from. 138 | :return: The unpacked values in a tuple. 139 | """ 140 | raise NotImplementedError 141 | 142 | def unpack_read(self, readable: BinaryIO) -> Iterable: 143 | """Unpack values from a readable file-like object, returning the values 144 | in a tuple. 145 | 146 | :param readable: A readable file-like object. 147 | :return: The unpacked values in a tuple. 148 | """ 149 | raise NotImplementedError 150 | 151 | # Internal methods useful for configuring / combining serializers 152 | def with_byte_order(self, byte_order: ByteOrder) -> Self: 153 | """Create a serializer with the same packing / unpacking logic, but 154 | configured to use the specified byte order. 155 | 156 | :param byte_order: ByteOrder to use with the new serializer. 157 | :return: A new serializer, or this one if no changes were needed. 158 | """ 159 | return self 160 | 161 | def is_final(self) -> bool: 162 | """Indicates if this serializer must be the final serializer in a 163 | chain. 164 | """ 165 | return self.get_final() is not None 166 | 167 | def get_final(self) -> Optional[Serializer]: 168 | """Get the serializer (if any) that makes this serializer the final 169 | serializer. 170 | """ 171 | return None 172 | 173 | def __add__( 174 | self, other: Serializer[Unpack[Ss]] 175 | ) -> CompoundSerializer[Unpack[Ts], Unpack[Ss]]: 176 | if isinstance(other, NullSerializer): 177 | # Allow __radd__ to work 178 | return NotImplemented 179 | elif self.is_final(): 180 | final = self.get_final() 181 | raise TypeError( 182 | f'{type(self).__name__} must be the final serializer (is or contains' 183 | f' {final}), but is followed by {other}' 184 | ) 185 | if isinstance(other, CompoundSerializer): 186 | # Allow __radd__ to work 187 | return NotImplemented 188 | elif isinstance(other, Serializer): 189 | # Default is to make a CompoundSerializer joining the two. 190 | # Subclasses can provide an __radd__ if optimizing can be done 191 | return CompoundSerializer((self, other)) 192 | return NotImplemented 193 | 194 | 195 | TSerializer = TypeVar('TSerializer', bound=Serializer) 196 | 197 | 198 | class NullSerializer(Serializer[Unpack[tuple[()]]]): 199 | """A dummy serializer to function as the initial value for sum(...)""" 200 | 201 | size: ClassVar[int] = 0 202 | num_values: ClassVar[int] = 0 203 | 204 | def pack(self, *values: Unpack[tuple[()]]) -> bytes: 205 | return b'' 206 | 207 | def pack_into( 208 | self, buffer: WritableBuffer, offset: int, *values: Unpack[tuple[()]] 209 | ) -> None: 210 | return 211 | 212 | def pack_write(self, writable: BinaryIO, *values: Unpack[tuple[()]]) -> None: 213 | return 214 | 215 | def unpack(self, buffer: ReadableBuffer) -> tuple[()]: 216 | return () 217 | 218 | def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> tuple[()]: 219 | return () 220 | 221 | def unpack_read(self, readable: BinaryIO) -> tuple[()]: 222 | return () 223 | 224 | def __add__(self, other: TSerializer) -> TSerializer: 225 | if isinstance(other, Serializer): 226 | return other 227 | return NotImplemented 228 | 229 | def __radd__(self, other: TSerializer) -> TSerializer: 230 | return self.__add__(other) 231 | 232 | 233 | class CompoundSerializer(Generic[Unpack[Ts]], Serializer[Unpack[Ts]]): 234 | """A serializer that chains together multiple serializers.""" 235 | 236 | serializers: tuple[Serializer, ...] 237 | 238 | def __init__(self, serializers: tuple[Serializer, ...]) -> None: 239 | self.serializers = serializers 240 | self.size = 0 241 | self.num_values = sum(serializer.num_values for serializer in serializers) 242 | if any( 243 | isinstance(serializer, CompoundSerializer) for serializer in serializers 244 | ): 245 | raise TypeError('cannot nest CompoundSerializers') 246 | self._needs_preprocess = any( 247 | ((ts := type(serializer)).prepack, ts.preunpack) 248 | != (Serializer.prepack, Serializer.preunpack) 249 | for serializer in serializers 250 | ) 251 | 252 | def get_final(self) -> Optional[Serializer]: 253 | return self.serializers[-1].get_final() 254 | 255 | def prepack(self, partial_object: Any) -> Serializer: 256 | return self.preprocess(partial_object) 257 | 258 | def preunpack(self, partial_object: Any) -> Serializer: 259 | return self.preprocess(partial_object) 260 | 261 | def preprocess(self, partial_object: Any) -> Serializer: 262 | if not self._needs_preprocess: 263 | return self 264 | else: 265 | return _SpecializedCompoundSerializer(self, partial_object) 266 | 267 | def _iter_packers( 268 | self, values: tuple[Unpack[Ts]] 269 | ) -> Iterable[tuple[Serializer, tuple[Any, ...], int]]: 270 | """Common boilerplate needed for iterating over sub-serializers and 271 | tracking which values get sent to which, as well as updating the total 272 | size. 273 | """ 274 | size = 0 275 | i = 0 276 | for serializer in self.serializers: 277 | count = serializer.num_values 278 | yield serializer, values[i : i + count], size 279 | size += serializer.size 280 | i += count 281 | self.size = size 282 | 283 | def pack(self, *values: Unpack[Ts]) -> bytes: 284 | with BytesIO() as out: 285 | for serializer, vals, _ in self._iter_packers(values): 286 | out.write(serializer.pack(*vals)) 287 | return out.getvalue() 288 | 289 | def pack_into( 290 | self, 291 | buffer: WritableBuffer, 292 | offset: int, 293 | *values: Unpack[Ts], 294 | ) -> None: 295 | for serializer, vals, size in self._iter_packers(values): 296 | serializer.pack_into(buffer, offset + size, *vals) 297 | 298 | def pack_write(self, writable: BinaryIO, *values: Unpack[Ts]) -> None: 299 | for serializer, vals, _ in self._iter_packers(values): 300 | serializer.pack_write(writable, *vals) 301 | 302 | def _iter_unpackers(self) -> Iterable[tuple[Serializer, int]]: 303 | """Common boilerplate needed for iterating over sub-serializers and 304 | tracking the total size upacked so far. 305 | """ 306 | size = 0 307 | for serializer in self.serializers: 308 | yield serializer, size 309 | size += serializer.size 310 | self.size = size 311 | 312 | def unpack(self, buffer: ReadableBuffer) -> Iterable: 313 | for serializer, size in self._iter_unpackers(): 314 | yield from serializer.unpack(buffer[size:]) 315 | 316 | def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> Iterable: 317 | for serializer, size in self._iter_unpackers(): 318 | yield from serializer.unpack_from(buffer, offset + size) 319 | 320 | def unpack_read(self, readable: BinaryIO) -> Iterable: 321 | for serializer, _ in self._iter_unpackers(): 322 | yield from serializer.unpack_read(readable) 323 | 324 | def with_byte_order(self, byte_order: ByteOrder) -> Self: 325 | serializers = tuple( 326 | serializer.with_byte_order(byte_order) for serializer in self.serializers 327 | ) 328 | return CompoundSerializer(serializers) 329 | 330 | def __add__( 331 | self, other: Serializer[Unpack[Ss]] 332 | ) -> CompoundSerializer[Unpack[Ts], Unpack[Ss]]: 333 | if isinstance(other, CompoundSerializer): 334 | to_append = list(other.serializers) 335 | elif isinstance(other, Serializer): 336 | to_append = [other] 337 | else: 338 | return super().__add__(other) 339 | serializers = list(self.serializers) 340 | return self._add_impl(serializers, to_append) 341 | 342 | @staticmethod 343 | def _add_impl( 344 | serializers: list[Serializer], to_append: Iterable[Serializer] 345 | ) -> CompoundSerializer: 346 | for candidate in to_append: 347 | # Here is where the .is_final() check happens 348 | joined = serializers[-1] + candidate 349 | if isinstance(joined, CompoundSerializer): 350 | # Don't need to make nested CompoundSerializers 351 | serializers.append(candidate) 352 | else: 353 | serializers[-1] = joined 354 | return CompoundSerializer(tuple(serializers)) 355 | 356 | def __radd__( 357 | self, other: Serializer[Unpack[Ss]] 358 | ) -> CompoundSerializer[Unpack[Ss], Unpack[Ts]]: 359 | # NOTE: CompountSerializer + CompoundSerializer will always call __add__ 360 | # so we only need to optimize for Serializer + CompoundSerializer 361 | if isinstance(other, Serializer): 362 | serializers = [other] 363 | else: 364 | return NotImplemented 365 | to_append = self.serializers[:] 366 | return self._add_impl(serializers, to_append) 367 | 368 | 369 | class _SpecializedCompoundSerializer( 370 | Generic[Unpack[Ts]], CompoundSerializer[Unpack[Ts]] 371 | ): 372 | """CompoundSerializer that will forward a partial_object to sub-serializers, 373 | and update the size of the originating CompoundSerializer. 374 | """ 375 | 376 | def __init__(self, origin: CompoundSerializer, partial_object: Any) -> None: 377 | self.origin = origin 378 | self.partial_object = partial_object 379 | self.serializers = origin.serializers 380 | self.size = origin.size 381 | self.num_values = origin.num_values 382 | 383 | def preprocess(self, partial_object: Any) -> Serializer: 384 | return self 385 | 386 | def _iter_packers( 387 | self, values: tuple[Unpack[Ts]] 388 | ) -> Iterable[tuple[Serializer, tuple[Any, ...], int]]: 389 | size = 0 390 | i = 0 391 | for serializer in self.serializers: 392 | specialized = serializer.prepack(self.partial_object) 393 | count = specialized.num_values 394 | yield specialized, values[i : i + count], size 395 | size += specialized.size 396 | i += count 397 | self.size = size 398 | self.origin.size = size 399 | 400 | def _iter_unpackers(self) -> Iterable[tuple[Serializer, int]]: 401 | size = 0 402 | for serializer in self.serializers: 403 | specialized = serializer.preunpack(self.partial_object) 404 | yield specialized, size 405 | size += specialized.size 406 | self.size = size 407 | self.origin.size = size 408 | -------------------------------------------------------------------------------- /structured/serializers/arrays.py: -------------------------------------------------------------------------------- 1 | """ 2 | All the serializers for arrays. Provides a general purpose ArraySerializer, 3 | plus specializations for arrays containing types with direct serialization via 4 | struct. 5 | """ 6 | 7 | __all__ = [ 8 | 'ArraySerializer', 9 | 'StaticStructArraySerializer', 10 | 'DynamicStructArraySerializer', 11 | 'HeaderSerializer', 12 | ] 13 | 14 | from ..base_types import ByteOrder 15 | from ..type_checking import ( 16 | BinaryIO, 17 | Generic, 18 | ReadableBuffer, 19 | Self, 20 | T, 21 | Union, 22 | Unpack, 23 | WritableBuffer, 24 | ) 25 | from .api import NullSerializer, Serializer 26 | from .structs import StructSerializer 27 | 28 | HeaderSerializer = Union[ 29 | NullSerializer, StructSerializer[int], StructSerializer[int, int] 30 | ] 31 | 32 | 33 | class ArraySerializer(Generic[T], Serializer[list[T]]): 34 | """Generic array serializer.""" 35 | 36 | def __init__( 37 | self, 38 | header_serializer: HeaderSerializer, 39 | item_serializer: Serializer[T], 40 | static_length: int = -1, 41 | ) -> None: 42 | self.header_serializer = header_serializer 43 | self.item_serializer = item_serializer 44 | self.static_length = static_length 45 | self.size = 0 46 | self.num_values = 1 47 | 48 | def with_byte_order(self, byte_order: ByteOrder) -> Self: 49 | return type(self)( 50 | self.header_serializer.with_byte_order(byte_order), 51 | self.item_serializer.with_byte_order(byte_order), 52 | self.static_length, 53 | ) 54 | 55 | def _header_pack_values(self, items: list[T], data_size: int) -> tuple[int, ...]: 56 | """Which values need to be passed to the header serializer for packing.""" 57 | count = len(items) 58 | if self.static_length >= 0: 59 | # Static sized 60 | if count != self.static_length: 61 | raise ValueError( 62 | f'Array length {count} does not match static length ' 63 | f'{self.static_length}' 64 | ) 65 | if self.header_serializer.num_values == 1: 66 | # With a data size 67 | return (data_size,) 68 | else: 69 | return () 70 | else: 71 | # Dynamic sized 72 | if self.header_serializer.num_values == 2: 73 | # With a data size 74 | return count, data_size 75 | else: 76 | return (count,) 77 | 78 | def _header_unpack_values(self, *header_values: int) -> tuple[int, int]: 79 | if self.static_length >= 0: 80 | # Static sized 81 | if self.header_serializer.num_values == 1: 82 | # With a data size 83 | return self.static_length, header_values[0] 84 | else: 85 | return self.static_length, -1 86 | else: 87 | # Dynamic sized 88 | if self.header_serializer.num_values == 2: 89 | # With a data size 90 | return header_values 91 | else: 92 | return header_values[0], -1 93 | 94 | def _check_data_size(self, expected: int, actual: int) -> None: 95 | if expected >= 0 and expected != actual: 96 | raise ValueError( 97 | f'Array data size {actual} does not match expected size {expected}' 98 | ) 99 | 100 | def prepack(self, partial_object) -> Self: 101 | self._partial_object = partial_object 102 | return self 103 | 104 | def preunpack(self, partial_object) -> Self: 105 | self._partial_object = partial_object 106 | return self 107 | 108 | def pack(self, *values: Unpack[tuple[list[T]]]) -> bytes: 109 | data = [b''] 110 | size = header_size = self.header_serializer.size 111 | item_serializer = self.item_serializer.prepack(self._partial_object) 112 | for item in values[0]: 113 | data.append(item_serializer.pack(item)) 114 | size += item_serializer.size 115 | header_values = self._header_pack_values(values[0], size - header_size) 116 | data[0] = self.header_serializer.pack(*header_values) 117 | self.size = size 118 | return b''.join(data) 119 | 120 | def pack_into( 121 | self, buffer: WritableBuffer, offset: int, *values: Unpack[tuple[list[T]]] 122 | ) -> None: 123 | items = values[0] 124 | size = header_size = self.header_serializer.size 125 | item_serializer = self.item_serializer.prepack(self._partial_object) 126 | for item in items: 127 | item_serializer.pack_into(buffer, offset + size, item) 128 | size += item_serializer.size 129 | header_values = self._header_pack_values(items, size - header_size) 130 | self.size = size 131 | self.header_serializer.pack_into(buffer, offset, *header_values) 132 | 133 | def pack_write(self, writable: BinaryIO, *values: Unpack[tuple[list[T]]]) -> None: 134 | # TODO: Why is the typechecker flagging this? 135 | writable.write(self.pack(*values)) # type: ignore 136 | 137 | def unpack(self, buffer: ReadableBuffer) -> tuple[list[T]]: 138 | header = self.header_serializer.unpack(buffer) 139 | count, data_size = self._header_unpack_values(*header) 140 | size = header_size = self.header_serializer.size 141 | item_serializer = self.item_serializer.preunpack(self._partial_object) 142 | items = [] 143 | for _ in range(count): 144 | items.extend(item_serializer.unpack(buffer[size:])) 145 | size += item_serializer.size 146 | self._check_data_size(data_size, size - header_size) 147 | self.size = size 148 | return (items,) 149 | 150 | def unpack_from(self, buffer: ReadableBuffer, offset: int) -> tuple[list[T]]: 151 | header = self.header_serializer.unpack_from(buffer, offset) 152 | count, data_size = self._header_unpack_values(*header) 153 | size = header_size = self.header_serializer.size 154 | item_serializer = self.item_serializer.preunpack(self._partial_object) 155 | items = [] 156 | for _ in range(count): 157 | items.extend(item_serializer.unpack_from(buffer, offset + size)) 158 | size += item_serializer.size 159 | self._check_data_size(data_size, size - header_size) 160 | self.size = size 161 | return (items,) 162 | 163 | def unpack_read(self, readable: BinaryIO) -> tuple[list[T]]: 164 | header = self.header_serializer.unpack_read(readable) 165 | count, data_size = self._header_unpack_values(*header) 166 | size = header_size = self.header_serializer.size 167 | item_serializer = self.item_serializer.preunpack(self._partial_object) 168 | items = [] 169 | for _ in range(count): 170 | items.extend(item_serializer.unpack_read(readable)) 171 | size += item_serializer.size 172 | self._check_data_size(data_size, size - header_size) 173 | self.size = size 174 | return (items,) 175 | 176 | 177 | class StaticStructArraySerializer(Generic[T], Serializer[list[T]]): 178 | """Specialization of ArraySerializer for static length arrays of items 179 | that can be unpacked with struct.Struct 180 | """ 181 | 182 | def __init__(self, count: int, item_serializer: StructSerializer[T]) -> None: 183 | self.count = count 184 | # Need to save the original for with_byte_order 185 | self.item_serializer = item_serializer 186 | self.serializer = item_serializer * count 187 | self.num_values = 1 188 | 189 | @property 190 | def size(self) -> int: 191 | return self.serializer.size 192 | 193 | def with_byte_order(self, byte_order: ByteOrder) -> Self: 194 | return type(self)(self.count, self.item_serializer.with_byte_order(byte_order)) 195 | 196 | def _check_length(self, items: list[T]) -> None: 197 | if len(items) != self.count: 198 | raise ValueError( 199 | f'Array length {len(items)} does not match static length {self.count}' 200 | ) 201 | 202 | def pack(self, *values: Unpack[tuple[list[T]]]) -> bytes: 203 | self._check_length(values[0]) 204 | return self.serializer.pack(*values[0]) 205 | 206 | def pack_into( 207 | self, buffer: WritableBuffer, offset: int, *values: Unpack[tuple[list[T]]] 208 | ) -> None: 209 | self._check_length(values[0]) 210 | self.serializer.pack_into(buffer, offset, *values[0]) 211 | 212 | def pack_write(self, writable: BinaryIO, *values: Unpack[tuple[list[T]]]) -> None: 213 | self._check_length(values[0]) 214 | self.serializer.pack_write(writable, *values[0]) 215 | 216 | def unpack(self, buffer: ReadableBuffer) -> tuple[list[T]]: 217 | return (list(self.serializer.unpack(buffer)),) 218 | 219 | def unpack_from(self, buffer: ReadableBuffer, offset: int) -> tuple[list[T]]: 220 | return (list(self.serializer.unpack_from(buffer, offset)),) 221 | 222 | def unpack_read(self, readable: BinaryIO) -> tuple[list[T]]: 223 | return (list(self.serializer.unpack_read(readable)),) 224 | 225 | 226 | class DynamicStructArraySerializer(Generic[T], Serializer[list[T]]): 227 | """Specialization of ArraySerializer for dynamic length arrays of items 228 | that can be unpacked with struct.Struct 229 | """ 230 | 231 | def __init__( 232 | self, 233 | count_serializer: StructSerializer[int], 234 | item_serializer: StructSerializer[T], 235 | ) -> None: 236 | self.count_serializer = count_serializer 237 | self.item_serializer = item_serializer 238 | self.num_values = 0 239 | self.size = 0 240 | 241 | def with_byte_order(self, byte_order: ByteOrder) -> Self: 242 | return type(self)( 243 | self.count_serializer.with_byte_order(byte_order), 244 | self.item_serializer.with_byte_order(byte_order), 245 | ) 246 | 247 | def _packer(self, values: tuple[list[T]]) -> tuple[Serializer, list[T]]: 248 | items = values[0] 249 | count = len(items) 250 | if count == 0: 251 | # Since we'll be modifying its .num_values, we want a copy 252 | serializer = StructSerializer(self.count_serializer.format) 253 | else: 254 | serializer = self.count_serializer + (self.item_serializer * count) 255 | serializer.num_values -= 1 256 | self.size = serializer.size 257 | return serializer, items 258 | 259 | def pack(self, *values: Unpack[tuple[list[T]]]) -> bytes: 260 | serializer, items = self._packer(values) 261 | return serializer.pack(serializer.num_values, *items) 262 | 263 | def pack_into( 264 | self, buffer: WritableBuffer, offset: int, *values: Unpack[tuple[list[T]]] 265 | ) -> None: 266 | serializer, items = self._packer(values) 267 | serializer.pack_into(buffer, offset, serializer.num_values, *items) 268 | 269 | def pack_write(self, writable: BinaryIO, *values: Unpack[tuple[list[T]]]) -> None: 270 | serializer, items = self._packer(values) 271 | serializer.pack_write(writable, serializer.num_values, *items) 272 | 273 | def unpack(self, buffer: ReadableBuffer) -> tuple[list[T]]: 274 | (count,) = self.count_serializer.unpack(buffer) 275 | size = self.count_serializer.size 276 | serializer = self.item_serializer * count 277 | items = serializer.unpack(buffer[size:]) 278 | self.size = size + serializer.size 279 | return (list(items),) 280 | 281 | def unpack_from(self, buffer: ReadableBuffer, offset: int) -> tuple[list[T]]: 282 | (count,) = self.count_serializer.unpack_from(buffer, offset) 283 | size = self.count_serializer.size 284 | serializer = self.item_serializer * count 285 | items = serializer.unpack_from(buffer, offset + size) 286 | self.size = size + serializer.size 287 | return (list(items),) 288 | 289 | def unpack_read(self, readable: BinaryIO) -> tuple[list[T]]: 290 | (count,) = self.count_serializer.unpack_read(readable) 291 | size = self.count_serializer.size 292 | serializer = self.item_serializer * count 293 | items = serializer.unpack_read(readable) 294 | self.size = size + serializer.size 295 | return (list(items),) 296 | -------------------------------------------------------------------------------- /structured/serializers/conditional.py: -------------------------------------------------------------------------------- 1 | """ 2 | Serializer that wraps another in a condition. When the condition evaluates 3 | to a Truthy value, the original serializer operates as normal. Otherwise, 4 | it acts as if the serializer was not there. 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | __all__ = [ 10 | 'ConditionalSerializer', 11 | ] 12 | 13 | from ..type_checking import ( 14 | TYPE_CHECKING, 15 | Any, 16 | BinaryIO, 17 | Callable, 18 | Generic, 19 | Optional, 20 | ReadableBuffer, 21 | Self, 22 | Ts, 23 | TypeVar, 24 | Unpack, 25 | WritableBuffer, 26 | ) 27 | from .api import ByteOrder, Serializer 28 | 29 | if TYPE_CHECKING: 30 | # *Only* used for type-hinting, so ok to guard with a TYPE_CHECKING 31 | from ..structured import Structured 32 | 33 | TStructured = TypeVar('TStructured', bound=Structured) 34 | else: 35 | TStructured = TypeVar('TStructured') 36 | 37 | 38 | class ConditionalSerializer(Generic[Unpack[Ts]], Serializer[Unpack[Ts]]): 39 | def __init__( 40 | self, 41 | serializer: Serializer[Unpack[Ts]], 42 | condition: Callable[[TStructured], bool], 43 | default: tuple[Unpack[Ts]], 44 | ) -> None: 45 | self.condition = condition 46 | self.serializers: dict[bool, Serializer[Unpack[Ts]]] = { 47 | True: serializer, 48 | False: SkipSerializer(default), 49 | } 50 | self.num_values = serializer.num_values 51 | if serializer.num_values != len(default): 52 | expected = len(default) 53 | raise ValueError( 54 | 'Not enough default arguments provided to Condition, expected ' 55 | f'{self.num_values}, got {expected}' 56 | ) 57 | 58 | def get_final(self) -> Optional[Serializer]: 59 | return self.serializers[True].get_final() 60 | 61 | def with_byte_order(self, byte_order: ByteOrder) -> Self: 62 | serializer = self.serializers[True].with_byte_order(byte_order) 63 | defaults = self.serializers[False].values 64 | return type(self)(serializer, self.condition, defaults) 65 | 66 | def prepack(self, partial_object: Any) -> Serializer[Unpack[Ts]]: 67 | return self.serializers[self.condition(partial_object)] 68 | 69 | def preunpack(self, partial_object: Any) -> Serializer[Unpack[Ts]]: 70 | return self.prepack(partial_object) 71 | 72 | 73 | class SkipSerializer(Generic[Unpack[Ts]], Serializer[Unpack[Ts]]): 74 | size: int = 0 75 | 76 | def __init__(self, values: tuple[Unpack[Ts]]): 77 | self.values = values 78 | self.num_values = len(values) 79 | 80 | def pack(self, *values: Unpack[Ts]) -> bytes: 81 | return b'' 82 | 83 | def pack_into( 84 | self, buffer: WritableBuffer, offset: int, *values: Unpack[Ts] 85 | ) -> None: 86 | pass 87 | 88 | def pack_write(self, writable: BinaryIO, *values: Unpack[Ts]) -> None: 89 | pass 90 | 91 | def unpack(self, buffer: ReadableBuffer) -> tuple[Unpack[Ts]]: 92 | return self.values 93 | 94 | def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> tuple[Unpack[Ts]]: 95 | return self.values 96 | 97 | def unpack_read(self, readable: BinaryIO) -> tuple: 98 | return self.values 99 | -------------------------------------------------------------------------------- /structured/serializers/self.py: -------------------------------------------------------------------------------- 1 | """ 2 | Serializer for special handling of the typing.Self typehint. 3 | """ 4 | 5 | __all__ = [ 6 | 'SelfSerializer', 7 | ] 8 | 9 | 10 | from ..type_checking import TYPE_CHECKING, Any, ClassVar, Self, annotated 11 | from .api import Serializer 12 | from .structured import StructuredSerializer 13 | 14 | if TYPE_CHECKING: 15 | from ..structured import Structured, _Proxy 16 | else: 17 | Structured = 'Structured' 18 | _Proxy = '_Proxy' 19 | 20 | 21 | class SelfSerializer(Serializer[Structured]): 22 | num_values: ClassVar[int] = 1 23 | 24 | def prepack(self, partial_object: Structured) -> Serializer: 25 | return StructuredSerializer(type(partial_object)) 26 | 27 | def preunpack(self, partial_object: _Proxy) -> Serializer: 28 | return StructuredSerializer(partial_object.cls) 29 | 30 | @classmethod 31 | def _transform(cls, base_type: Any, hint: Any) -> Any: 32 | if base_type is Self: 33 | return cls() 34 | 35 | 36 | annotated.register_transform(SelfSerializer._transform) 37 | -------------------------------------------------------------------------------- /structured/serializers/strings.py: -------------------------------------------------------------------------------- 1 | """ 2 | The serializers for packing/unpacking strings. The basic serializers are the 3 | char serializers, then the unicode serializers just wrap those with automatic 4 | encoding/decoding. 5 | """ 6 | 7 | import io 8 | import struct 9 | 10 | __all__ = [ 11 | 'TerminatedCharSerializer', 12 | 'DynamicCharSerializer', 13 | 'ConsumingCharSerializer', 14 | 'NETCharSerializer', 15 | 'static_char_serializer', 16 | 'UnicodeSerializer', 17 | 'TCharSerializer', 18 | ] 19 | 20 | from ..base_types import ByteOrder 21 | from ..type_checking import ( 22 | BinaryIO, 23 | Callable, 24 | ClassVar, 25 | Optional, 26 | ReadableBuffer, 27 | Self, 28 | Union, 29 | Unpack, 30 | WritableBuffer, 31 | cast, 32 | ) 33 | from .api import Serializer 34 | from .structs import StructSerializer 35 | 36 | _single_char = StructSerializer[bytes]('s') 37 | 38 | 39 | def static_char_serializer(count: int) -> StructSerializer[bytes]: 40 | """Create a char serializer for statically sized bytes.""" 41 | return _single_char @ count 42 | 43 | 44 | class DynamicCharSerializer(Serializer[bytes]): 45 | """Serializer for handling variable length strings with their size stored 46 | just prior to the string data. 47 | 48 | :param count_serializer: A StructSerializer unpacking the uint* type holding 49 | the bytestring length. 50 | """ 51 | 52 | num_values: ClassVar[int] = 1 53 | 54 | def __init__(self, count_serializer: StructSerializer[int]) -> None: 55 | self.st = count_serializer 56 | self.size = 0 57 | 58 | def with_byte_order(self, byte_order: ByteOrder) -> Self: 59 | return type(self)(self.st.with_byte_order(byte_order)) 60 | 61 | def _st_count_data( 62 | self, 63 | values: tuple[bytes], 64 | ) -> tuple[StructSerializer[int, bytes], int, bytes]: 65 | """Given data pack as passed in from a Structured object, create a 66 | struct for packing it and its length. Returns the struct, length, and 67 | bytestring to pack. 68 | 69 | :param values: Bytestring as passed from a Structured object. 70 | :return: The struct instance, length, and bystring. 71 | """ 72 | raw_data = values[0] 73 | count = len(raw_data) 74 | st = self.st + _single_char @ count 75 | self.size = st.size 76 | return st, count, raw_data 77 | 78 | def pack(self, *values: Unpack[tuple[bytes]]) -> bytes: 79 | """Pack a dynamically sized bytestring into bytes.""" 80 | st, count, raw = self._st_count_data(values) 81 | return st.pack(count, raw) 82 | 83 | def pack_into( 84 | self, 85 | buffer: WritableBuffer, 86 | offset: int, 87 | *values: Unpack[tuple[bytes]], 88 | ) -> None: 89 | """Pack a dynamically sized bytestring into a buffer supporting the 90 | Buffer Protocol. 91 | 92 | :param buffer: A buffer supporting the Buffer Protocol. 93 | :param offset: Location in the buffer to place the size and bytestring 94 | """ 95 | st, count, raw = self._st_count_data(values) 96 | st.pack_into(buffer, offset, count, raw) 97 | 98 | def pack_write(self, writable: BinaryIO, *values: Unpack[tuple[bytes]]) -> None: 99 | """Pack a dynamically sized bytestring and write it to a file-like 100 | object. 101 | 102 | :param writable: A writable file-like object. 103 | """ 104 | st, count, raw = self._st_count_data(values) 105 | st.pack_write(writable, count, raw) 106 | 107 | def unpack(self, buffer: ReadableBuffer) -> tuple[bytes]: 108 | """Unpack a dynamically sized bytestring from a bytes-like object. 109 | 110 | :param buffer: The bytes-like object holding the length and bytestring. 111 | :return: The unpacked bytestring 112 | """ 113 | count = self.st.unpack(buffer)[0] 114 | self.size = self.st.size + count 115 | return (bytes(buffer[self.st.size : self.size]),) 116 | 117 | def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> tuple[bytes]: 118 | """Unpack a dynamically sized bytestring from a buffer supporting the 119 | Buffer Protocol. 120 | 121 | :param buffer: A buffer supporting the Buffer Protocol. 122 | :param offset: Location in the buffer to the length marker of the 123 | bytestring. 124 | :return: The unpacked bytestring. 125 | """ 126 | count = self.st.unpack_from(buffer, offset)[0] 127 | self.size = self.st.size + count 128 | st = _single_char @ count 129 | return st.unpack_from(buffer, offset + self.st.size) 130 | 131 | def unpack_read(self, readable: BinaryIO) -> tuple: 132 | """Unpack a dynamically sized bytestring from a file-like object. 133 | 134 | :param readable: A readable file-like object. 135 | :return: The unpacked bytestring. 136 | """ 137 | count = self.st.unpack_read(readable)[0] 138 | self.size = self.st.size + count 139 | st = _single_char @ count 140 | return st.unpack_read(readable) 141 | 142 | 143 | class ConsumingCharSerializer(Serializer[bytes]): 144 | """Serializer for using all remainging bytes for the character data.""" 145 | 146 | num_values: ClassVar[int] = 1 147 | 148 | def __init__(self) -> None: 149 | self.size = 0 150 | 151 | def get_final(self) -> Self: 152 | return self 153 | 154 | def pack(self, *values: Unpack[tuple[bytes]]) -> bytes: 155 | self.size = len(values[0]) 156 | return values[0] 157 | 158 | def pack_into( 159 | self, buffer: WritableBuffer, offset: int, *values: Unpack[tuple[bytes]] 160 | ) -> None: 161 | data = values[0] 162 | self.size = len(data) 163 | buffer[offset : offset + self.size] = data 164 | 165 | def pack_write(self, writable: BinaryIO, *values: Unpack[tuple[bytes]]) -> None: 166 | data = values[0] 167 | self.size = len(data) 168 | writable.write(data) 169 | 170 | def unpack(self, buffer: ReadableBuffer) -> tuple[bytes]: 171 | self.size = len(buffer) 172 | return (buffer,) 173 | 174 | def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> tuple[bytes]: 175 | self.size = len(buffer) - offset 176 | return (buffer[offset:],) 177 | 178 | def unpack_read(self, readable: BinaryIO) -> tuple[bytes]: 179 | data = readable.read() 180 | self.size = len(data) 181 | return (data,) 182 | 183 | 184 | class TerminatedCharSerializer(Serializer[bytes]): 185 | """Serializer for handling terminated strings (typically null-terminated).""" 186 | 187 | num_values: ClassVar[int] = 1 188 | size: int 189 | 190 | def __init__(self, terminator: bytes) -> None: 191 | self.size = 0 192 | if len(terminator) != 1: 193 | raise ValueError('string terminator must be a single byte') 194 | self.terminator = terminator 195 | 196 | def _st_data(self, values: tuple[bytes]) -> tuple[StructSerializer, bytes]: 197 | """Common packing logic.""" 198 | raw_data = values[0] 199 | if not raw_data or raw_data[-1] != self.terminator: 200 | # Insert terminator if needed 201 | raw_data += self.terminator 202 | count = len(raw_data) 203 | self.size = count 204 | return _single_char @ count, raw_data 205 | 206 | def pack(self, *values: Unpack[tuple[bytes]]) -> bytes: 207 | st, data = self._st_data(values) 208 | return st.pack(data) 209 | 210 | def pack_into( 211 | self, buffer: WritableBuffer, offset: int, *values: Unpack[tuple[bytes]] 212 | ) -> None: 213 | st, data = self._st_data(values) 214 | st.pack_into(buffer, offset, data) 215 | 216 | def pack_write(self, writable: BinaryIO, *values: Unpack[tuple[bytes]]) -> None: 217 | st, data = self._st_data(values) 218 | st.pack_write(writable, data) 219 | 220 | def unpack(self, buffer: ReadableBuffer) -> tuple[bytes]: 221 | return self.unpack_from(buffer) 222 | 223 | def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> tuple[bytes]: 224 | end = offset 225 | try: 226 | while buffer[end] not in (self.terminator, ord(self.terminator)): 227 | end += 1 228 | except IndexError: 229 | raise ValueError('unterminated string.') from None 230 | self.size = end + 1 231 | return (bytes(buffer[offset:end]),) 232 | 233 | def unpack_read(self, readable: BinaryIO) -> tuple[bytes]: 234 | size = 0 235 | READ_SIZE = 256 236 | start_pos = readable.tell() 237 | with io.BytesIO() as out: 238 | while chunk := readable.read(READ_SIZE): 239 | offset = chunk.find(self.terminator) 240 | if offset == -1: 241 | out.write(chunk) 242 | size += READ_SIZE 243 | else: 244 | out.write(chunk[:offset]) 245 | size += offset + 1 246 | readable.seek(start_pos + size) 247 | break 248 | else: 249 | raise ValueError('unterminated string.') 250 | self.size = size 251 | return (out.getvalue(),) 252 | 253 | 254 | class NETCharSerializer(Serializer[bytes]): 255 | """A .NET string serializer. Note that the variable sized length encoding 256 | is dubious. 257 | """ 258 | 259 | num_values: ClassVar[int] = 1 260 | 261 | def __init__(self) -> None: 262 | # TODO: Determine if we should add the given ByteOrder, or 263 | # always use a specific one (need to find some docs *somewhere* on 264 | # this format, other than old WryeBase code.) 265 | self.short_len = StructSerializer('B') 266 | self.long_len = StructSerializer('H') 267 | self.size = 0 268 | 269 | def _st_count_data( 270 | self, 271 | values: tuple[bytes], 272 | ) -> tuple[StructSerializer, int, bytes]: 273 | raw = values[0] 274 | count = len(raw) 275 | if count < 128: 276 | st = self.short_len + _single_char @ count 277 | elif count > 0x7FFF: 278 | raise ValueError('.NET string length too long to encode.') 279 | else: 280 | st = self.long_len + _single_char @ count 281 | count = 0x80 | count & 0x7F | (count & 0xFF80) << 1 282 | self.size = st.size 283 | return st, count, raw 284 | 285 | def pack(self, *values: Unpack[tuple[bytes]]) -> bytes: 286 | st, count_mark, raw = self._st_count_data(values) 287 | return st.pack(count_mark, raw) 288 | 289 | def pack_into( 290 | self, 291 | buffer: WritableBuffer, 292 | offset: int, 293 | *values: Unpack[tuple[bytes]], 294 | ) -> None: 295 | st, count_mark, raw = self._st_count_data(values) 296 | st.pack_into(buffer, offset, count_mark, raw) 297 | 298 | def pack_write(self, writable: BinaryIO, *values: Unpack[tuple[bytes]]) -> None: 299 | st, count_mark, raw = self._st_count_data(values) 300 | writable.write(st.pack(count_mark, raw)) 301 | 302 | @staticmethod 303 | def _decode_length(count: int) -> int: 304 | count = count & 0x7F | (count >> 1) & 0xFF80 305 | if count > 0x7FFF: 306 | raise ValueError('.NET string length too big to encode.') 307 | return count 308 | 309 | def unpack(self, buffer: ReadableBuffer) -> tuple[bytes]: 310 | count = self.short_len.unpack(buffer)[0] 311 | if count >= 128: 312 | count = self.long_len.unpack(buffer)[0] 313 | count = self._decode_length(count) 314 | size = self.long_len.size 315 | else: 316 | size = self.short_len.size 317 | self.size = size + count 318 | return (cast(bytes, buffer[size : size + count]),) 319 | 320 | def unpack_from( 321 | self, 322 | buffer: ReadableBuffer, 323 | offset: int = 0, 324 | ) -> tuple[bytes]: 325 | count = self.short_len.unpack_from(buffer, offset)[0] 326 | if count >= 128: 327 | count = self.long_len.unpack_from(buffer, offset)[0] 328 | count = self._decode_length(count) 329 | size = self.long_len.size 330 | else: 331 | size = self.short_len.size 332 | self.size = size + count 333 | return struct.unpack_from(f'{count}s', buffer, offset + size) 334 | 335 | def unpack_read(self, readable: BinaryIO) -> tuple[bytes]: 336 | count_pos = readable.tell() 337 | count = self.short_len.unpack_read(readable)[0] 338 | if count >= 128: 339 | readable.seek(count_pos) 340 | count = self.long_len.unpack_read(readable)[0] 341 | count = self._decode_length(count) 342 | size = self.long_len.size 343 | else: 344 | size = self.short_len.size 345 | self.size = size + count 346 | return (readable.read(count),) 347 | 348 | 349 | Encoder = Callable[[str], bytes] 350 | Decoder = Callable[[bytes], str] 351 | # NOTE: not just Serializer[bytes], because we're implicitly using that these 352 | # serializers return tuple[bytes], not just Iterable[bytes] 353 | TCharSerializer = Union[ 354 | StructSerializer[bytes], 355 | DynamicCharSerializer, 356 | NETCharSerializer, 357 | TerminatedCharSerializer, 358 | ConsumingCharSerializer, 359 | ] 360 | 361 | 362 | class UnicodeSerializer(Serializer[str]): 363 | num_values: ClassVar[int] = 1 364 | 365 | def __init__( 366 | self, char_serializer: TCharSerializer, encoder: Encoder, decoder: Decoder 367 | ) -> None: 368 | self.serializer = char_serializer 369 | self.encoder = encoder 370 | self.decoder = decoder 371 | 372 | def get_final(self) -> Optional[Serializer]: 373 | return self.serializer.get_final() 374 | 375 | @property 376 | def size(self) -> int: 377 | return self.serializer.size 378 | 379 | def pack(self, *values: Unpack[tuple[str]]) -> bytes: 380 | return self.serializer.pack(self.encoder(values[0])) 381 | 382 | def pack_into( 383 | self, buffer: WritableBuffer, offset: int, *values: Unpack[tuple[str]] 384 | ) -> None: 385 | self.serializer.pack_into(buffer, offset, self.encoder(values[0])) 386 | 387 | def pack_write(self, writable: BinaryIO, *values: Unpack[tuple[str]]) -> None: 388 | self.serializer.pack_write(writable, self.encoder(values[0])) 389 | 390 | def unpack(self, buffer: ReadableBuffer) -> tuple[str]: 391 | return (self.decoder(self.serializer.unpack(buffer)[0]).rstrip('\0'),) 392 | 393 | def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> tuple[str]: 394 | return ( 395 | self.decoder(self.serializer.unpack_from(buffer, offset)[0]).rstrip('\0'), 396 | ) 397 | 398 | def unpack_read(self, readable: BinaryIO) -> tuple[str]: 399 | return (self.decoder(self.serializer.unpack_read(readable)[0]).rstrip('\0'),) 400 | -------------------------------------------------------------------------------- /structured/serializers/structs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Basic buliding block serializer for most other serializers. Just thin wrappers 3 | around struct.Struct. 4 | """ 5 | 6 | from __future__ import annotations 7 | 8 | __all__ = [ 9 | 'StructSerializer', 10 | 'StructActionSerializer', 11 | 'noop_action', 12 | ] 13 | 14 | import re 15 | import struct 16 | from functools import cached_property, partial, reduce 17 | from itertools import chain, repeat 18 | from typing import overload 19 | 20 | from ..base_types import ByteOrder 21 | from ..type_checking import ( 22 | TYPE_CHECKING, 23 | Any, 24 | BinaryIO, 25 | Callable, 26 | Generic, 27 | ReadableBuffer, 28 | Self, 29 | Ss, 30 | T, 31 | Ts, 32 | Unpack, 33 | ) 34 | from .api import NullSerializer, Serializer 35 | 36 | 37 | def noop_action(x: T) -> T: 38 | """A noop for StructActionSerializers where no additional wrapping is 39 | needed. 40 | """ 41 | return x 42 | 43 | 44 | def compute_num_values(st: struct.Struct, *, __cache: dict[str, int] = {}) -> int: 45 | """Determine how many values are used in packing/unpacking a struct format.""" 46 | try: 47 | return __cache[st.format] 48 | except KeyError: 49 | buffer = bytearray(st.size) 50 | # Use struct.Struct so this can be called before full initialization of 51 | # subclasses. 52 | count = len(struct.Struct.unpack_from(st, buffer)) 53 | __cache[st.format] = count 54 | return count 55 | 56 | 57 | _struct_chars = r'xcbB\?hHiIlLqQnNefdspP' 58 | _re_end: re.Pattern[str] = re.compile(rf'(.*?)(\d*)([{_struct_chars}])$') 59 | _re_start: re.Pattern[str] = re.compile(rf'^(\d*)([{_struct_chars}])(.*?)') 60 | 61 | 62 | def fold_overlaps(format1: str, format2: str, combine_strings: bool = False) -> str: 63 | """Combines two format strings into one, combining common types into counted 64 | versions, i.e.: 'h' + 'h' -> '2h'. The format strings must not contain 65 | byte order specifiers. 66 | 67 | :param format1: First format string to combine, may be empty. 68 | :param format2: Second format string to combine, may be empty. 69 | :return: The combined format string. 70 | """ 71 | start2 = _re_start.match(format2) 72 | end1 = _re_end.match(format1) 73 | if start2 and end1: 74 | prelude, count1, overlap1 = end1.groups() 75 | count2, overlap2, epilogue = start2.groups() 76 | if overlap1 == overlap2 and (combine_strings or overlap1 not in ('s', 'p')): 77 | count1 = int(count1) if count1 else 1 78 | count2 = int(count2) if count2 else 1 79 | return f'{prelude}{count1 + count2}{overlap1}{epilogue}' 80 | return format1 + format2 81 | 82 | 83 | def split_byte_order(format: str) -> tuple[ByteOrder, str]: 84 | if format: 85 | try: 86 | return ByteOrder(format[0]), format[1:] 87 | except ValueError: 88 | pass 89 | return ByteOrder.DEFAULT, format 90 | 91 | 92 | class StructSerializer(Generic[Unpack[Ts]], struct.Struct, Serializer[Unpack[Ts]]): 93 | """A Serializer that is a thin wrapper around struct.Struct, class creation 94 | is cached. 95 | """ 96 | 97 | @property 98 | def byte_order(self) -> ByteOrder: 99 | return self._split_format[0] 100 | 101 | @property 102 | def base_format(self) -> str: 103 | return self._split_format[1] 104 | 105 | @cached_property 106 | def _split_format(self) -> tuple[ByteOrder, str]: 107 | return split_byte_order(self.format) 108 | 109 | def __init__( 110 | self, 111 | format: str, 112 | byte_order: ByteOrder = ByteOrder.DEFAULT, 113 | ) -> None: 114 | super().__init__(byte_order.value + format) 115 | self.num_values = compute_num_values(self) 116 | 117 | def __str__(self) -> str: 118 | return f'{type(self).__name__}({self.format}, {self.num_values})' 119 | 120 | def with_byte_order(self, byte_order: ByteOrder) -> Self: 121 | old_byte_order, fmt = self._split_format 122 | if old_byte_order is byte_order: 123 | return self 124 | else: 125 | return type(self)(fmt, byte_order) 126 | 127 | def unpack(self, buffer: ReadableBuffer) -> tuple[Unpack[Ts]]: 128 | return super().unpack(buffer[: self.size]) # type: ignore 129 | 130 | if TYPE_CHECKING: 131 | 132 | def unpack_from( 133 | self, buffer: ReadableBuffer, offset: int = 0 134 | ) -> tuple[Unpack[Ts]]: ... 135 | 136 | def unpack_read(self, readable: BinaryIO) -> tuple[Unpack[Ts]]: 137 | # NOTE: use super-class's unpack to not interfere with custom 138 | # logic in subclasses 139 | return super().unpack(readable.read(self.size)) # type: ignore 140 | 141 | def pack_write(self, writable: BinaryIO, *values: Unpack[Ts]) -> None: 142 | # NOTE: Call the super-class's pack, so we don't interfere with 143 | # any custom logic in pack_write for subclasses 144 | writable.write(super().pack(*values)) 145 | 146 | @overload 147 | def __add__( 148 | self, other: StructSerializer[Unpack[Ss]] 149 | ) -> StructSerializer[Unpack[Ts], Unpack[Ss]]: ... 150 | 151 | @overload 152 | def __add__( 153 | self, other: Serializer[Unpack[Ss]] 154 | ) -> Serializer[Unpack[Ts], Unpack[Ss]]: ... 155 | 156 | def __add__(self, other: Serializer) -> Serializer: 157 | if isinstance(other, StructSerializer): 158 | # Don't need a CompoundSerializer for joining with another Struct 159 | byte_order, lfmt = self._split_format 160 | byte_order2, rfmt = other._split_format 161 | if byte_order is not byte_order2: 162 | raise ValueError( 163 | f'Cannot join StructSerializers with different byte orders: ' 164 | f'{self} + {other}' 165 | ) 166 | return type(self)( 167 | fold_overlaps(lfmt, rfmt), 168 | byte_order, 169 | ) 170 | return super().__add__(other) 171 | 172 | def __mul__(self, other: int) -> Self: # no tool to hint the [] yet 173 | """Return a new StructSerializer that unpacks `other` copies of the 174 | kine this one does. I.e: for non-string types it puts a multiplier 175 | number in front of the format specifier, for strings it repeats the 176 | format specifier. 177 | """ 178 | return self._mul_impl(other) 179 | 180 | def __matmul__(self, other: int) -> Self: 181 | """Return a new StructSerializer that folds strings together when 182 | multiplying. NOTE: This is only supported for StructSerializers that 183 | consist of either a 's', 'p', or 'x' format specifier. 184 | """ 185 | return self._mul_impl(other, True) 186 | 187 | def _mul_impl(self, other: int, combine_strings: bool = False) -> Self: 188 | if not isinstance(other, int): 189 | return NotImplemented 190 | elif other == 0: 191 | return NullSerializer() 192 | elif other < 0: 193 | raise ValueError('count must be non-negative') 194 | elif other == 1: 195 | return self 196 | byte_order, fmt = self._split_format 197 | fmt = reduce( 198 | partial(fold_overlaps, combine_strings=combine_strings), repeat(fmt, other) 199 | ) 200 | return type(self)(fmt, byte_order) 201 | 202 | def __eq__(self, other: StructSerializer) -> bool: 203 | if isinstance(other, StructSerializer): 204 | return self.format == other.format and self.num_values == other.num_values 205 | else: 206 | return NotImplemented 207 | 208 | def __hash__(self) -> int: 209 | return hash((self.format, self.num_values)) 210 | 211 | 212 | class StructActionSerializer(Generic[Unpack[Ts]], StructSerializer[Unpack[Ts]]): 213 | """A Serializer acting as a thin wrapper around struct.Struct, with 214 | transformations applied to unpacked values. 215 | """ 216 | 217 | actions: tuple[Callable[[Any], Any], ...] 218 | 219 | def __new__( 220 | cls, 221 | fmt: str, 222 | byte_order: ByteOrder = ByteOrder.DEFAULT, 223 | actions: tuple[Callable[[Any], Any], ...] = (), 224 | ) -> Self: 225 | return super().__new__(cls, fmt, byte_order) 226 | 227 | def __init__( 228 | self, 229 | fmt: str, 230 | byte_order: ByteOrder = ByteOrder.DEFAULT, 231 | actions: tuple[Callable[[Any], Any], ...] = (), 232 | ) -> None: 233 | super().__init__(fmt, byte_order) 234 | if len(actions) < self.num_values: 235 | actions = tuple( 236 | chain(actions, repeat(noop_action, self.num_values - len(actions))) 237 | ) 238 | self.actions = actions 239 | 240 | def unpack(self, buffer: ReadableBuffer) -> tuple[Unpack[Ts]]: 241 | return tuple( 242 | action(value) for action, value in zip(self.actions, super().unpack(buffer)) 243 | ) # type: ignore 244 | 245 | def unpack_from( 246 | self, buffer: ReadableBuffer, offset: int = ... 247 | ) -> tuple[Unpack[Ts]]: 248 | return tuple( 249 | action(value) 250 | for action, value in zip(self.actions, super().unpack_from(buffer, offset)) 251 | ) # type: ignore 252 | 253 | def unpack_read(self, readable: BinaryIO) -> tuple[Unpack[Ts]]: 254 | return tuple( 255 | action(value) 256 | for action, value in zip(self.actions, super().unpack_read(readable)) 257 | ) # type: ignore 258 | 259 | def with_byte_order(self, byte_order: ByteOrder) -> Self: 260 | res = super().with_byte_order(byte_order) 261 | if res is not self: 262 | res.actions = self.actions 263 | return res 264 | 265 | @overload 266 | def __add__( 267 | self, other: StructSerializer[Unpack[Ss]] 268 | ) -> StructActionSerializer[Unpack[Ts], Unpack[Ss]]: ... 269 | 270 | @overload 271 | def __add__( 272 | self, other: Serializer[Unpack[Ss]] 273 | ) -> Serializer[Unpack[Ts], Unpack[Ss]]: ... 274 | 275 | def __add__(self, other: Serializer) -> Serializer: 276 | if isinstance(other, StructActionSerializer): 277 | actions = other.actions 278 | elif isinstance(other, StructSerializer): 279 | actions = () 280 | else: 281 | return super().__add__(other) 282 | byte_order, lfmt = self._split_format 283 | _, rfmt = other._split_format 284 | fmt = fold_overlaps(lfmt, rfmt) 285 | actions = tuple(chain(self.actions, actions)) 286 | return type(self)(fmt, byte_order, actions) 287 | 288 | def __radd__( 289 | self, other: StructSerializer[Unpack[Ss]] 290 | ) -> StructActionSerializer[Unpack[Ss], Unpack[Ts]]: 291 | if isinstance(other, StructSerializer): 292 | actions = repeat(noop_action, other.num_values) 293 | else: 294 | return NotImplemented 295 | byte_order, lfmt = other._split_format 296 | _, rfmt = self._split_format 297 | fmt = fold_overlaps(lfmt, rfmt) 298 | actions = tuple(chain(actions, self.actions)) 299 | return type(self)(fmt, byte_order, actions) # type: ignore 300 | 301 | def __mul__(self, other: int) -> StructActionSerializer: # no way to hint this yet 302 | res = super().__mul__(other) 303 | res.actions = tuple(chain.from_iterable(repeat(self.actions, other))) 304 | return res 305 | 306 | def __eq__(self, other: StructSerializer) -> bool: 307 | if isinstance(other, StructActionSerializer): 308 | return self.format == other.format and self.actions == other.actions 309 | elif isinstance(other, StructSerializer): 310 | return False 311 | else: 312 | return NotImplemented 313 | 314 | def __hash__(self) -> int: 315 | return hash((self.format, self.num_values, self.actions)) 316 | -------------------------------------------------------------------------------- /structured/serializers/structured.py: -------------------------------------------------------------------------------- 1 | """ 2 | Serializer for packing/unpacking a Structured-derived object. 3 | """ 4 | 5 | __all__ = [ 6 | 'StructuredSerializer', 7 | ] 8 | 9 | from ..type_checking import ( 10 | TYPE_CHECKING, 11 | Any, 12 | BinaryIO, 13 | ClassVar, 14 | Generic, 15 | Optional, 16 | ReadableBuffer, 17 | TypeVar, 18 | WritableBuffer, 19 | annotated, 20 | get_args, 21 | get_origin, 22 | safe_issubclass, 23 | ) 24 | from ..utils import StructuredAlias 25 | from .api import Serializer 26 | 27 | if TYPE_CHECKING: 28 | # *Only* used for type-hinting, so ok to guard with a TYPE_CHECKING 29 | from ..structured import Structured 30 | 31 | TStructured = TypeVar('TStructured', bound=Structured) 32 | else: 33 | TStructured = TypeVar('TStructured') 34 | 35 | 36 | class StructuredSerializer(Generic[TStructured], Serializer[TStructured]): 37 | """Serializer which unpacks a Structured-derived instance.""" 38 | 39 | _specializations: ClassVar[dict] = {} 40 | 41 | num_values: ClassVar[int] = 1 42 | obj_type: type[TStructured] 43 | 44 | def __init__(self, obj_type: type[TStructured]) -> None: 45 | self.obj_type = obj_type 46 | self.size = 0 47 | 48 | def get_final(self) -> Optional[Serializer]: 49 | return self.obj_type.serializer.get_final() 50 | 51 | def pack(self, values: TStructured) -> bytes: 52 | data = values.pack() 53 | self.size = values.serializer.size 54 | return data 55 | 56 | def pack_into( 57 | self, buffer: WritableBuffer, offset: int, values: TStructured 58 | ) -> None: 59 | values.pack_into(buffer, offset) 60 | self.size = values.serializer.size 61 | 62 | def pack_write(self, writable: BinaryIO, values: TStructured) -> None: 63 | values.pack_write(writable) 64 | self.size = values.serializer.size 65 | 66 | def unpack(self, buffer: ReadableBuffer) -> tuple[TStructured]: 67 | value = self.obj_type.create_unpack(buffer) 68 | self.size = self.obj_type.serializer.size 69 | return (value,) 70 | 71 | def unpack_from( 72 | self, buffer: ReadableBuffer, offset: int = 0 73 | ) -> tuple[TStructured]: 74 | value = self.obj_type.create_unpack_from(buffer, offset) 75 | self.size = self.obj_type.serializer.size 76 | return (value,) 77 | 78 | def unpack_read(self, readable: BinaryIO) -> tuple[TStructured]: 79 | value = self.obj_type.create_unpack_read(readable) 80 | self.size = self.obj_type.serializer.size 81 | return (value,) 82 | 83 | @classmethod 84 | def _transform(cls, base_type: Any, hint: Any) -> Any: 85 | from ..structured import Structured 86 | 87 | if safe_issubclass(base_type, Structured): 88 | return StructuredSerializer(base_type) 89 | elif safe_issubclass((origin := get_origin(base_type)), Structured): 90 | spec_args = get_args(base_type) 91 | key = (origin, spec_args) 92 | if all(not isinstance(arg, TypeVar) for arg in spec_args): 93 | # Fully specialized, first try the cache 94 | try: 95 | return cls._specializations[key] 96 | except KeyError: 97 | pass 98 | 99 | class _Specialized(base_type): 100 | pass 101 | 102 | serializer = StructuredSerializer(_Specialized) 103 | cls._specializations[key] = StructuredSerializer(_Specialized) 104 | return serializer 105 | else: 106 | # Not fully specialized, return a StructuredAlias so it 107 | # can potentially be fully speciailized by a further 108 | # subclassing of the containing class. 109 | return StructuredAlias(base_type, spec_args) 110 | 111 | 112 | annotated.register_transform(StructuredSerializer._transform) 113 | -------------------------------------------------------------------------------- /structured/serializers/tuples.py: -------------------------------------------------------------------------------- 1 | """ 2 | TupleSerializer, a serializer very similar to a CompoundSerializer, but returns 3 | all of the contained values grouped into a tuple. 4 | """ 5 | 6 | __all__ = [ 7 | 'TupleSerializer', 8 | ] 9 | 10 | 11 | from ..type_checking import ( 12 | Any, 13 | BinaryIO, 14 | ClassVar, 15 | Generic, 16 | Iterable, 17 | Optional, 18 | ReadableBuffer, 19 | Ts, 20 | Unpack, 21 | WritableBuffer, 22 | ) 23 | from .api import NullSerializer, Serializer 24 | 25 | 26 | class TupleSerializer(Generic[Unpack[Ts]], Serializer[Unpack[Ts]]): 27 | num_values: ClassVar[int] = 1 28 | 29 | def __init__(self, serializers: Iterable[Serializer]) -> None: 30 | self.serializer = sum(serializers, NullSerializer()) 31 | 32 | def get_final(self) -> Optional[Serializer]: 33 | return self.serializer.get_final() 34 | 35 | @property 36 | def size(self) -> int: 37 | return self.serializer.size 38 | 39 | def prepack(self, partial_object: Any) -> Serializer: 40 | self._partial_obj = partial_object 41 | return self 42 | 43 | def pack(self, values: tuple[Unpack[Ts]]) -> bytes: 44 | return self.serializer.prepack(self._partial_obj).pack(*values) 45 | 46 | def pack_into( 47 | self, buffer: WritableBuffer, offset: int, values: tuple[Unpack[Ts]] 48 | ) -> None: 49 | self.serializer.prepack(self._partial_obj).pack_into(buffer, offset, *values) 50 | 51 | def pack_write(self, writable: BinaryIO, values: tuple[Unpack[Ts]]) -> None: 52 | self.serializer.prepack(self._partial_obj).pack_write(writable, *values) 53 | 54 | def preunpack(self, partial_object: Any) -> Serializer: 55 | self._partial_obj = partial_object 56 | return self 57 | 58 | def unpack(self, buffer: ReadableBuffer) -> tuple[Iterable[Any]]: 59 | return (self.serializer.preunpack(self._partial_obj).unpack(buffer),) 60 | 61 | def unpack_from(self, buffer: ReadableBuffer, offset: int) -> tuple[Iterable[Any]]: 62 | return ( 63 | self.serializer.preunpack(self._partial_obj).unpack_from(buffer, offset), 64 | ) 65 | 66 | def unpack_read(self, readable: BinaryIO) -> tuple[Iterable[Any]]: 67 | return (self.serializer.preunpack(self._partial_obj).unpack_read(readable),) 68 | -------------------------------------------------------------------------------- /structured/serializers/unions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Serializers for handling union type-hints. Must be supplied by hinting the 3 | union with typing.Annotated. 4 | """ 5 | 6 | from __future__ import annotations 7 | 8 | __all__ = [ 9 | 'AUnion', 10 | 'LookbackDecider', 11 | 'LookaheadDecider', 12 | ] 13 | 14 | import os 15 | 16 | from ..type_checking import ( 17 | Any, 18 | BinaryIO, 19 | Callable, 20 | ClassVar, 21 | Iterable, 22 | Optional, 23 | ReadableBuffer, 24 | WritableBuffer, 25 | annotated, 26 | get_union_args, 27 | isunion, 28 | ) 29 | from .api import Serializer 30 | 31 | 32 | class AUnion(Serializer): 33 | """Base class for union serializers, which are used to determine which 34 | serializer to use for a given value. 35 | """ 36 | 37 | num_values: ClassVar[int] = 1 38 | result_map: dict[Any, Serializer] 39 | default: Optional[Serializer] 40 | _last_serializer: Optional[Serializer] 41 | 42 | def __init__(self, result_map: dict[Any, Any], default: Any = None) -> None: 43 | """result_map should be a mapping of possible return values from `decider` 44 | to `Annotated` instances with a Serializer as an extra argument. The 45 | default should either be `None` to raise an error if the decider returns 46 | an unmapped value, or an `Annotated` instance with a Serializer as an 47 | extra argument. 48 | """ 49 | self.default = None if not default else self.validate_serializer(default) 50 | self.result_map = { 51 | key: self.validate_serializer(serializer) 52 | for key, serializer in result_map.items() 53 | } 54 | self.size = 0 55 | 56 | def get_final(self) -> Optional[Serializer]: 57 | for serializer in self.result_map.values(): 58 | if serializer.is_final(): 59 | return serializer.get_final() 60 | 61 | @staticmethod 62 | def validate_serializer(hint) -> Serializer: 63 | serializer = annotated.transform(hint) 64 | if not isinstance(serializer, Serializer): 65 | raise TypeError(f'Union results must be serializable types, got {hint!r}.') 66 | elif serializer.num_values != 1: 67 | raise ValueError('Union results must serializer a single item.') 68 | return serializer 69 | 70 | def prepack(self, partial_object) -> Serializer: 71 | self._partial_object = partial_object 72 | return self 73 | 74 | def preunpack(self, partial_object) -> Serializer: 75 | self._partial_object = partial_object 76 | return self 77 | 78 | def get_serializer(self, decider_result: Any, packing: bool) -> Serializer: 79 | """Given a target used to decide, return a serializer used to unpack.""" 80 | if self.default is None: 81 | try: 82 | serializer = self.result_map[decider_result] 83 | except KeyError: 84 | raise ValueError( 85 | f'Union decider returned an unmapped value {decider_result!r}' 86 | ) from None 87 | else: 88 | serializer = self.result_map.get(decider_result, self.default) 89 | if packing: 90 | return serializer.prepack(self._partial_object) 91 | else: 92 | return serializer.preunpack(self._partial_object) 93 | 94 | @staticmethod 95 | def _transform(base_type: Any, hint: Any) -> Any: 96 | if isinstance(hint, AUnion): 97 | if isunion(base_type) and (union_args := get_union_args(base_type)): 98 | union_args = tuple(map(annotated.transform, union_args)) 99 | if all(isinstance(x, Serializer) for x in union_args): 100 | return hint 101 | else: 102 | raise TypeError('Decider hinted on non-union type') 103 | 104 | 105 | annotated.register_transform(AUnion._transform) 106 | 107 | 108 | class LookbackDecider(AUnion): 109 | # NOTE: Union types are not allowed in TypeVarTuples, so we can't hint this 110 | """Serializer to handle loading of attributes with multiple types, type is 111 | decided just prior to packing/unpacking the attribute via inspection of the 112 | values already unpacked on the object. 113 | """ 114 | 115 | def __init__( 116 | self, 117 | decider: Callable[[Any], Any], 118 | result_map: dict[Any, Any], 119 | default: Any = None, 120 | ) -> None: 121 | """result_map should be a mapping of possible return values from `decider` 122 | to `Annotated` instances with a Serializer as an extra argument. The 123 | default should either be `None` to raise an error if the decider returns 124 | an unmapped value, or an `Annotated` instance with a Serializer as an 125 | extra argument. 126 | """ 127 | super().__init__(result_map, default) 128 | self.decider = decider 129 | 130 | def decide(self, packing: bool) -> Serializer: 131 | result = self.decider(self._partial_object) 132 | return self.get_serializer(result, packing) 133 | 134 | def pack(self, *values: Any) -> bytes: 135 | serializer = self.decide(True) 136 | data = serializer.pack(*values) 137 | self.size = serializer.size 138 | return data 139 | 140 | def pack_into(self, buffer: WritableBuffer, offset: int, *values: Any) -> None: 141 | serializer = self.decide(True) 142 | serializer.pack_into(buffer, offset, *values) 143 | self.size = serializer.size 144 | 145 | def pack_write(self, writable: BinaryIO, *values: Any) -> None: 146 | serializer = self.decide(True) 147 | serializer.pack_write(writable, *values) 148 | self.size = serializer.size 149 | 150 | def unpack(self, buffer: ReadableBuffer) -> Iterable: 151 | serializer = self.decide(False) 152 | value = serializer.unpack(buffer) 153 | self.size = serializer.size 154 | return value 155 | 156 | def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> Iterable: 157 | serializer = self.decide(False) 158 | value = serializer.unpack_from(buffer, offset) 159 | self.size = serializer.size 160 | return value 161 | 162 | def unpack_read(self, readable: BinaryIO) -> Iterable: 163 | serializer = self.decide(False) 164 | value = serializer.unpack_read(readable) 165 | self.size = serializer.size 166 | return value 167 | 168 | 169 | class LookaheadDecider(AUnion): 170 | """Union serializer that reads ahead into the input stream to determine how 171 | to unpack the next value. For packing, a write decider method is used to 172 | determine how to pack the next value.""" 173 | 174 | read_ahead_serializer: Serializer 175 | 176 | def __init__( 177 | self, 178 | read_ahead_serializer: Any, 179 | write_decider: Callable[[Any], Any], 180 | result_map: dict[Any, Any], 181 | default: Any = None, 182 | ) -> None: 183 | super().__init__(result_map, default) 184 | self.decider = write_decider 185 | serializer = annotated.transform(read_ahead_serializer) 186 | if not isinstance(serializer, Serializer): 187 | raise TypeError( 188 | 'read_ahead_serializer must be a Serializer, got ' 189 | f'{read_ahead_serializer!r}.' 190 | ) 191 | self.read_ahead_serializer = serializer 192 | 193 | def pack(self, *values: Any) -> bytes: 194 | result = self.decider(self._partial_object) 195 | serializer = self.get_serializer(result, True) 196 | data = serializer.pack(*values) 197 | self.size = serializer.size 198 | return data 199 | 200 | def pack_into(self, buffer: WritableBuffer, offset: int, *values: Any) -> None: 201 | result = self.decider(self._partial_object) 202 | serializer = self.get_serializer(result, True) 203 | serializer.pack_into(buffer, offset, *values) 204 | self.size = serializer.size 205 | 206 | def pack_write(self, writable: BinaryIO, *values: Any) -> None: 207 | result = self.decider(self._partial_object) 208 | serializer = self.get_serializer(result, True) 209 | serializer.pack_write(writable, *values) 210 | self.size = serializer.size 211 | 212 | def unpack(self, buffer: ReadableBuffer) -> Iterable: 213 | result = tuple(self.read_ahead_serializer.unpack(buffer))[0] 214 | serializer = self.get_serializer(result, False) 215 | values = serializer.unpack(buffer) 216 | self.size = serializer.size 217 | return values 218 | 219 | def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> Iterable: 220 | result = tuple(self.read_ahead_serializer.unpack_from(buffer, offset))[0] 221 | serializer = self.get_serializer(result, False) 222 | values = serializer.unpack_from(buffer, offset) 223 | self.size = serializer.size 224 | return values 225 | 226 | def unpack_read(self, readable: BinaryIO) -> Iterable: 227 | result = tuple(self.read_ahead_serializer.unpack_read(readable))[0] 228 | readable.seek(-self.read_ahead_serializer.size, os.SEEK_CUR) 229 | serializer = self.get_serializer(result, False) 230 | values = serializer.unpack_read(readable) 231 | self.size = serializer.size 232 | return values 233 | -------------------------------------------------------------------------------- /structured/structured.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provides the Structured class, which pulls all the type-hints together and 3 | drives the serialization process. 4 | """ 5 | 6 | from __future__ import annotations 7 | 8 | import sys 9 | 10 | __all__ = [ 11 | 'Structured', 12 | ] 13 | 14 | import operator 15 | from functools import reduce 16 | from itertools import count 17 | 18 | from .base_types import ByteOrder, ByteOrderMode 19 | from .serializers import NullSerializer, Serializer, StructSerializer 20 | from .type_checking import ( 21 | Any, 22 | BinaryIO, 23 | Callable, 24 | ClassVar, 25 | Generic, 26 | Iterable, 27 | Iterator, 28 | Optional, 29 | ReadableBuffer, 30 | Self, 31 | TypeVar, 32 | Union, 33 | WritableBuffer, 34 | annotated, 35 | get_annotations, 36 | get_args, 37 | get_origin, 38 | get_type_hints, 39 | get_union_args, 40 | isclassvar, 41 | update_annotations, 42 | ) 43 | from .utils import StructuredAlias, attrgetter, zips 44 | 45 | 46 | def ispad(annotation: Any) -> bool: 47 | """Detect pad[x] generated StructSerializers.""" 48 | serializer = annotated.transform(annotation) 49 | return ( 50 | isinstance(serializer, StructSerializer) 51 | and serializer.num_values == 0 52 | and serializer.format.endswith('x') 53 | ) 54 | 55 | 56 | def final_transform(base_type: Any, hint: Any) -> Optional[Serializer]: 57 | if isinstance(hint, Serializer): 58 | return hint 59 | 60 | 61 | annotated.register_final_transform(final_transform) 62 | 63 | 64 | def transform_typehint(hint: Any) -> Union[Serializer, None]: 65 | """Read in a typehint, and apply any transformations that need to be done 66 | to it. If the result is a hint Structured is concerned about, return it, 67 | otherwise returns None. 68 | 69 | The resulting types we're looking for are instances of Serializers 70 | """ 71 | if isclassvar(hint): 72 | return None 73 | hint = annotated.transform(hint) 74 | if isinstance(hint, Serializer): 75 | return hint 76 | 77 | 78 | def filter_typehints( 79 | typehints: dict[str, Any], 80 | cls: type[Structured], 81 | ) -> dict[str, Serializer]: 82 | """Filters a typehints dictionary of a class for only the types which 83 | Structured uses to generate serializers. 84 | 85 | :param typehints: A class's typehints dictionary. NOTE: This needs to be 86 | obtained via `get_type_hints(..., include_extras=True)`. 87 | :return: A filtered dictionary containing only attributes with types used 88 | by Structured. 89 | """ 90 | return { 91 | attr: transformed 92 | for attr, hint in typehints.items() 93 | if (transformed := transform_typehint(hint)) is not None 94 | } 95 | 96 | 97 | def get_structured_base(cls: type[Structured]) -> Optional[type[Structured]]: 98 | """Given a Structured derived class, find any base classes which are also 99 | Structured derived. If multiple are found, raise TypeError. 100 | 101 | :param cls: Structured derived class to analyze. 102 | :return: The direct base class which is Structured derived and not the 103 | Structured class itself, or None if no such base class exists. 104 | """ 105 | bases = tuple( 106 | base 107 | for base in cls.__bases__ 108 | if issubclass(base, Structured) and base is not Structured 109 | ) 110 | if len(bases) > 1: 111 | raise TypeError( 112 | 'Multiple inheritence from Structured base classes is not allowed.' 113 | ) 114 | elif bases: 115 | return bases[0] 116 | else: 117 | return None 118 | 119 | 120 | def gen_init( 121 | args: dict[str, Any], 122 | *, 123 | globalsns: Optional[dict[str, Any]] = None, 124 | localsns: Optional[dict[str, Any]] = None, 125 | ) -> Callable: 126 | """Generates an __init__ method for a class. `args` should be a mapping of 127 | arguments to type annotations to be used in the method definition. 128 | 129 | :param args: Mapping of argument names to argument type annotations, including self. 130 | :param globalsns: Any globals needed to be accessed by this method. 131 | :param localsns: Any locals needed to be accessed by this method. 132 | :return: The generated __init__, without __qualname__set. 133 | """ 134 | if localsns is None: 135 | localsns = {} 136 | local_vars = ', '.join(localsns.keys()) 137 | # Transform types to strings 138 | args_items = [] 139 | for name, annotation in args.items(): 140 | if union_args := get_union_args(annotation): 141 | union_text = ', '.join(arg.__name__ for arg in union_args) 142 | args_items.append(f'{name}: Union[{union_text}]') 143 | else: 144 | ann_name = getattr(annotation, '__name__', None) 145 | if not ann_name: 146 | # Python 3.9 typing.Tuple, etc have no __name__, instead they 147 | # have _name 148 | ann_name = getattr(annotation, '_name', None) 149 | if ann_name: 150 | args_items.append(f'{name}: {ann_name}') 151 | else: 152 | # Couldn't get the type-hint text 153 | args_items.append(name) 154 | # Inner function text 155 | args_txt = ', '.join(args_items) 156 | def_txt = f' def __init__({args_txt}) -> None:' 157 | body_lines = [f' self.{name} = {name}' for name in args.keys() if name != 'self'] 158 | body_lines.append(' self.__post_init__()') 159 | body_txt = '\n'.join(body_lines) 160 | inner_txt = f'{def_txt}\n{body_txt}' 161 | # Outer creation function 162 | txt = f'def __create_fn__({local_vars}):\n' 163 | txt += f'{inner_txt}\n' 164 | txt += ' return __init__' 165 | namespace = {} 166 | exec(txt, globalsns, namespace) 167 | return namespace['__create_fn__'](**localsns) 168 | 169 | 170 | class MetaDict(dict): 171 | """Dictionary which assigns unique names for variables named `_` and 172 | annotated with `pad`. 173 | """ 174 | 175 | _unique_id: ClassVar[Iterator[int]] = count() 176 | 177 | def __setitem__(self, key, value): 178 | if key == '_' and ispad(value): 179 | # Generate a unique name, we'll use ones that are invalid attribute 180 | # names so they won't accidentally overwrite anything a use sets. 181 | key = f'{next(self._unique_id)}_pad_' 182 | super().__setitem__(key, value) 183 | 184 | 185 | class StructuredMeta(type): 186 | """Metaclass that simply sets the annotations dict to one that automatically 187 | renames `_` variables annotated with a `pad` to unique names. 188 | """ 189 | 190 | def __prepare__(cls, bases, **kwargs): 191 | namespace = { 192 | '__annotations__': MetaDict(), 193 | } 194 | return namespace 195 | 196 | 197 | class Structured(metaclass=StructuredMeta): 198 | """Base class for classes which can be packed/unpacked using Python's 199 | struct module.""" 200 | 201 | __slots__ = () 202 | serializer: ClassVar[Serializer] = StructSerializer('') 203 | attrs: ClassVar[tuple[str, ...]] = () 204 | _attrgetter: ClassVar[Callable[[Structured], tuple[Any, ...]]] 205 | byte_order: ClassVar[ByteOrder] = ByteOrder.DEFAULT 206 | 207 | def __post_init__(self) -> None: 208 | """Initialize any instance variables not handled by the Structured 209 | unpacking logic. 210 | """ 211 | 212 | def with_byte_order(self, byte_order: ByteOrder) -> Self: 213 | if byte_order == self.byte_order: 214 | return self 215 | serializer = self.serializer.with_byte_order(byte_order) 216 | new_obj = type(self)(*type(self)._attrgetter(self)) 217 | new_obj.serializer = serializer 218 | new_obj.byte_order = byte_order 219 | return new_obj 220 | 221 | # General packers/unpackers 222 | def _serializer(self, packing: bool) -> Serializer: 223 | if packing: 224 | return self.serializer.prepack(self) 225 | else: 226 | return self.serializer.preunpack(self) 227 | 228 | def unpack(self, buffer: ReadableBuffer) -> None: 229 | """Unpack values from the bytes-like `buffer` and assign them to members 230 | 231 | :param buffer: A bytes-like object. 232 | """ 233 | for attr, value in zips( 234 | self.attrs, self._serializer(False).unpack(buffer), strict=True 235 | ): 236 | setattr(self, attr, value) 237 | 238 | def unpack_read(self, readable: BinaryIO) -> None: 239 | """Read data from a file-like object and unpack it into values, assigned 240 | to this class's attributes. 241 | 242 | :param readable: readable file-like object. 243 | """ 244 | for attr, value in zips( 245 | self.attrs, self._serializer(False).unpack_read(readable), strict=True 246 | ): 247 | setattr(self, attr, value) 248 | 249 | def unpack_from(self, buffer: ReadableBuffer, offset: int = 0) -> None: 250 | """Unpack values from a `buffer` implementing the buffer protocol 251 | starting at index `offset`, and assign them to their associated class 252 | members. 253 | 254 | :param buffer: buffer to unpack from. 255 | :param offset: position in the buffer to start from. 256 | """ 257 | for attr, value in zips( 258 | self.attrs, self._serializer(False).unpack_from(buffer, offset), strict=True 259 | ): 260 | setattr(self, attr, value) 261 | 262 | def pack(self) -> bytes: 263 | """Pack the class's values according to the format string.""" 264 | return self._serializer(True).pack(*type(self)._attrgetter(self)) 265 | 266 | def pack_write(self, writable: BinaryIO) -> None: 267 | """Pack the class's values according to the format string, then write 268 | the result to a file-like object. 269 | 270 | :param writable: writable file-like object. 271 | """ 272 | self._serializer(True).pack_write(writable, *type(self)._attrgetter(self)) 273 | 274 | def pack_into(self, buffer: WritableBuffer, offset: int = 0): 275 | """Pack the class's values according to the format string, pkacing the 276 | result into `buffer` starting at position `offset`. 277 | 278 | :param stream: buffer to pack into. 279 | :param offset: position in the buffer to start writing data to. 280 | """ 281 | self._serializer(True).pack_into(buffer, offset, *type(self)._attrgetter(self)) 282 | 283 | # Creation of objects from unpackable types 284 | @classmethod 285 | def _create_proxy(cls) -> tuple[_Proxy, Serializer]: 286 | """Create a proxy object for this class, which can be used to create 287 | new instances of this class. 288 | """ 289 | proxy = _Proxy(cls) 290 | return proxy, cls.serializer.preunpack(proxy) 291 | 292 | @classmethod 293 | def create_unpack(cls, buffer: ReadableBuffer) -> Self: 294 | """Create a new instance, initialized with values unpacked from a 295 | bytes-like buffer. 296 | 297 | :param buffer: A bytes-like object. 298 | :return: A new Structured object unpacked from the buffer. 299 | """ 300 | proxy, serializer = cls._create_proxy() 301 | proxy(serializer.unpack(buffer)) 302 | return cls(*proxy) 303 | 304 | @classmethod 305 | def create_unpack_from(cls, buffer: ReadableBuffer, offset: int = 0) -> Self: 306 | """Create a new instance, initialized with values unpacked from a buffer 307 | supporting the Buffer Protocol. 308 | 309 | :param buffer: An object supporting the Buffer Protocol. 310 | :param offset: Location in the buffer to begin unpacking. 311 | :return: A new Structured object unpacked from the buffer. 312 | """ 313 | proxy, serializer = cls._create_proxy() 314 | proxy(serializer.unpack_from(buffer, offset)) 315 | return cls(*proxy) 316 | 317 | @classmethod 318 | def create_unpack_read(cls, readable: BinaryIO) -> Self: 319 | """Create a new instance, initialized with values unpacked from a 320 | readable file-like object. 321 | 322 | :param readable: A readable file-like object. 323 | :return: A new Structured object unpacked from the readable object. 324 | """ 325 | proxy, serializer = cls._create_proxy() 326 | proxy(serializer.unpack_read(readable)) 327 | return cls(*proxy) 328 | 329 | def __str__(self) -> str: 330 | """Descriptive representation of this class.""" 331 | vals = ', '.join((f'{attr}={getattr(self, attr)}' for attr in self.attrs)) 332 | return f'{type(self).__name__}({vals})' 333 | 334 | def __repr__(self) -> str: 335 | return f'<{self}>' 336 | 337 | def __eq__(self, other) -> bool: 338 | if type(other) is type(self): 339 | return all( 340 | (getattr(self, attr) == getattr(other, attr) for attr in self.attrs) 341 | ) 342 | return NotImplemented 343 | 344 | def __init_subclass__( 345 | cls, 346 | byte_order: ByteOrder = ByteOrder.DEFAULT, 347 | byte_order_mode: ByteOrderMode = ByteOrderMode.STRICT, 348 | init: bool = True, 349 | **kwargs, 350 | ) -> None: 351 | """Subclassing a Structured type. We need to compute new values for the 352 | serializer and attrs. 353 | 354 | :param byte_order: Which byte order to use for struct packing/unpacking. 355 | Defaults to no byte order marker. 356 | :param byte_order_mode: Mode to use when resolving conflicts with super 357 | class's byte order. 358 | :param init: Whether to generate an __init__ method for this class (for 359 | example, set this to false if you wish to use @dataclass). 360 | :raises ValueError: If ByteOrder conflicts with the base class and is 361 | not specified as overridden. 362 | """ 363 | super().__init_subclass__(**kwargs) 364 | # Check for byte order conflicts 365 | if base := get_structured_base(cls): 366 | if ( 367 | byte_order_mode is ByteOrderMode.STRICT 368 | and base.byte_order is not byte_order 369 | ): 370 | raise ValueError( 371 | 'Incompatable byte order specifications between class ' 372 | f'{cls.__name__} ({byte_order.name}) and base class ' 373 | f'{base.__name__} ({base.byte_order.name}). ' 374 | 'If this is intentional, use `byte_order_mode=OVERRIDE`.' 375 | ) 376 | # Evaluta any generics in base class 377 | if base: 378 | orig_bases = getattr(cls, '__orig_bases__', ()) 379 | base_to_origbase = { 380 | origin: orig_base 381 | for orig_base in orig_bases 382 | if (origin := get_origin(orig_base)) and issubclass(origin, Structured) 383 | } 384 | orig_base = base_to_origbase.get(base, None) 385 | if orig_base: 386 | annotations = base._get_specialization_hints(*get_args(orig_base)) 387 | update_annotations(cls, annotations) 388 | # Analyze the class 389 | typehints = get_type_hints(cls, include_extras=True) 390 | applicable_typehints = filter_typehints(typehints, cls) 391 | # Which variables show up in the __init__ 392 | # Need to ensure 'self' shows up first 393 | typehints = get_type_hints(cls) 394 | init_vars = {'self': Self} 395 | init_vars |= { 396 | attr: typehints.get(attr, Any) 397 | for attr in applicable_typehints 398 | if not ispad(applicable_typehints[attr]) 399 | } 400 | # But also don't want 'self' to show up in attrs 401 | attrs = tuple(init_vars.keys())[1:] 402 | serializer = sum( 403 | applicable_typehints.values(), NullSerializer() 404 | ).with_byte_order(byte_order) 405 | if init: 406 | # Generate an init method 407 | if cls.__module__ in sys.modules: 408 | globals = sys.modules[cls.__module__].__dict__ 409 | else: 410 | globals = {} 411 | 412 | init_fn = gen_init(init_vars, globalsns=globals) 413 | init_fn.__qualname__ = f'{cls.__qualname__}.__init__' 414 | cls.__init__ = init_fn 415 | # And set the updated class attributes 416 | cls.serializer = serializer 417 | cls.attrs = attrs 418 | cls._attrgetter = attrgetter(*attrs) 419 | cls.byte_order = byte_order 420 | 421 | @classmethod 422 | def _get_specialization_hints( 423 | cls, 424 | *args, 425 | ) -> dict[str, Any]: 426 | """Get needed updates to __annotations__ and if this class were 427 | to be specialized with `args`, 428 | """ 429 | supers: dict[type[Structured], Any] = {} 430 | tvars = () 431 | for base in getattr(cls, '__orig_bases__', ()): 432 | if (origin := get_origin(base)) is Generic: 433 | tvars = get_args(base) 434 | elif origin and issubclass(origin, Structured): 435 | supers[origin] = base 436 | tvar_map = dict(zip(tvars, args)) 437 | if not tvar_map: 438 | raise TypeError(f'{cls.__name__} is not a Generic') 439 | # First handle the direct base class 440 | annotations = {} 441 | 442 | def alias_tvar_check(base_type: Any, hint: Any): 443 | if isinstance(base_type, (StructuredAlias, TypeVar)): 444 | return base_type 445 | if isinstance(hint, (StructuredAlias, TypeVar)): 446 | return hint 447 | 448 | cls_annotations = get_annotations(cls) 449 | for attr, attr_type in get_type_hints(cls, include_extras=True).items(): 450 | if attr in cls_annotations: 451 | unwrapped = annotated.with_final(alias_tvar_check).transform(attr_type) 452 | if isinstance(unwrapped, TypeVar): 453 | if remapped_type := tvar_map.get(unwrapped, None): 454 | annotations[attr] = remapped_type 455 | elif isinstance(unwrapped, StructuredAlias): 456 | annotations[attr] = unwrapped.resolve(tvar_map) 457 | # Now any classes higher in the chain 458 | all_annotations = [annotations] 459 | for base, alias in supers.items(): 460 | args = get_args(alias) 461 | args = (tvar_map.get(arg, arg) for arg in args) 462 | super_annotations = base._get_specialization_hints(*args) 463 | all_annotations.append(super_annotations) 464 | return reduce(operator.or_, reversed(all_annotations)) 465 | 466 | 467 | class _Proxy: 468 | """Proxy object for a Structured instance, used as a placeholder for the 469 | create_unpack_*** methods to recieve values, and still allow Union deciders 470 | to work. 471 | """ 472 | 473 | # NOTE: Only using __dunder__ methods, so any attributes on the class this 474 | # is a proxy for won't be shadowed. 475 | def __init__(self, cls: type[Structured]) -> None: 476 | self.__attrs = cls.attrs 477 | self.cls = cls 478 | 479 | def __call__(self, values: Iterable[Any]) -> None: 480 | for attr, value in zips(self.__attrs, values, strict=True): 481 | setattr(self, attr, value) 482 | 483 | def __iter__(self): 484 | return (getattr(self, attr) for attr in self.__attrs) 485 | -------------------------------------------------------------------------------- /structured/type_checking.py: -------------------------------------------------------------------------------- 1 | # pragma: no cover 2 | """ 3 | Central location for importing typing memebers, with fallbacks for older Python 4 | versions pulling from typing_extensions. Also provides a few helper methods 5 | to simplify some common patterns, as well as a method for extracting desired 6 | hints from Annotated types. 7 | """ 8 | from __future__ import annotations 9 | 10 | import sys 11 | import typing 12 | from itertools import chain 13 | from typing import ( 14 | TYPE_CHECKING, 15 | Annotated, 16 | Any, 17 | BinaryIO, 18 | Callable, 19 | ClassVar, 20 | Container, 21 | Generic, 22 | Iterable, 23 | Iterator, 24 | Literal, 25 | NewType, 26 | NoReturn, 27 | Optional, 28 | Tuple, 29 | Type, 30 | TypeVar, 31 | Union, 32 | cast, 33 | get_args, 34 | get_origin, 35 | get_type_hints, 36 | overload, 37 | ) 38 | 39 | if sys.version_info < (3, 10): 40 | from typing_extensions import ParamSpec, TypeAlias, TypeGuard 41 | 42 | UnionType = NewType('UnionType', object) # needed for TypeGuard on 3.9 43 | union_types = (Union,) 44 | else: 45 | from types import UnionType 46 | from typing import ParamSpec, TypeAlias, TypeGuard 47 | 48 | union_types = (Union, UnionType) 49 | 50 | 51 | if sys.version_info < (3, 11): 52 | from typing_extensions import Self, TypeVarTuple, Unpack, dataclass_transform 53 | else: 54 | from typing import Self, TypeVarTuple, Unpack, dataclass_transform 55 | 56 | 57 | S = TypeVar('S') 58 | T = TypeVar('T') 59 | U = TypeVar('U') 60 | V = TypeVar('V') 61 | W = TypeVar('W') 62 | Ts = TypeVarTuple('Ts') 63 | Ss = TypeVarTuple('Ss') 64 | P = ParamSpec('P') 65 | 66 | 67 | def update_annotations(cls: type, annotations: dict[str, Any]) -> None: 68 | """Python <3.10 compatible way to update a class's annotations dict. See: 69 | 70 | https://docs.python.org/3/howto/annotations.html#accessing-the-annotations-dict-of-an-object-in-python-3-9-and-older 71 | """ 72 | if '__annotations__' in cls.__dict__: 73 | cls.__annotations__.update(annotations) 74 | else: 75 | setattr(cls, '__annotations__', annotations) 76 | 77 | 78 | def get_annotations(cls: type) -> dict[str, Any]: 79 | """Python <3.10 compatible way to get a class's annotations dict. See: 80 | 81 | https://docs.python.org/3/howto/annotations.html#accessing-the-annotations-dict-of-an-object-in-python-3-9-and-older 82 | """ 83 | return cls.__dict__.get('__annotations__', {}) 84 | 85 | 86 | def isclassvar(annotation: Any) -> bool: 87 | """Determine if a type annotations is for a class variable. 88 | 89 | :param annotation: Fully resolved type annotation to test. 90 | """ 91 | return get_origin(annotation) is ClassVar 92 | 93 | 94 | def isunion(annotation: Any) -> TypeGuard[UnionType]: 95 | """Determine if a type annotation is a union. 96 | 97 | :param annotation: Fully resolved type annotation to test. 98 | """ 99 | return get_origin(annotation) in union_types 100 | 101 | 102 | def get_union_args(annotation: Any) -> tuple[Any, ...]: 103 | """Get the arguments of a union type annotation, or an empty tuple if the 104 | annotation is not a union. 105 | 106 | :param annotation: Fully resolved type annotation to test. 107 | """ 108 | if isunion(annotation): 109 | return get_args(annotation) 110 | else: 111 | return () 112 | 113 | 114 | def istuple(annotation: Any) -> TypeGuard[tuple]: 115 | return get_origin(annotation) in (tuple, Tuple) 116 | 117 | 118 | def get_tuple_args( 119 | annotation: Any, fixed_size: bool = True 120 | ) -> Optional[tuple[Any, ...]]: 121 | """Get the arguments to a tuple type hint, or None if the annotation is not 122 | a tuple hint. If `fixed_size` is True (default), then the tuple must be 123 | a fixed length tuple hint. 124 | """ 125 | if get_origin(annotation) in (tuple, Tuple): 126 | args = get_args(annotation) 127 | if fixed_size and args and args[-1] is Ellipsis: 128 | return None 129 | return args 130 | return None 131 | 132 | 133 | class _annotated(Generic[Unpack[Ts]]): 134 | _transforms: ClassVar[list[Callable]] = [] 135 | 136 | def __init__(self, *transforms: Callable) -> None: 137 | if transforms: 138 | self._transforms = type(self)._transforms[:] 139 | self._transforms.extend(transforms) 140 | 141 | @classmethod 142 | def register_transform( 143 | cls, transformer: Callable[[Any, Any], Union[Unpack[Ts]]] 144 | ) -> None: 145 | cls._transforms.append(transformer) 146 | 147 | @staticmethod 148 | def flatten_Annotated(hint: Any) -> tuple[Any, ...]: 149 | def _iter(h, *, start=0): 150 | if get_origin(h) is Annotated: 151 | for sub_h in get_args(h)[start:]: 152 | yield from _iter(sub_h, start=1) 153 | else: 154 | yield h 155 | 156 | return tuple(_iter(hint)) 157 | 158 | def transform(self, typehint: Any): 159 | base_type, *annotations = self.flatten_Annotated(typehint) 160 | annotations = (None,) + tuple(annotations) 161 | for annotation in annotations: 162 | for transform in reversed(self._transforms): 163 | new_type = transform(base_type, annotation) 164 | if new_type is not None: 165 | base_type = new_type 166 | return base_type 167 | 168 | @classmethod 169 | def register_final_transform(cls, transform: Callable[[Any, Any], Any]): 170 | cls._transforms.insert(0, transform) 171 | 172 | def with_final(self, check: Callable[[Any, Any], Any]) -> Self: 173 | return type(self)(check) 174 | 175 | 176 | annotated = _annotated() 177 | 178 | 179 | @overload 180 | def safe_issubclass(a, cls: type[T]) -> TypeGuard[type[T]]: ... 181 | 182 | 183 | @overload 184 | def safe_issubclass( 185 | a, cls: tuple[Unpack[Ts]] 186 | ) -> TypeGuard[type[Union[Unpack[Ts]]]]: ... 187 | 188 | 189 | def safe_issubclass(a, cls): # type: ignore 190 | """issubclass check without having to check if isinstance(a, type) first.""" 191 | try: 192 | return issubclass(a, cls) 193 | except TypeError: 194 | return False 195 | 196 | 197 | if typing.TYPE_CHECKING: 198 | import array 199 | import ctypes 200 | import io 201 | import mmap 202 | import pickle 203 | import sys 204 | 205 | ReadOnlyBuffer: TypeAlias = bytes 206 | # Anything that implements the read-write buffer interface. The buffer 207 | # interface is defined purely on the C level, so we cannot define a normal 208 | # Protocol for it (until PEP 688 is implemented). Instead we have to list 209 | # the most common stdlib buffer classes in a Union. 210 | if sys.version_info >= (3, 8): 211 | WritableBuffer: TypeAlias = Union[ 212 | bytearray, 213 | memoryview, 214 | array.array[Any], 215 | mmap.mmap, 216 | # ctypes._CData, pickle.PickleBuffer 217 | ] 218 | else: 219 | WritableBuffer: TypeAlias = Union[ # type: ignore 220 | bytearray, 221 | memoryview, 222 | array.array[Any], 223 | mmap.mmap, 224 | # ctypes._CData 225 | ] 226 | ReadableBuffer: TypeAlias = Union[ReadOnlyBuffer, WritableBuffer] 227 | else: 228 | WritableBuffer: TypeAlias = bytearray 229 | ReadableBuffer: TypeAlias = Union[bytes, bytearray] 230 | -------------------------------------------------------------------------------- /structured/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Various utility methods. 3 | """ 4 | 5 | import operator 6 | import sys 7 | import warnings 8 | from functools import wraps 9 | 10 | from .type_checking import Any, Callable, Optional, ParamSpec, T, TypeVar 11 | 12 | if sys.version_info < (3, 10): 13 | from typing import overload 14 | 15 | from .type_checking import Iterable, S 16 | 17 | @overload 18 | def zips(iterable1: Iterable[S], *, strict: bool = ...) -> Iterable[tuple[S]]: ... 19 | 20 | @overload 21 | def zips( # noqa: F811 22 | iterable1: Iterable[S], iterable2: Iterable[T], *, strict: bool = ... 23 | ) -> Iterable[tuple[S, T]]: ... 24 | 25 | @overload 26 | def zips( # noqa: F811 27 | iterable1: Iterable[Any], 28 | iterable2: Iterable[Any], 29 | *iterables: Iterable[Any], 30 | strict: bool = ..., 31 | ) -> Iterable[tuple[Any, ...]]: ... 32 | 33 | def zips(*iterables: Iterable, strict: bool = False): # noqa: F811 34 | """Python 3.9 compatible way of emulating zip(..., strict=True)""" 35 | if not strict: 36 | yield from zip(*iterables) 37 | else: 38 | iterators = [iter(it) for it in iterables] 39 | yield from zip(*iterators) 40 | # Check all consumed: 41 | for it in iterators: 42 | try: 43 | next(it) 44 | except StopIteration: 45 | pass 46 | else: 47 | raise ValueError('iterables must be of equal length') 48 | 49 | else: 50 | zips = zip 51 | 52 | 53 | def attrgetter(*attr_names: str) -> Callable[[Any], tuple[Any, ...]]: 54 | """Create an operator.attrgetter-like callable. The differences are if no 55 | attributes are specified, the callable returns and empty tuple, and a single 56 | attribute supplied still returns the attribute in a tuple. 57 | """ 58 | if not attr_names: 59 | return lambda x: () 60 | _get = operator.attrgetter(*attr_names) 61 | if len(attr_names) == 1: 62 | return lambda x: (_get(x),) 63 | else: 64 | return _get 65 | 66 | 67 | class StructuredAlias: 68 | """Class to hold one of the structured types that takes types as arguments, 69 | which has been passes either another StructuredAlias or a TypeVar. 70 | """ 71 | 72 | cls: type 73 | args: tuple 74 | 75 | def __init__(self, cls: type, args: tuple[Any, ...]) -> None: 76 | """Wrap a generic class along with whatever generic arguments it was 77 | created with. 78 | """ 79 | self.cls = cls 80 | self.args = args 81 | 82 | def resolve(self, tvar_map: dict[TypeVar, Any]): 83 | """Attempt to resolve the specific generic specialization given a map 84 | of TypeVars to concrete types. If any TypeVars remain, return a new 85 | StructuredAlias that can be further resolved. 86 | 87 | :param tvar_map: A map of TypeVars to concrete types. 88 | :return: The fully specialized class, or a new StructuredAlias 89 | """ 90 | resolved = [] 91 | for arg in self.args: 92 | arg = tvar_map.get(arg, arg) 93 | if isinstance(arg, StructuredAlias): 94 | arg = arg.resolve(tvar_map) 95 | resolved.append(arg) 96 | resolved = tuple(resolved) 97 | if any((isinstance(arg, (TypeVar, StructuredAlias)) for arg in resolved)): 98 | # Act as immutable, so create a new instance, since these objects 99 | # are often cached in type factory indexing methods. 100 | return StructuredAlias(self.cls, resolved) 101 | else: 102 | return self.cls[resolved] # type: ignore 103 | 104 | 105 | # nice deprecation warnings, ideas taken from Trio 106 | class StructuredDeprecationWarning(FutureWarning): 107 | """Warning emitted if you use deprecated Structured functionality. This 108 | feature will be removed in a future version. Despite the name, this class 109 | currently inherits from :class:`FutureWarning`, not 110 | :class:`DeprecationWarning`, because we want these warning to be visible by 111 | default. You can hide them by installing a filter or with the ``-W`` 112 | switch. 113 | """ 114 | 115 | 116 | def _stringify(x: Any) -> str: 117 | """Attempt to make a nice string representation of `x` if possible. 118 | 119 | :param x: Object to stringize, usually a method or class. 120 | :return: The best human readable string representation of `x` that this 121 | method can achieve. 122 | """ 123 | try: 124 | return f'{x.__module__}.{x.__qualname__}' 125 | except AttributeError: 126 | return str(x) 127 | 128 | 129 | def _issue_url(issue: int) -> str: 130 | """Generate a uri to the repository issues for a specific issue number. 131 | 132 | :param issue: Issue number to link to. 133 | :return: The uri to the issue. 134 | """ 135 | return ( 136 | f'https://github.com/lojack5/structured/issuespython-trio/trio/issues/{issue}' 137 | ) 138 | 139 | 140 | def warn_deprecated( 141 | x: Any, 142 | version: str, 143 | removal: str, 144 | *, 145 | issue: Optional[int], 146 | use_instead: Any, 147 | stacklevel: int = 2, 148 | ) -> None: 149 | """Emit a deprecation warning for using object `x`. 150 | 151 | :param x: The object that is being used. 152 | :param version: Version this object was deprecated. 153 | :param removal: Version this object will be removed completely. 154 | :param issue: GitHub issue number mentioning this. 155 | :param use_instead: Alternative to use instead of `x`. 156 | :param stacklevel: Stack-frame to have this warning show up in. 157 | """ 158 | stacklevel += 1 159 | msg = ( 160 | f'{_stringify(x)} is deprecated since structured-classes {version} ' 161 | f'and will be removed in structured-classes {removal}' 162 | ) 163 | if use_instead is None: 164 | msg += ' with no replacement' 165 | else: 166 | msg += f'; use {_stringify(use_instead)} instead' 167 | if issue is not None: 168 | msg += f' ({_issue_url(issue)})' 169 | warnings.warn(StructuredDeprecationWarning(msg), stacklevel=stacklevel) 170 | 171 | 172 | P = ParamSpec('P') 173 | T = TypeVar('T') 174 | 175 | 176 | def deprecated( 177 | version: str, 178 | removal: str, 179 | *, 180 | x: Any = None, 181 | issue: int, 182 | use_instead: Any, 183 | ) -> Callable[[Callable[P, T]], Callable[P, T]]: 184 | """Decorate a callable as deprecated. 185 | Usage: 186 | @deprecated(version, removal [, issue=..., use_instead=...]) 187 | def deprecated_method(...): 188 | ... 189 | 190 | :param version: Version the callable was deprecated. 191 | :param removal: Version the callable will be removed completely. 192 | :param issue: GitHub issue number mentioning this. 193 | :param use_instead: Alternative to use instead of the callable. 194 | :param x: Callable to mark as deprecated. 195 | :return: The wrapped callable which emits deprecation warning. 196 | """ 197 | 198 | def inner(fn: Callable[P, T]) -> Callable[P, T]: 199 | nonlocal x 200 | 201 | @wraps(fn) 202 | def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: 203 | warn_deprecated(x, version, removal, use_instead=use_instead, issue=issue) 204 | return fn(*args, **kwargs) 205 | 206 | # If our __module__ or __qualname__ get modified, we want to pick up 207 | # on that, so we read them off the wrapper object instead of the (now 208 | # hidden) fn object 209 | if x is None: 210 | x = wrapper 211 | 212 | if wrapper.__doc__ is not None: 213 | doc = wrapper.__doc__ 214 | doc = doc.rstrip() + f'\n\n .. deprecated:: {version}\n' 215 | if use_instead is not None: 216 | doc += f' Use {_stringify(use_instead)} instead.\n' 217 | if issue is not None: 218 | doc += f' For details, see `issue #{issue} ' 219 | doc += f'<{_issue_url(issue)}>`__.\n' 220 | doc += '\n' 221 | wrapper.__doc__ = doc 222 | 223 | return wrapper 224 | 225 | return inner 226 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import io 2 | 3 | from structured import Structured, Serializer 4 | 5 | 6 | class Final(Serializer): 7 | num_values = 1 8 | size = 0 9 | 10 | def get_final(self): 11 | return self 12 | 13 | 14 | def standard_tests(target_obj: Structured, target_data: bytes): 15 | target_size = len(target_data) 16 | assert target_obj.pack() == target_data 17 | assert type(target_obj).create_unpack(target_data) == target_obj 18 | assert target_obj.serializer.size == target_size, f'{target_obj.serializer.size} != {target_size}' 19 | 20 | buffer = bytearray(len(target_data)) 21 | target_obj.pack_into(buffer) 22 | assert bytes(buffer) == target_data 23 | assert target_obj.serializer.size == target_size 24 | assert type(target_obj).create_unpack_from(buffer) == target_obj 25 | assert target_obj.serializer.size == target_size 26 | 27 | with io.BytesIO() as stream: 28 | target_obj.pack_write(stream) 29 | assert target_obj.serializer.size == target_size 30 | assert stream.getvalue() == target_data 31 | stream.seek(0) 32 | assert type(target_obj).create_unpack_read(stream) == target_obj 33 | assert target_obj.serializer.size == target_size 34 | -------------------------------------------------------------------------------- /tests/test_annotations.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Annotated, ClassVar, get_origin 4 | 5 | from structured import * 6 | from structured.base_types import requires_indexing 7 | 8 | 9 | class A: pass 10 | 11 | 12 | def test_eval_annotation() -> None: 13 | # Prior versions of the code would fail to evaluate `ClassVar[A]` 14 | class Base(Structured): 15 | a: ClassVar[A] 16 | b: int8 17 | 18 | 19 | def test_for_annotated() -> None: 20 | """Ensure all usable types are an Annotated, with a few exceptions. 21 | 22 | Current exceptions are unindexed pad, char, and unicode, but those are an 23 | error to not index. 24 | """ 25 | # Basic types 26 | for kind in (bool8, int8, uint8, int16, uint16, int32, uint32, int64, uint64, float16, float32, float64): 27 | assert get_origin(kind) is Annotated 28 | # Simple type: pad 29 | assert get_origin(pad) is not Annotated 30 | assert get_origin(pad[3]) is Annotated 31 | # Complex types: array 32 | assert get_origin(array) is not Annotated 33 | assert get_origin(array[Header[1], int8]) is Annotated 34 | # Complex types: char 35 | assert get_origin(char) is not Annotated 36 | assert get_origin(char[10]) is Annotated 37 | assert get_origin(char[uint32]) is Annotated 38 | # Complex types: unicode 39 | assert get_origin(unicode) is not Annotated 40 | assert issubclass(unicode, requires_indexing) 41 | assert get_origin(unicode[10]) is Annotated 42 | assert get_origin(unicode[uint32]) is Annotated 43 | 44 | 45 | def test_alternate_syntax() -> None: 46 | # Test using only Annotated 47 | class Base(Structured): 48 | a: Annotated[int, int8] 49 | b: Annotated[bytes, char[10]] 50 | c: Annotated[str, unicode[15]] 51 | d: Annotated[list[int8], array[Header[2], int8]] 52 | e: Annotated[None, pad[3]] 53 | assert Base.attrs == ('a', 'b', 'c', 'd', ) 54 | -------------------------------------------------------------------------------- /tests/test_arrays.py: -------------------------------------------------------------------------------- 1 | import struct 2 | import io 3 | from typing import Annotated 4 | 5 | import pytest 6 | 7 | from structured import * 8 | 9 | from . import standard_tests, Final 10 | 11 | 12 | class Item(Structured): 13 | a: int32 14 | b: uint8 15 | c: char[uint8] 16 | 17 | 18 | @pytest.fixture 19 | def items() -> list[Item]: 20 | return [ 21 | Item(1, 11, b'foo'), 22 | Item(2, 22, b'bar'), 23 | Item(3, 33, b'Hello'), 24 | ] 25 | 26 | 27 | def test_errors(): 28 | ## Number of args 29 | with pytest.raises(TypeError): 30 | Header[1, 2, 3] # too many 31 | with pytest.raises(TypeError): 32 | array[Header[1]] # type: ignore (not enough) 33 | with pytest.raises(TypeError): 34 | array[Header[1], 2, 3, 4] # type: ignore (too many) 35 | ## Header type 36 | with pytest.raises(TypeError): 37 | array[Header, int32] 38 | ## Array type 39 | with pytest.raises(TypeError): 40 | array[Header[1], 2] # type: ignore 41 | with pytest.raises(TypeError): 42 | array[Header[1], int] # type: ignore 43 | ## Size check type 44 | with pytest.raises(TypeError): 45 | Header[1, 2] 46 | with pytest.raises(TypeError): 47 | Header[1, int] 48 | ## Array size 49 | with pytest.raises(ValueError): 50 | Header[-1] # invalid size 51 | with pytest.raises(ValueError): 52 | Header[-1, uint32] # invalid size 53 | with pytest.raises(TypeError): 54 | Header[int] # invalid type 55 | with pytest.raises(TypeError): 56 | Header[int8] # invalid type 57 | 58 | 59 | def test_static_format(): 60 | class Static(Structured): 61 | a: int32 62 | b: Annotated[list[uint32], array[Header[5], uint32]] 63 | target_obj = Static(42, [1, 2, 3, 4, 5]) 64 | 65 | st = struct.Struct('i5I') 66 | target_data = st.pack(42, 1, 2, 3, 4, 5) 67 | 68 | standard_tests(target_obj, target_data) 69 | 70 | # Incorrect array size 71 | target_obj.b = [] 72 | with pytest.raises(ValueError): 73 | target_obj.pack() 74 | 75 | 76 | def test_static_format_action(): 77 | class WrappedInt: 78 | def __init__(self, wrapped: int): 79 | self._wrapped = wrapped 80 | 81 | def __index__(self) -> int: 82 | return self._wrapped 83 | 84 | def __eq__(self, other): 85 | if isinstance(other, type(self)): 86 | return self._wrapped == other._wrapped 87 | return NotImplemented 88 | WrappedInt8 = Annotated[WrappedInt, SerializeAs(int8)] 89 | 90 | class StaticAction(Structured): 91 | a: array[Header[3], WrappedInt8] 92 | 93 | target_obj = StaticAction(list(map(WrappedInt, (1 ,2, 3)))) 94 | target_data = struct.pack('3b', 1, 2, 3) 95 | 96 | standard_tests(target_obj, target_data) 97 | 98 | 99 | def test_static_structured(items: list[Item]): 100 | class Compound(Structured): 101 | a: Annotated[list[Item], array[Header[3], Item]] 102 | 103 | target_obj = Compound(items) 104 | with io.BytesIO() as out: 105 | for item in target_obj.a: 106 | # Using the fact that Structured.pack is tested already on basic 107 | # types to ensure this data is correct 108 | item.pack_write(out) 109 | target_data = out.getvalue() 110 | 111 | standard_tests(target_obj, target_data) 112 | 113 | # incorrect array size 114 | target_obj.a = [] 115 | with pytest.raises(ValueError): 116 | target_obj.pack() 117 | 118 | 119 | def test_static_checked_structured(items: list[Item]): 120 | class Compound(Structured): 121 | a: Annotated[list[Item], array[Header[3, uint32], Item]] 122 | target_obj = Compound(items) 123 | 124 | with io.BytesIO() as stream: 125 | stream.write(struct.pack('I', 0)) 126 | data_size = 0 127 | for item in target_obj.a: 128 | item.pack_write(stream) 129 | data_size += item.serializer.size 130 | stream.seek(0) 131 | stream.write(struct.pack('I', data_size)) 132 | target_data = stream.getvalue() 133 | 134 | standard_tests(target_obj, target_data) 135 | 136 | # Incorrect array size 137 | target_obj.a = [] 138 | with pytest.raises(ValueError): 139 | target_obj.pack() 140 | 141 | # Test malformed data_size 142 | buffer = bytearray(target_data) 143 | struct.pack_into('I', buffer, 0, 0) 144 | with pytest.raises(ValueError): 145 | Compound.create_unpack_from(buffer) 146 | 147 | 148 | def test_dynamic_format(): 149 | class Compound(Structured): 150 | a: array[Header[uint32], int8] 151 | assert isinstance(Compound.serializer, DynamicStructArraySerializer) 152 | target_obj = Compound([1, 2, 3]) 153 | target_data = struct.pack('I3b', 3, 1, 2, 3) 154 | 155 | standard_tests(target_obj, target_data) 156 | 157 | 158 | def test_zero_length(): 159 | class EmptyList(Structured): 160 | a: array[Header[uint32], uint8] 161 | 162 | target_obj = EmptyList([]) 163 | target_data = struct.pack('I', 0) 164 | 165 | standard_tests(target_obj, target_data) 166 | 167 | 168 | def test_dynamic_structured(items: list[Item]): 169 | class Compound(Structured): 170 | a: array[Header[uint32], Item] 171 | target_obj = Compound(items) 172 | 173 | with io.BytesIO() as out: 174 | # Item uses a plain struct serializer, already tested 175 | # So no need to construct the data fully from struct.pack 176 | out.write(struct.pack('I', 3)) 177 | for item in target_obj.a: 178 | item.pack_write(out) 179 | target_data = out.getvalue() 180 | 181 | standard_tests(target_obj, target_data) 182 | 183 | 184 | def test_dynamic_checked_structured(items: list[Item]): 185 | class Compound(Structured): 186 | b: uint32 187 | a: array[Header[uint32, uint32], Item] 188 | assert isinstance(Compound.serializer, CompoundSerializer) 189 | array_serializer = Compound.serializer.serializers[1] 190 | assert isinstance(array_serializer, ArraySerializer) 191 | assert isinstance(array_serializer.header_serializer, StructSerializer) 192 | assert array_serializer.header_serializer.format == '2I' 193 | assert array_serializer.header_serializer.num_values == 2 194 | assert array_serializer.static_length == -1 195 | 196 | target_obj = Compound(42, items) 197 | 198 | with io.BytesIO() as out: 199 | out.write(struct.pack('3I', 42, 3, 0)) 200 | data_size = 0 201 | for item in target_obj.a: 202 | item.pack_write(out) 203 | data_size += item.serializer.size 204 | out.seek(0) 205 | out.write(struct.pack('3I', 42, 3, data_size)) 206 | target_data = out.getvalue() 207 | 208 | standard_tests(target_obj, target_data) 209 | 210 | # Test malformed data_size 211 | buffer = bytearray(target_data) 212 | struct.pack_into('3I', buffer, 0, 42, 3, 0) # write over data_size with 0 213 | with pytest.raises(ValueError): 214 | Compound.create_unpack_from(buffer) 215 | 216 | 217 | def test_finality() -> None: 218 | with pytest.raises(TypeError): 219 | array[Header[3], Final()] 220 | -------------------------------------------------------------------------------- /tests/test_base_types.py: -------------------------------------------------------------------------------- 1 | import io 2 | import struct 3 | 4 | from typing import Annotated 5 | 6 | import pytest 7 | 8 | from structured import * 9 | from structured.serializers import noop_action 10 | from structured.type_checking import annotated 11 | 12 | from . import standard_tests 13 | 14 | 15 | ## Only tests needed for lines not tested by Structured tests 16 | 17 | 18 | def test_counted() -> None: 19 | cls = annotated.transform(pad[2]) 20 | assert isinstance(cls, StructSerializer) 21 | assert cls.format == '2x' 22 | assert cls.num_values == 0 23 | 24 | with pytest.raises(TypeError): 25 | pad[''] 26 | with pytest.raises(ValueError): 27 | pad[-1] 28 | 29 | 30 | class TestCustomType: 31 | def test_subclassing_any(self) -> None: 32 | class MutableType: 33 | def __init__(self, value: int): 34 | self._value = value 35 | 36 | def __int__(self) -> int: 37 | return self._value 38 | 39 | def __index__(self) -> int: 40 | # For struct.pack 41 | return self._value 42 | 43 | def negate(self) -> None: 44 | self._value = -self._value 45 | 46 | def __eq__(self, other) -> bool: 47 | if isinstance(other, type(self)): 48 | return self._value == other._value 49 | else: 50 | return self._value == other 51 | 52 | for bo in ByteOrder: 53 | class Base(Structured, byte_order=bo): 54 | a: Annotated[MutableType, SerializeAs(int16)] 55 | b: Annotated[MutableType, SerializeAs(uint32)] 56 | assert isinstance(Base.serializer, StructActionSerializer) 57 | assert Base.serializer.actions == (MutableType, MutableType) 58 | target_obj = Base(MutableType(11), MutableType(42)) 59 | target_data = Base.serializer.pack(11, 42) 60 | 61 | standard_tests(target_obj, target_data) 62 | 63 | b = Base.create_unpack(target_data) 64 | assert isinstance(b.a, MutableType) 65 | 66 | 67 | def test_custom_action(self) -> None: 68 | class MutableType: 69 | _wrapped: int 70 | 71 | def __init__(self, not_an_int, value: int): 72 | self._wrapped = value 73 | 74 | def __index__(self) -> int: 75 | return self._wrapped 76 | 77 | @classmethod 78 | def from_int(cls, value: int): 79 | return cls(None, value) 80 | 81 | def __eq__(self, other): 82 | if isinstance(other, MutableType): 83 | return self._wrapped == other._wrapped 84 | else: 85 | return self._wrapped == other 86 | MutableType8 = Annotated[MutableType, SerializeAs(int8).with_factory(MutableType.from_int)] 87 | 88 | class Base(Structured): 89 | a: MutableType8 90 | b: int8 91 | assert isinstance(Base.serializer, StructActionSerializer) 92 | assert Base.serializer.actions == (MutableType.from_int, noop_action) 93 | assert Base.serializer.format == '2b' 94 | assert Base.serializer.num_values == 2 95 | 96 | target_obj = Base(MutableType(None, 42), 10) 97 | target_data = struct.pack('2b', 42, 10) 98 | 99 | standard_tests(target_obj, target_data) 100 | 101 | def test_errors(self) -> None: 102 | with pytest.raises(TypeError): 103 | # Must serialize as a StructSerializer with 1 value 104 | SerializeAs(pad[1]) 105 | with pytest.raises(TypeError): 106 | SerializeAs(StructSerializer('2b')) 107 | with pytest.raises(TypeError): 108 | # Not a struct serializer 109 | SerializeAs(array) 110 | -------------------------------------------------------------------------------- /tests/test_conditional.py: -------------------------------------------------------------------------------- 1 | import struct 2 | 3 | import pytest 4 | 5 | from structured import * 6 | from structured.type_checking import Annotated 7 | 8 | from . import standard_tests, Final 9 | 10 | 11 | class TestConditional: 12 | def test_errors(self) -> None: 13 | with pytest.raises(TypeError): 14 | class Base(Structured): 15 | a: Annotated[int, Condition(lambda x: True, 0)] 16 | 17 | with pytest.raises(ValueError): 18 | class Base(Structured): 19 | a: Annotated[uint8, Condition(lambda x: True)] 20 | 21 | with pytest.raises(ValueError): 22 | class Base(Structured): 23 | a: Annotated[uint8, Condition(lambda x: True, 1, 2)] 24 | 25 | def test_condition(self) -> None: 26 | # See docs NOTE about using Condition with simple types and padding, to 27 | # avoid having to do that here, we'll use the NETWORK byte order specifier 28 | class Versioned(Structured, byte_order=ByteOrder.NETWORK): 29 | version: uint8 30 | field1: uint32 31 | field5: Annotated[int16, Condition(lambda x: x.version > 2, -1)] # Added in version 3 32 | field2: char[2] 33 | field4: Annotated[float32, Condition(lambda x: x.version > 1, 0.0)] # Added in version 2 34 | field3: uint16 35 | 36 | v1_data = struct.pack('!BI2sH', 1, 42, b'Hi', 69) 37 | v1_obj = Versioned(1, 42, -1, b'Hi', 0.0, 69) 38 | # Note: Pick float values that can be represented exactly 39 | v2_data = struct.pack('!BI2sfH', 2, 42, b'Hi', 1.125, 69) 40 | v2_obj = Versioned(2, 42, -1, b'Hi', 1.125, 69) 41 | v3_data = struct.pack('!BIh2sfH', 3, 42, -42, b'Hi', 1.125, 69) 42 | v3_obj = Versioned(3, 42, -42, b'Hi', 1.125, 69) 43 | 44 | standard_tests(v1_obj, v1_data) 45 | standard_tests(v2_obj, v2_data) 46 | standard_tests(v3_obj, v3_data) 47 | 48 | v2_obj.version = 1 49 | assert v2_obj.pack() == v1_data 50 | v3_obj.version = 2 51 | assert v3_obj.pack() == v2_data 52 | v3_obj.version = 1 53 | assert v3_obj.pack() == v1_data 54 | 55 | def test_finality(self) -> None: 56 | class Base(Structured): 57 | a: Annotated[int, Final(), Condition(lambda x: True, 0)] 58 | 59 | assert isinstance(Base.serializer, ConditionalSerializer) 60 | assert Base.serializer.serializers[True].is_final() 61 | assert Base.serializer.is_final() 62 | -------------------------------------------------------------------------------- /tests/test_generics.py: -------------------------------------------------------------------------------- 1 | import struct 2 | from typing import Annotated, Generic, TypeVar, Union, get_type_hints 3 | 4 | import pytest 5 | 6 | from structured import * 7 | from structured.utils import StructuredAlias 8 | 9 | 10 | _Byte = TypeVar('_Byte', uint8, int8) 11 | _String = TypeVar('_String', bound=Union[char, pascal, unicode]) 12 | _Size = TypeVar('_Size', uint8, uint16, uint32, uint64) 13 | T = TypeVar('T', bound=Structured) 14 | U = TypeVar('U') 15 | V = TypeVar('V') 16 | 17 | 18 | class Base(Generic[_Byte, _String], Structured): 19 | a: _Byte 20 | b: _String 21 | 22 | 23 | class UnsignedUnicode(Base[uint8, unicode[uint8]]): 24 | pass 25 | 26 | 27 | class Item(Structured): 28 | a: int8 29 | 30 | 31 | class TestAliasing: 32 | tvar_map = { 33 | _Size: uint32, 34 | _Byte: uint8, 35 | T: Item, 36 | } 37 | 38 | def test_unicode(self) -> None: 39 | obj = unicode[_Size] 40 | assert isinstance(obj, StructuredAlias) 41 | assert obj.cls is unicode 42 | assert obj.args == (_Size, 'utf8') 43 | assert obj.resolve(self.tvar_map) is unicode[uint32] 44 | 45 | def test_Header(self) -> None: 46 | obj = Header[1, _Size] 47 | assert isinstance(obj, StructuredAlias) 48 | assert obj.cls is Header 49 | assert obj.args == (1, _Size) 50 | assert obj.resolve(self.tvar_map) is Header[1, uint32] 51 | 52 | def test_char(self) -> None: 53 | obj = char[_Size] 54 | assert isinstance(obj, StructuredAlias) 55 | assert obj.cls is char 56 | assert obj.args == (_Size,) 57 | assert obj.resolve(self.tvar_map) is char[uint32] 58 | 59 | def test_array(self) -> None: 60 | # same typevar 61 | obj = array[Header[_Size], _Size] 62 | assert isinstance(obj, StructuredAlias) 63 | assert obj.cls is array 64 | assert obj.args == (Header[_Size], _Size) 65 | assert obj.resolve(self.tvar_map) is array[Header[uint32], uint32] 66 | 67 | # different typevars 68 | obj = array[Header[_Size, _Byte], T] 69 | assert isinstance(obj, StructuredAlias) 70 | assert obj.cls is array 71 | assert obj.args == (Header[_Size, _Byte], T) 72 | 73 | obj1 = obj.resolve({_Size: uint32}) 74 | assert isinstance(obj1, StructuredAlias) 75 | assert isinstance(obj1.args[0], StructuredAlias) 76 | assert obj1.args[0].args == (uint32, _Byte) 77 | assert obj1.args[1] is T 78 | 79 | obj2 = obj1.resolve({_Byte: uint8}) 80 | assert isinstance(obj2, StructuredAlias) 81 | assert obj2.cls is array 82 | assert obj2.args == (Header[uint32, uint8], T) 83 | 84 | obj3 = obj2.resolve({T: Item}) 85 | assert obj3 is array[Header[uint32, uint8], Item] 86 | assert obj.resolve(self.tvar_map) is array[Header[uint32, uint8], Item] 87 | 88 | 89 | def test_automatic_resolution(): 90 | class Item(Structured): 91 | a: int8 92 | 93 | class Base(Generic[_Size, T, U, V], Structured): 94 | a: _Size 95 | b: unicode[U] 96 | c: array[Header[1, V], T] 97 | 98 | class PartiallySpecialized(Generic[U, T], Base[uint8, T, uint32, U]): pass 99 | class FullySpecialized1(Base[uint8, Item, uint32, uint16]): pass 100 | class FullySpecialized2(PartiallySpecialized[uint16, Item]): pass 101 | 102 | assert PartiallySpecialized.attrs == ('a', 'b') 103 | hints = get_type_hints(PartiallySpecialized, include_extras=True) 104 | assert hints['a'] is uint8 105 | assert hints['b'] is unicode[uint32] 106 | assert isinstance(hints['c'], StructuredAlias) 107 | 108 | assert FullySpecialized1.attrs == FullySpecialized2.attrs 109 | assert FullySpecialized1.attrs == ('a', 'b', 'c') 110 | assert get_type_hints(FullySpecialized1) == get_type_hints(FullySpecialized2) 111 | 112 | 113 | def test_auto_subclassing_simple(): 114 | # Tests the simple case of auto-subclassing: when a concrete type is 115 | # specified from a non-generic container class. 116 | class Inner(Generic[U], Structured): 117 | a: U 118 | 119 | class Outer(Structured): 120 | a: Inner[uint8] 121 | 122 | assert isinstance(Inner.serializer, NullSerializer) 123 | assert isinstance(Outer.serializer, StructuredSerializer) 124 | assert Outer.serializer.obj_type.attrs == ('a', ) 125 | 126 | 127 | def test_auto_subclassing_complex(): 128 | # Tests the complicated case of auto-subclassing: when the containing class 129 | # is also generic, and later is subclassed. 130 | class Inner(Generic[U], Structured): 131 | a: U 132 | 133 | class Outer(Generic[U], Structured): 134 | a: Inner[U] 135 | 136 | assert isinstance(Outer.serializer, NullSerializer) 137 | 138 | class ConcreteOuter(Outer[uint8]): 139 | pass 140 | 141 | assert isinstance(ConcreteOuter.serializer, StructuredSerializer) 142 | assert ConcreteOuter.serializer.obj_type.attrs == ('a', ) 143 | 144 | 145 | def test_serialized_generics() -> None: 146 | class Base(Generic[_Size], Structured): 147 | a: Annotated[list[_Size], array[Header[3], _Size]] 148 | 149 | class Concrete(Base[uint32]): 150 | pass 151 | assert isinstance(Concrete.serializer, StaticStructArraySerializer) 152 | assert Concrete.serializer.serializer.format == '3I' 153 | 154 | assert Concrete.attrs == ('a',) 155 | target_data = struct.pack(f'3I', 1, 2, 3) 156 | target_obj = Concrete.create_unpack(target_data) 157 | assert target_obj.a == [1, 2, 3] 158 | 159 | 160 | def test_errors() -> None: 161 | class NotGeneric(Structured): 162 | a: uint8 163 | 164 | with pytest.raises(TypeError): 165 | NotGeneric._get_specialization_hints(uint8) -------------------------------------------------------------------------------- /tests/test_self.py: -------------------------------------------------------------------------------- 1 | import struct 2 | from operator import attrgetter 3 | 4 | import pytest 5 | 6 | from structured import * 7 | from structured.type_checking import Self, Generic, TypeVar, Annotated 8 | 9 | from . import standard_tests 10 | 11 | 12 | class TestSelf: 13 | def test_detection(self) -> None: 14 | class Base(Structured): 15 | a: Self 16 | 17 | assert isinstance(Base.serializer, SelfSerializer) 18 | 19 | with pytest.raises(RecursionError): 20 | Base.create_unpack(b'') 21 | 22 | class Derived(Base): 23 | b: int8 24 | 25 | assert isinstance(Derived.serializer, CompoundSerializer) 26 | assert isinstance(Derived.serializer.serializers[0], SelfSerializer) 27 | 28 | 29 | T = TypeVar('T') 30 | class BaseGeneric(Generic[T], Structured): 31 | a: T 32 | b: Self 33 | 34 | class DerivedGeneric(BaseGeneric[int8]): 35 | pass 36 | 37 | assert isinstance(DerivedGeneric.serializer, CompoundSerializer) 38 | assert isinstance(DerivedGeneric.serializer.serializers[1], SelfSerializer) 39 | assert isinstance(DerivedGeneric.serializer.serializers[0], struct.Struct) 40 | assert DerivedGeneric.serializer.serializers[0].format == 'b' 41 | 42 | 43 | def test_arrays(self) -> None: 44 | # Test nesting to at least 2 levels 45 | class Base(Structured): 46 | a: array[Header[uint32], Self] 47 | b: uint8 48 | 49 | level2_items = [ 50 | Base([], 42), 51 | ] 52 | level1_items = [ 53 | Base([], 1), 54 | Base([], 2), 55 | Base(level2_items, 3), 56 | ] 57 | level0_item = Base(level1_items, 0) 58 | 59 | # Level 2 data 60 | item_data = struct.pack('IB', 0, 42) 61 | # Level 1 data 62 | item1_data = struct.pack('IB', 0, 1) 63 | item2_data = struct.pack('IB', 0, 2) 64 | item3_data = struct.pack('I', 1) + item_data + struct.pack('B', 3) 65 | # Level 0 data 66 | container_data = struct.pack('I', 3) + item1_data + item2_data + item3_data + struct.pack('B', 0) 67 | 68 | standard_tests(level0_item, container_data) 69 | 70 | unpacked_obj = Base.create_unpack(container_data) 71 | assert isinstance(unpacked_obj.a[0], Base) 72 | 73 | def test_unions(self) -> None: 74 | decider = LookbackDecider( 75 | attrgetter('type_flag'), 76 | { 77 | 0: Self, 78 | 1: uint64, 79 | } 80 | ) 81 | class Base(Structured): 82 | type_flag: uint8 83 | data: Annotated[Self | uint64, decider] 84 | 85 | nested_obj = Base(1, 42) 86 | # Note: not the same as pack('BQ', ...), because of padding inserted 87 | nested_data = struct.pack('B', 1) + struct.pack('Q', 42) 88 | container_obj = Base(0, nested_obj) 89 | container_data = struct.pack('B', 0) + nested_data 90 | assert container_obj.pack() == container_data 91 | standard_tests(container_obj, container_data) 92 | -------------------------------------------------------------------------------- /tests/test_serializers.py: -------------------------------------------------------------------------------- 1 | """Serializer tests which aren't covered by the tests of full Structured 2 | classes""" 3 | from typing import Union, Annotated 4 | 5 | import pytest 6 | 7 | from structured import * 8 | from . import Final 9 | 10 | 11 | class TestStructSerializers: 12 | def test_errors(self) -> None: 13 | # Base class invalid multipliers 14 | st = StructSerializer[int]('b') 15 | with pytest.raises(TypeError): 16 | st * 1.0 # type: ignore 17 | with pytest.raises(ValueError): 18 | st * -1 # type: ignore 19 | 20 | st2 = StructSerializer[int]('b', byte_order=ByteOrder.NETWORK) 21 | with pytest.raises(ValueError): 22 | st + st2 # type: ignore 23 | 24 | st3 = StructActionSerializer('b') 25 | with pytest.raises(TypeError): 26 | 1 + st3 # type: ignore 27 | 28 | def test_add(self) -> None: 29 | st = StructSerializer('b') 30 | st2 = StructActionSerializer('b') 31 | 32 | # According to https://docs.python.org/3/reference/datamodel.html#object.__radd__, 33 | # StructActionSerializer.__radd__ should be called here, but instead 34 | # STructSerializer.__add__ is 35 | # Why?? 36 | res = st + st2 37 | assert isinstance(res, StructActionSerializer) 38 | assert res.format == '2b' 39 | 40 | null = NullSerializer() 41 | assert st2 + null is st2 42 | 43 | def test_eq(self) -> None: 44 | # This should hit the NotImplemented case, which falls back to 45 | # object.__eq__, which will be False 46 | st1 = StructSerializer('b') 47 | st2 = StructActionSerializer('b') 48 | null = NullSerializer() 49 | assert st1 != null 50 | assert st2 != null 51 | 52 | assert st2 == st2 53 | assert st1 != st2 54 | 55 | def test_hash(self) -> None: 56 | st1 = StructSerializer('b') 57 | st2 = StructActionSerializer('b') 58 | st3 = StructActionSerializer('b', actions=(lambda x: x, )) 59 | 60 | d = { 61 | st1: 1, 62 | st2: 2, 63 | st3: 3, 64 | } 65 | assert len(d) == 3 66 | 67 | def test_with_byte_order(self) -> None: 68 | st = StructActionSerializer('b') 69 | st2 = st.with_byte_order(ByteOrder.NETWORK) 70 | assert st2.base_format == 'b' 71 | assert st2.byte_order == ByteOrder.NETWORK 72 | 73 | 74 | class TestNullSerializer: 75 | def test_pack(self) -> None: 76 | null = NullSerializer() 77 | assert null.pack() == b'' 78 | 79 | def test_add(self) -> None: 80 | # Error case is the only one not tested by other tests 81 | null = NullSerializer() 82 | final = Final() 83 | with pytest.raises(TypeError): 84 | null + 1 # type: ignore 85 | 86 | serializer = StructSerializer('B') 87 | assert serializer + null is serializer 88 | assert null + serializer is serializer 89 | 90 | assert final + null is final 91 | assert null + final is final 92 | 93 | 94 | class TestCompoundSerializer: 95 | class Base1(Structured): 96 | # Force a compound serializer 97 | a: int8 98 | b: Annotated[Union[int8, char[1]], LookbackDecider(lambda x: 0, {0: int8})] 99 | 100 | def test_add(self) -> None: 101 | # The rest are tested by Structured class creation 102 | # Compound + Compound 103 | st = self.Base1.serializer + self.Base1.serializer 104 | assert isinstance(st, CompoundSerializer) 105 | assert len(st.serializers) == 4 106 | 107 | # Compound + Struct 108 | st2 = StructSerializer('b') 109 | st3 = self.Base1.serializer + st2 110 | assert isinstance(st3, CompoundSerializer) 111 | assert len(st3.serializers) == 3 112 | 113 | # Struct + Compound 114 | st4 = st2 + self.Base1.serializer 115 | assert isinstance(st4, CompoundSerializer) 116 | assert len(st4.serializers) == 2 # the nested StructSerializer is combined 117 | 118 | # Error cases 119 | with pytest.raises(TypeError): 120 | 1 + self.Base1.serializer # type: ignore 121 | with pytest.raises(TypeError): 122 | self.Base1.serializer + 1 # type: ignore 123 | # Test explicitly here since CompoundSerializer.__add__ has its own 124 | # implementation 125 | final_compound = self.Base1.serializer + Final() 126 | with pytest.raises(TypeError): 127 | final_compound + StructSerializer('B') 128 | 129 | 130 | def test_preprocess(self) -> None: 131 | # Sort of a silly test, but here it is 132 | preprocessed = self.Base1.serializer.preunpack(None) 133 | assert preprocessed is preprocessed.preunpack(None) 134 | 135 | def test_finality(self) -> None: 136 | final = Final() 137 | class Base(Structured): 138 | a: uint8 139 | b: final 140 | 141 | assert isinstance(Base.serializer, CompoundSerializer) 142 | assert Base.serializer.is_final() 143 | 144 | 145 | class TestUnionSerializer: 146 | def test_lookahead(self) -> None: 147 | serializer = LookbackDecider(lambda x: 0, {0: int8}) 148 | assert serializer.size == 0 149 | 150 | with pytest.raises(TypeError): 151 | LookaheadDecider(1, lambda x: 0, {0: int8}) 152 | 153 | def test_finality(self) -> None: 154 | class Base(Structured): 155 | # Note: the actual serializer(s) used for unpacking occur in the LookbackDecider 156 | a: Annotated[Union[uint32,float32], LookbackDecider(lambda x: 0, {0:uint32, 1:Final()})] 157 | 158 | assert Base.serializer.is_final() 159 | -------------------------------------------------------------------------------- /tests/test_strings.py: -------------------------------------------------------------------------------- 1 | import io 2 | import struct 3 | import math 4 | 5 | import pytest 6 | 7 | from structured import * 8 | from structured.type_checking import annotated 9 | 10 | from . import standard_tests 11 | 12 | 13 | def test_errors() -> None: 14 | with pytest.raises(TypeError): 15 | unicode[int8] 16 | with pytest.raises(TypeError): 17 | unicode[5, uint32] 18 | with pytest.raises(TypeError): 19 | # NOTE: hash(1) == hash(1.0), so because instance creation is cached 20 | # based on the args, technically char[1.0] won't fail if char[1] has 21 | # already been created. 22 | # TODO: Probably fix this, don't want unintended behavior like this. 23 | char[1.1] 24 | with pytest.raises(ValueError): 25 | char[b'aa'] 26 | 27 | 28 | class TestChar: 29 | def test_static(self) -> None: 30 | unwrapped = annotated.transform(char[13]) 31 | assert isinstance(unwrapped, StructSerializer) 32 | assert unwrapped.format == '13s' 33 | 34 | def test_dynamic(self) -> None: 35 | class Base(Structured): 36 | a: int16 37 | b: char[uint8] 38 | c: int32 39 | d: int32 40 | 41 | assert isinstance(Base.serializer, CompoundSerializer) 42 | assert [serializer.num_values for serializer in Base.serializer.serializers] == [ 43 | # 'a' uint16 can't be combined with 'b' char 44 | 1, 45 | # 'b' char can't be combined with 'c' int32 46 | 1, 47 | # 'c' and 'd' int32 can be combined 48 | 2, 49 | ] 50 | assert Base.attrs == ('a', 'b', 'c', 'd') 51 | 52 | st = struct.Struct('hBs2I') 53 | 54 | target_obj = Base(10, b'a', 42, 11) 55 | target_data = st.pack(10, 1, b'a', 42, 11) 56 | 57 | standard_tests(target_obj, target_data) 58 | 59 | def test_null(self) -> None: 60 | class Base(Structured): 61 | name: null_char 62 | 63 | source_data = b'Hello\0Extra' 64 | target_data = b'Hello\0' 65 | target_obj = Base(b'Hello') 66 | 67 | standard_tests(target_obj, target_data) 68 | 69 | assert Base.create_unpack(source_data) == target_obj 70 | 71 | buffer = bytearray(source_data) 72 | assert Base.create_unpack_from(buffer) == target_obj 73 | 74 | with io.BytesIO(source_data) as stream: 75 | assert Base.create_unpack_read(stream) == target_obj 76 | 77 | # Non-terminated 78 | error_data = b'Hello' 79 | with pytest.raises(ValueError): 80 | Base.create_unpack(error_data) 81 | with pytest.raises(ValueError): 82 | with io.BytesIO(error_data) as ins: 83 | Base.create_unpack_read(ins) 84 | with pytest.raises(ValueError): 85 | Base.create_unpack_from(bytearray(error_data)) 86 | 87 | # empty string 88 | target_obj.name = b'' 89 | assert target_obj.pack() == b'\0' 90 | # no delim 91 | target_obj.name = b'Hi' 92 | assert target_obj.pack() == b'Hi\0' 93 | 94 | def test_custom_delim(self) -> None: 95 | class Base(Structured): 96 | name: char[b'H'] 97 | 98 | target_obj = Base(b'Foo') 99 | assert target_obj.pack() == b'FooH' 100 | assert Base.create_unpack(b'FooHBar') == target_obj 101 | 102 | 103 | def test_net_errors(self) -> None: 104 | class Base(Structured): 105 | a: char[NET] 106 | error_length = 0x8000 107 | 108 | error_str = b'a' * error_length 109 | error_obj = Base(error_str) 110 | error_data = struct.pack('H', error_length) 111 | error_size_mark = 0x80 | error_length & 0x7F | (error_length & 0xFF80) << 1 112 | ## NOTE: Not sure if it's even possible to encode a length marker that 113 | ## would fail decoding. Will have to dig into and reverse engineer 114 | ## bit manipulation to find out. 115 | #error_data = struct.pack('H', error_size_mark) 116 | 117 | with pytest.raises(ValueError): 118 | error_obj.pack() 119 | #with pytest.raises(ValueError): 120 | # Base.create_unpack(error_data) 121 | 122 | def test_net(self) -> None: 123 | # NOTE: Code for encoding/decoding the string length is dubious. 124 | # Source is old Wrye Base code for reading/writing OMODs, but I've 125 | # and it seems to work properly, so these tests just exercise the 126 | # code lines, but don't verify their accuracy. 127 | class Base(Structured): 128 | short: char[NET] 129 | long: char[NET] 130 | assert isinstance(Base.serializer, CompoundSerializer) 131 | assert [serializer.num_values for serializer in Base.serializer.serializers] == [ 132 | # char[NET] can't be combined 133 | 1, 134 | 1, 135 | ] 136 | assert Base.attrs == ('short', 'long') 137 | target_obj = Base(b'Hello', b'a'*200) 138 | 139 | st = struct.Struct('B5s') 140 | partial_target = st.pack(5, b'Hello') 141 | partial_size = st.size 142 | target_size = 1 + 5 + 2 + 200 143 | 144 | # pack/unpack 145 | packed_data = target_obj.pack() 146 | packed_size = len(packed_data) 147 | assert packed_data[:partial_size] == partial_target 148 | assert Base.create_unpack(packed_data) == target_obj 149 | assert packed_size == target_size 150 | 151 | # from/into 152 | buffer = bytearray(packed_size) 153 | target_obj.pack_into(buffer) 154 | assert target_obj.serializer.size == packed_size 155 | assert bytes(buffer) == packed_data 156 | assert bytes(buffer)[:partial_size] == partial_target 157 | assert Base.create_unpack_from(buffer) == target_obj 158 | 159 | # read/write 160 | with io.BytesIO() as stream: 161 | target_obj.pack_write(stream) 162 | assert target_obj.serializer.size == packed_size 163 | assert stream.getvalue() == packed_data 164 | assert stream.getvalue()[:partial_size] == partial_target 165 | stream.seek(0) 166 | assert Base.create_unpack_read(stream) == target_obj 167 | 168 | def test_finality(self) -> None: 169 | assert isinstance(s := annotated.transform(char[math.inf]), Serializer) 170 | assert s.is_final() 171 | 172 | def test_consuming(self) -> None: 173 | class Base(Structured): 174 | a: uint8 175 | b: char[math.inf] 176 | 177 | target_data1 = struct.pack('B', 42) + b'Hello' 178 | target_obj = Base(42, b'Hello') 179 | standard_tests(target_obj, target_data1) 180 | 181 | target_data2 = struct.pack('B', 42) + b'' 182 | target_obj.b = b'' 183 | standard_tests(target_obj, target_data2) 184 | 185 | 186 | class TestUnicode: 187 | def test_default(self) -> None: 188 | target_str = '你好' 189 | target_data = target_str.encode('utf8') 190 | target_len = len(target_data) 191 | 192 | class Base(Structured): 193 | a: unicode[target_len] 194 | target_obj = Base(target_str) 195 | 196 | standard_tests(target_obj, target_data) 197 | 198 | def test_custom(self) -> None: 199 | class Custom(EncoderDecoder): 200 | @classmethod 201 | def encode(cls, strng: str) -> bytes: 202 | return b'Hello!' 203 | 204 | @classmethod 205 | def decode(cls, byts: bytes) -> str: 206 | return 'banana' 207 | 208 | class Base(Structured): 209 | a: unicode[6, Custom] 210 | target_obj = Base('banana') 211 | target_data = b'Hello!' 212 | size = 6 213 | 214 | # pack/unpack 215 | assert target_obj.pack() == target_data 216 | test = Base.create_unpack(target_data) 217 | assert isinstance(test.a, str) 218 | assert test == target_obj 219 | assert test.serializer.size == size 220 | assert Base.create_unpack(b'123456') == target_obj 221 | 222 | # from/into 223 | buffer = bytearray(size) 224 | target_obj.pack_into(buffer) 225 | assert target_obj.serializer.size == size 226 | assert bytes(buffer) == target_data 227 | assert Base.create_unpack_from(buffer) == target_obj 228 | 229 | # read/write 230 | with io.BytesIO() as stream: 231 | target_obj.pack_write(stream) 232 | assert stream.getvalue() == target_data 233 | stream.seek(0) 234 | assert Base.create_unpack_read(stream) == target_obj 235 | 236 | 237 | def test_dynamic(self) -> None: 238 | target_str = '你好' 239 | target_bytes = target_str.encode() 240 | target_len = len(target_bytes) 241 | st = struct.Struct(f'I{target_len}s') 242 | target_data = st.pack(target_len, target_bytes) 243 | 244 | class Base(Structured): 245 | a: unicode[uint32] 246 | target_obj = Base(target_str) 247 | 248 | standard_tests(target_obj, target_data) 249 | 250 | def test_null(self) -> None: 251 | class Base(Structured): 252 | name: null_unicode 253 | 254 | source_data = b'Hello\0Extra' 255 | target_data = b'Hello\0' 256 | target_obj = Base('Hello') 257 | target_size = 6 258 | 259 | # pack/unpack 260 | assert target_obj.pack() == target_data 261 | assert Base.create_unpack(source_data) == target_obj 262 | assert Base.serializer.size == target_size 263 | assert Base.create_unpack(target_data) == target_obj 264 | assert Base.serializer.size == target_size 265 | 266 | # into/from 267 | source_buffer = bytearray(source_data) 268 | buffer = bytearray(6) 269 | target_obj.pack_into(buffer) 270 | assert bytes(buffer) == target_data 271 | assert Base.serializer.size == target_size 272 | assert Base.create_unpack_from(buffer) == target_obj 273 | assert Base.serializer.size == target_size 274 | assert Base.create_unpack_from(source_buffer) == target_obj 275 | assert Base.serializer.size == target_size 276 | 277 | # read/write 278 | with io.BytesIO() as stream: 279 | target_obj.pack_write(stream) 280 | assert Base.serializer.size == target_size 281 | assert stream.getvalue() == target_data 282 | stream.seek(0) 283 | assert Base.create_unpack_read(stream) == target_obj 284 | with io.BytesIO(source_data) as stream: 285 | assert Base.create_unpack_read(stream) == target_obj 286 | assert Base.serializer.size == target_size 287 | 288 | error_data = b'Hello' 289 | with pytest.raises(ValueError): 290 | Base.create_unpack(error_data) 291 | with pytest.raises(ValueError): 292 | with io.BytesIO(error_data) as ins: 293 | Base.create_unpack_read(ins) 294 | 295 | def test_finality(self) -> None: 296 | assert isinstance(s := annotated.transform(unicode[math.inf]), Serializer) 297 | assert s.is_final() 298 | -------------------------------------------------------------------------------- /tests/test_structured.py: -------------------------------------------------------------------------------- 1 | __author__ = 'Lojack' 2 | 3 | import io 4 | import struct 5 | 6 | import pytest 7 | 8 | from structured import * 9 | 10 | from . import standard_tests 11 | 12 | 13 | class TestStructured: 14 | def test_multiple_pad(self) -> None: 15 | class Base(Structured): 16 | _ : pad[1] 17 | _ : pad[3] 18 | _ : pad[4] 19 | assert isinstance(Base.serializer, struct.Struct) 20 | assert Base.serializer.format == '8x' 21 | 22 | class Derived(Base): 23 | _ : pad[2] 24 | assert isinstance(Derived.serializer, struct.Struct) 25 | assert Derived.serializer.format == '10x' 26 | 27 | def test_init__(self) -> None: 28 | class Base(Structured): 29 | a: int8 30 | b: int32 31 | c: int16 32 | 33 | with pytest.raises(TypeError): 34 | Base(1, 2, 3, 4) # Too many args 35 | with pytest.raises(TypeError): 36 | Base(1) # not enough args 37 | with pytest.raises(TypeError): 38 | Base(2, 3, a=1) # a both positional and keyword 39 | with pytest.raises(TypeError): 40 | Base(1, 2, 3, foo=1) # Extra argument 41 | 42 | a = Base(1, 2, 3) 43 | assert a.a == 1 44 | assert a.b == 2 45 | assert a.c == 3 46 | 47 | b = Base(1, 2, c=3) 48 | assert b == a 49 | 50 | def test_byte_order(self) -> None: 51 | for byte_order in ByteOrder: 52 | class Base(Structured, byte_order=byte_order): 53 | a: int8 54 | assert isinstance(Base.serializer, struct.Struct) 55 | if byte_order is ByteOrder.DEFAULT: 56 | assert Base.serializer.format[0] == 'b' 57 | else: 58 | assert Base.serializer.format[0] == byte_order.value 59 | 60 | def test_folding(self) -> None: 61 | class Base(Structured): 62 | a: int8 63 | b: int8 64 | c: int32 65 | d: int64 66 | assert isinstance(Base.serializer, struct.Struct) 67 | assert Base.serializer.format == '2biq' 68 | 69 | def test_types(self) -> None: 70 | class Base(Structured): 71 | _: pad[2] 72 | __: pad[1] 73 | a: int8 74 | A: uint8 75 | b: int16 76 | B: uint16 77 | c: int32 78 | C: uint32 79 | d: int64 80 | D: uint64 81 | e: float16 82 | f: float32 83 | g: float64 84 | h: char[2] 85 | i: char[1] 86 | j: pascal[2] 87 | k: pascal[1] 88 | l: bool8 89 | 90 | other_member: int 91 | 92 | def method(self): 93 | return 'foo' 94 | 95 | assert isinstance(Base.serializer, struct.Struct) 96 | assert ''.join(Base.attrs) == 'aAbBcCdDefghijkl' 97 | assert Base.serializer.format == '3xbBhHiIqQefd2ss2pp?' 98 | 99 | def test_extending(self) -> None: 100 | # Test non-string types are folded in the format string 101 | class Base(Structured): 102 | a: int8 103 | b: int16 104 | c: int16 105 | class Derived(Base): 106 | d: int16 107 | assert isinstance(Derived.serializer, struct.Struct) 108 | assert Derived.serializer.format == 'b3h' 109 | # Test string types aren't folded 110 | # We shouldn't do 111 | ## 112 | ## for string_type in (char, pascal): 113 | ## class Base2(Structured): 114 | ## a: string_type[10] 115 | # Since if annotations are strings (the will be in future python 116 | # versions), even `typing.get_type_hints` would fail to get the 117 | # type hints on Base2 118 | class Base2(Structured): 119 | a: char[10] 120 | class Derived2(Base2): 121 | b: char[3] 122 | assert isinstance(Derived2.serializer, struct.Struct) 123 | assert Derived2.serializer.format == '10s3s' 124 | 125 | class Base3(Structured): 126 | a: pascal[10] 127 | class Derived3(Base3): 128 | b: pascal[3] 129 | assert isinstance(Derived3.serializer, struct.Struct) 130 | assert Derived3.serializer.format == '10p3p' 131 | 132 | with pytest.raises(TypeError): 133 | class Base4(Base2, Base3): 134 | pass 135 | 136 | def test_override_types(self) -> None: 137 | class Base1(Structured): 138 | a: int8 139 | b: int16 140 | class Derived1(Base1): 141 | a: int16 142 | assert isinstance(Derived1.serializer, struct.Struct) 143 | assert Derived1.serializer.format == '2h' 144 | 145 | class Base2(Structured): 146 | a: int8 147 | b: int8 148 | c: int8 149 | class Derived2(Base2): 150 | b: None 151 | assert isinstance(Derived2.serializer, struct.Struct) 152 | assert Derived2.serializer.format == '2b' 153 | assert tuple(Derived2.attrs) == ('a', 'c') 154 | 155 | 156 | def test_mismatched_byte_order(self) -> None: 157 | class Base(Structured): 158 | a: int8 159 | with pytest.raises(ValueError): 160 | class Derived(Base, byte_order=ByteOrder.LE): 161 | b: int8 162 | class Derived2(Base, byte_order=ByteOrder.LE, byte_order_mode=ByteOrderMode.OVERRIDE): 163 | b: int8 164 | assert isinstance(Derived2.serializer, struct.Struct) 165 | assert Derived2.serializer.format == ByteOrder.LE.value + '2b' 166 | 167 | def test_with_byte_order(self) -> None: 168 | class Base(Structured): 169 | a: uint16 170 | objLE = Base(0xF0).with_byte_order(ByteOrder.LE) 171 | objBE = Base(0xF0).with_byte_order(ByteOrder.BE) 172 | target_le_data = struct.pack(f'{ByteOrder.LE.value}H', 0xF0) 173 | target_be_data = struct.pack(f'{ByteOrder.BE.value}H', 0xF0) 174 | assert target_le_data != target_be_data 175 | assert objLE.pack() == target_le_data 176 | assert objBE.pack() == target_be_data 177 | 178 | def test_unpack_read(self) -> None: 179 | class Base(Structured): 180 | a: int8 181 | b = Base(42) 182 | data = b.pack() 183 | with io.BytesIO(data) as stream: 184 | b.a = 0 185 | b.unpack_read(stream) 186 | assert b.a == 42 187 | 188 | def test_pack_write(self) -> None: 189 | class Base(Structured): 190 | a: int8 191 | b = Base(42) 192 | data = b.pack() 193 | with io.BytesIO() as stream: 194 | b.pack_write(stream) 195 | assert data == stream.getvalue() 196 | 197 | def test_pack_unpack(self) -> None: 198 | class Base(Structured): 199 | a: int8 200 | _: pad[2] 201 | b: char[6] 202 | 203 | assert isinstance(Base.serializer, struct.Struct) 204 | 205 | target_obj = Base(1, b'Hello!') 206 | 207 | st = struct.Struct('b2x6s') 208 | target_data = st.pack(1, b'Hello!') 209 | 210 | assert target_obj.pack() == target_data 211 | assert Base.create_unpack(target_data) == target_obj 212 | 213 | test_obj = Base(0, b'') 214 | test_obj.unpack(target_data) 215 | assert test_obj == target_obj 216 | 217 | 218 | def test_pack_unpack_into(self) -> None: 219 | class Base(Structured): 220 | a: int8 221 | b: char[6] 222 | target_obj = Base(1, b'Hello!') 223 | 224 | assert isinstance(Base.serializer, struct.Struct) 225 | 226 | buffer = bytearray(Base.serializer.size) 227 | target_obj.pack_into(buffer) 228 | assert bytes(buffer) == target_obj.pack() 229 | assert Base.create_unpack_from(buffer) == target_obj 230 | 231 | test_obj = Base(0, b'') 232 | test_obj.unpack_from(buffer) 233 | assert test_obj == target_obj 234 | 235 | def test_str(self) -> None: 236 | class Base(Structured): 237 | a: int8 238 | _: pad[3] 239 | b: char[6] 240 | b = Base(10, b'Hello!') 241 | assert str(b) == "Base(a=10, b=b'Hello!')" 242 | 243 | def test_eq(self) -> None: 244 | class Base(Structured): 245 | a: int8 246 | b: char[6] 247 | class Derived(Base): 248 | pass 249 | 250 | a = Base(1, 'Hello!') 251 | b = Base(1, 'Hello!') 252 | c = Derived(1, 'Hello!') 253 | d = Base(0, 'Hello!') 254 | e = Base(1, 'banana') 255 | 256 | assert a == b 257 | assert a != c 258 | assert a != d 259 | assert a != e 260 | 261 | def test_structured_hint(self) -> None: 262 | class Inner(Structured): 263 | a: uint32 264 | b: char[4] 265 | 266 | class Outer(Structured): 267 | a: uint32 268 | b: Inner 269 | 270 | target_obj = Outer(1, Inner(2, b'abcd')) 271 | target_data = struct.pack('2I4s', 1, 2, b'abcd') 272 | 273 | standard_tests(target_obj, target_data) 274 | 275 | 276 | def test_fold_overlaps() -> None: 277 | # Test the branch not exercised by the above tests. 278 | serializer = StructSerializer('b') + StructSerializer('') 279 | assert serializer.format == 'b' 280 | serializer = StructSerializer('4sI') + StructSerializer('I') 281 | assert serializer.format == '4s2I' 282 | serializer = StructSerializer('') + StructSerializer('b') 283 | assert serializer.format == 'b' 284 | 285 | 286 | def test_create_serializers() -> None: 287 | # Just the bits not tested by the above 288 | with pytest.raises(TypeError): 289 | class Error(Structured): 290 | a: array 291 | -------------------------------------------------------------------------------- /tests/test_tuples.py: -------------------------------------------------------------------------------- 1 | import struct 2 | from typing import Tuple, Generic, TypeVar 3 | 4 | from structured import * 5 | 6 | from . import standard_tests, Final 7 | 8 | 9 | T = TypeVar('T') 10 | U = TypeVar('U') 11 | 12 | 13 | def test_tuple_detection(): 14 | class Base(Structured): 15 | a: tuple[int8, int16] 16 | 17 | assert isinstance(Base.serializer, TupleSerializer) 18 | assert Base.serializer.serializer.format == 'bh' 19 | 20 | target_obj = Base((1, 2)) 21 | target_data = struct.pack('bh', 1, 2) 22 | standard_tests(target_obj, target_data) 23 | 24 | class Base2(Structured): 25 | a: Tuple[int8, int16] 26 | assert isinstance(Base2.serializer, TupleSerializer) 27 | 28 | target_obj = Base2((1, 2)) 29 | standard_tests(target_obj, target_data) 30 | 31 | 32 | def test_non_detection(): 33 | class Base(Structured): 34 | a: tuple[int8, int] 35 | assert Base.attrs == () 36 | 37 | class Base2(Structured): 38 | a: tuple[int8, ...] 39 | assert Base2.attrs == () 40 | 41 | 42 | def test_generics(): 43 | class GenericStruct(Generic[T, U], Structured): 44 | a: tuple[T, U] 45 | assert isinstance(GenericStruct.serializer, NullSerializer) 46 | 47 | class ConcreteStruct1(GenericStruct[int8, int]): 48 | pass 49 | assert isinstance(ConcreteStruct1.serializer, NullSerializer) 50 | 51 | class ConcreteStruct2(GenericStruct[int8, int16]): 52 | pass 53 | assert isinstance(ConcreteStruct2.serializer, TupleSerializer) 54 | 55 | 56 | def test_finality(): 57 | final = Final() 58 | class Base(Structured): 59 | a: tuple[int8, final] 60 | assert Base.serializer.is_final() -------------------------------------------------------------------------------- /tests/test_unions.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import struct 3 | from operator import attrgetter 4 | import io 5 | from typing import Union, Annotated 6 | 7 | import pytest 8 | 9 | from structured import * 10 | from structured.serializers import AUnion 11 | 12 | from . import standard_tests 13 | 14 | 15 | def test_errors() -> None: 16 | with pytest.raises(TypeError): 17 | # Invalid type in the result map 18 | LookbackDecider(lambda x: x, {0: int}, int32) 19 | with pytest.raises(TypeError): 20 | # Invalid default serializer 21 | LookbackDecider(lambda x: x, {0: int32}, int) 22 | with pytest.raises(ValueError): 23 | # Mapped result serializer must unpack a single value 24 | LookbackDecider(lambda x: x, {1: pad[1]}, None) 25 | with pytest.raises(ValueError): 26 | # Default serializer must unpack a single value 27 | LookbackDecider(lambda x: x, {1: int32}, pad[1]) 28 | 29 | class Proxy: 30 | a = 0 31 | a = Proxy() 32 | 33 | # No default specified, and decider returned an invalid value 34 | serializer = LookbackDecider(attrgetter('a'), {1: int32}, None) 35 | with pytest.raises(ValueError): 36 | serializer = serializer.prepack(a) 37 | serializer.pack(1) 38 | 39 | 40 | def test_lookback() -> None: 41 | class Base(Structured): 42 | a: Annotated[Union[int8, char[1]], LookbackDecider(lambda x: 0, {0: int8}, int8)] 43 | 44 | assert isinstance(Base.serializer, LookbackDecider) 45 | 46 | test_data = struct.pack('b', 42) 47 | test_obj = Base(42) 48 | 49 | standard_tests(test_obj, test_data) 50 | 51 | 52 | def test_lookahead() -> None: 53 | class Record(Structured): 54 | sig: char[4] 55 | 56 | class IntRecord(Record): 57 | value: uint32 58 | 59 | class FloatRecord(Record): 60 | value: float32 61 | 62 | class Outer(Structured): 63 | record: Annotated[Union[IntRecord, FloatRecord], LookaheadDecider(char[4], attrgetter('record.sig'), {b'IINT': IntRecord, b'FLOA': FloatRecord}, None)] 64 | 65 | int_data = struct.pack('4sI', b'IINT', 42) 66 | float_data = struct.pack('4sf', b'FLOA', 1.125) # NOTE: exact float in binary 67 | 68 | int_obj = Outer(IntRecord(b'IINT', 42)) 69 | float_obj = Outer(FloatRecord(b'FLOA', 1.125)) 70 | 71 | for obj, data in ((int_obj, int_data), (float_obj, float_data)): 72 | standard_tests(obj, data) 73 | 74 | 75 | @pytest.mark.skipif(sys.version_info < (3, 10), reason='requires Python 3.10 or higher') 76 | def test_union_syntax() -> None: 77 | class Base(Structured): 78 | a: Annotated[int8 | char[4], LookbackDecider(lambda x: 0, {0: int8}, int8)] 79 | 80 | assert isinstance(Base.serializer, AUnion) 81 | 82 | 83 | def test_compound_serializer() -> None: 84 | class Base(Structured): 85 | a_type: uint8 86 | a: Annotated[Union[uint32, float32, char[4]], LookbackDecider(attrgetter('a_type'), {0: uint32, 1: float32}, char[4])] 87 | 88 | assert Base.attrs == ('a_type', 'a') 89 | assert isinstance(Base.serializer, CompoundSerializer) 90 | 91 | # NOTE: Using a float that can be represented exactly in binary 92 | test_data = [struct.pack('=BI', 0, 42), struct.pack('=Bf', 1, 1.125), struct.pack('=B4s', 2, b'FOOD')] 93 | test_objs = [Base(0, 42), Base(1, 1.125), Base(2, b'FOOD')] 94 | 95 | # Check size to ensure the preprocessing serializers are correctly updating 96 | # their origin serializers. 97 | for data, obj in zip(test_data, test_objs): 98 | standard_tests(obj, data) 99 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from structured import * 2 | 3 | # No tests at the moment 4 | --------------------------------------------------------------------------------