├── .gitignore ├── LICENSE ├── README.md └── restruct.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.pyc 3 | *.pyo 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020, Shiz 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | restruct 2 | ======== 3 | 4 | Declarative binary file format parser and emitter for Python 3. 5 | 6 | Quickstart 7 | ---------- 8 | 9 | ```python3 10 | from restruct import parse, emit, UInt, Arr 11 | 12 | # Parse a simple little-endian integer 13 | >>> parse(UInt(32), b'\x39\x05\x00\x00') 14 | 1337 15 | 16 | # Parse an array that continues 'til end-of-stream 17 | >>> parse(Arr(UInt(8)), b'\x45\x45\x45\x45') 18 | [69, 69, 69, 69] 19 | 20 | # Parse any bytes-like structures and files! 21 | >>> parse(Arr(UInt(8)), open('foo.bin', 'rb')) 22 | [13, 37] 23 | 24 | # Emit data out again! 25 | >>> emit(UInt(32), 420, open('bar.bin', 'wb')) 26 | 27 | # Or emit to a BytesIO object if none given 28 | >>> emit(Arr(UInt(8)), [13, 37, 69, 69]) 29 | <_io.BytesIO object at 0x106cc0810> 30 | >>> _.getvalue().hex() 31 | '0d254545' 32 | ``` 33 | 34 | Structures 35 | ---------- 36 | 37 | ```python3 38 | from restruct import parse, Struct 39 | 40 | class Test(Struct): 41 | # restruct standard types are injected by default, so no need to import them 42 | foo: UInt(32) 43 | bar: Str(type='c') 44 | 45 | >>> parse(Test, b'\x39\x05\x00\x00Hello world!\x00Garbage') 46 | Test { 47 | foo: 1337, 48 | bar: 'Hello world!' 49 | } 50 | >>> _.foo 51 | 1337 52 | ``` 53 | 54 | Hooks 55 | ----- 56 | 57 | ```python3 58 | from restruct import parse, Struct 59 | 60 | class Stats(Struct): 61 | engine_level: UInt(32) 62 | rpm: UInt(16) 63 | 64 | class Message(Struct): 65 | message: Str(length=64, exact=True) 66 | priority: UInt(8) 67 | 68 | def on_emit_priority(self, spec, context): 69 | # called before `priority` is emitted 70 | self.priority = len(self.message) 71 | 72 | class Test(Struct): 73 | type: UInt(32) 74 | contents: Switch(options={ 75 | 1: Stats, 76 | 2: Message, 77 | }) 78 | 79 | def on_parse_type(self, spec, context): 80 | # called when `type` field is set, spec contains the field types 81 | spec.contents.selector = self.type 82 | 83 | >>> parse(Test, b"\x02\x00\x00\x00Did you expect a cute foo? Too bad, it's just me, bar!\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x63") 84 | Test { 85 | type: 2, 86 | contents: Message { 87 | message: Did you expect a cute foo? Too bad, it's just me, bar!, 88 | priority: 99 89 | } 90 | } 91 | ``` 92 | 93 | Inheritance 94 | ----------- 95 | 96 | ```python3 97 | from restruct import parse, Struct 98 | 99 | class Base(Struct): 100 | a: UInt(8) 101 | b: UInt(8) 102 | 103 | class Derived(Base): 104 | c: UInt(8) 105 | d: UInt(8) 106 | 107 | # just works! 108 | >>> parse(Derived, b'\x01\x02\x03\x04') 109 | Derived { 110 | a: 1, 111 | b: 2, 112 | c: 3, 113 | d: 4 114 | } 115 | ``` 116 | 117 | Partials 118 | -------- 119 | TODO 120 | 121 | References 122 | ---------- 123 | 124 | ```python3 125 | from restruct import parse, Struct 126 | 127 | class ReferencedStruct(Struct): 128 | magic: Fixed(b'ENTRY') 129 | foo: UInt(32) 130 | bar: Str(type='c') 131 | 132 | # declare variable name, one for each reference 133 | class HasReference(Struct, partials={'P'}): 134 | magic: Fixed(b'TARR') 135 | offset: UInt(32) @ P.point 136 | data: Ref(Arr(ReferencedStruct)) @ P 137 | 138 | >>> parse(HasReference, b'TARR\x10\x00\x00\x00\xDE\xAD\xBE\xEF\xDE\xAD\xC0\xDEENTRY\x2A\x00\x00\x00First entry\x00ENTRY\x45\x00\x00\x00Second entry\x00') 139 | HasReference { 140 | magic: [54 41 52 52], 141 | offset: 16, 142 | data: [ReferencedStruct { 143 | magic: [45 4e 54 52 59], 144 | foo: 42, 145 | bar: 'First entry' 146 | }, ReferencedStruct { 147 | magic: [45 4e 54 52 59], 148 | foo: 69, 149 | bar: 'Second entry' 150 | }] 151 | } 152 | # compressed and with correct offsets! 153 | >>> emit(HasReference, _).getvalue() 154 | b'TARR\x08\x00\x00\x00ENTRY*\x00\x00\x00First entry\x00ENTRYE\x00\x00\x00Second entry\x00' 155 | 156 | # use two partials: one for the reference, one for the array count 157 | class TrickierReference(Struct, partials={'R', 'A'}): 158 | magic: Fixed(b'WARR') 159 | offset: UInt(32) @ R.point 160 | count: UInt(32) @ A.count 161 | data: Ref(Arr(ReferencedStruct) @ A) @ R 162 | 163 | >>> parse(TrickierReference, b'WARR\x10\x00\x00\x00\x01\x00\x00\x00\xDE\xAD\xC0\xDEENTRY\x2A\x00\x00\x00There can only be one\x00ENTRY\x45\x00\x00\x00Second nonsense entry\x00') 164 | TrickierReference { 165 | magic: [57 41 52 52], 166 | offset: 16, 167 | count: 1, 168 | data: [ReferencedStruct { 169 | magic: [45 4e 54 52 59], 170 | foo: 42, 171 | bar: 'There can only be one' 172 | }] 173 | } 174 | >>> emit(TrickierReference, _).getvalue() 175 | b'WARR\x0c\x00\x00\x00\x01\x00\x00\x00ENTRY*\x00\x00\x00There can only be one\x00' 176 | ``` 177 | 178 | TODO: discuss streams 179 | 180 | Generics 181 | -------- 182 | 183 | ```python3 184 | from restruct import parse, Struct, UInt 185 | 186 | class GenericTest(Struct, generics=['T']): 187 | # now you can use the variable T to stand in for any type and most values! 188 | foo: UInt(32) 189 | bar: Arr(T) 190 | 191 | # use [] syntax on the type to resolve the generic 192 | >>> parse(GenericTest[UInt(16)], b'\x39\x05\x00\x00\x45\x00\xa4\x01\x11\x22') 193 | GenericTest[UInt(16, le)] { 194 | foo: 1337, 195 | bar: [69, 420, 8721] 196 | } 197 | 198 | # failing to resolve all generics before parsing predictably fails 199 | >>> parse(GenericTest, b'\x39\x05\x00\x00huh?') 200 | Traceback (most recent call last): 201 | File "", line 1, in 202 | File "restruct.py", line 1225, in parse 203 | return type.parse(io, context) 204 | File "restruct.py", line 695, in parse 205 | val = parse(type, io, context) 206 | File "restruct.py", line 1225, in parse 207 | return type.parse(io, context) 208 | File "restruct.py", line 619, in parse 209 | raise Error(context, 'unresolved generic') 210 | restruct.Error: [bar] ValueError: unresolved generic 211 | 212 | # also works with inheritance! 213 | ``` 214 | 215 | Error handling 216 | -------------- 217 | 218 | ```python3 219 | from restruct import parse, Struct 220 | 221 | class Inner(Struct): 222 | foo: Str(length=32, type='c') 223 | 224 | class Nested(Struct): 225 | level: UInt(8) 226 | inner: Arr(Inner, count=4) 227 | 228 | class Base(Struct): 229 | version: UInt(8) 230 | nested: Nested 231 | 232 | # errors contain the full path through the structures to the error'd value 233 | >>> parse(Base, b'\x01\x45All\x00Good\x00So\x00\x81hmm\x00\x00') 234 | Traceback (most recent call last): 235 | File "restruct.py", line 1225, in parse 236 | return type.parse(io, context) 237 | File "restruct.py", line 695, in parse 238 | val = parse(type, io, context) 239 | File "restruct.py", line 1225, in parse 240 | return type.parse(io, context) 241 | File "restruct.py", line 695, in parse 242 | val = parse(type, io, context) 243 | File "restruct.py", line 1225, in parse 244 | return type.parse(io, context) 245 | File "restruct.py", line 907, in parse 246 | elem = parse(type, io, context) 247 | File "restruct.py", line 1225, in parse 248 | return type.parse(io, context) 249 | File "restruct.py", line 695, in parse 250 | val = parse(type, io, context) 251 | File "restruct.py", line 1225, in parse 252 | return type.parse(io, context) 253 | File "restruct.py", line 1133, in parse 254 | return raw.decode(encoding) 255 | UnicodeDecodeError: 'utf-8' codec can't decode byte 0x81 in position 0: invalid start byte 256 | 257 | During handling of the above exception, another exception occurred: 258 | 259 | Traceback (most recent call last): 260 | File "", line 1, in 261 | File "restruct.py", line 1230, in parse 262 | raise Error(context, e) 263 | restruct.Error: [nested.inner[3].foo] UnicodeDecodeError: 'utf-8' codec can't decode byte 0x81 in position 0: invalid start byte 264 | # access the path programmatically 265 | >>> e.path 266 | [('nested', ), ('inner', [4])>), (3, ), ('foo', )] 267 | # access the original exception 268 | >>> e.exception 269 | UnicodeDecodeError('utf-8', b'\x81', 0, 1, 'invalid start byte') 270 | ``` 271 | 272 | Standard types 273 | -------------- 274 | 275 | * `Int(bits, order='le', signed=True):` two's-complement integer 276 | * `UInt(bits, order='le'):` two's-complement unsigned integer 277 | * `Float(bits):` IEEE754 binary float 278 | * `Str(length?=None, type='c', encoding='utf-8', terminator?=None, exact=False, length_unit=1, length_type=UInt(8)):` string, supported types are `raw`, `c` and `pascal` 279 | * `Bool(type=UInt(8), true_value=1, false_value=0):` generic boolean 280 | 281 | --- 282 | 283 | * `Nothing:` parses nothing and emits nothing, returns `None` 284 | * `Implied(value):` parses nothing and emits nothing, returns `value` 285 | * `Fixed(value):` reads bytes and emits bytes, making sure they equate `value` 286 | * `Pad(size, value?=b'\x00'):` parses and emits padding bytes, returns `None` 287 | * `Data(size?=None):` parses and returns raw bytes 288 | * `Enum(enum, type):` parses and emits `type` and constructs `enum.Enum` subclass `enum` with its result 289 | 290 | --- 291 | 292 | * `StructType(fields, cls, generics=[], union=False, partial=False, bound=[]):` type class used by `MetaStruct` 293 | * `Struct:` base class for automatic struct type generation through `MetaStruct` meta-class and field annotations 294 | * `Union:` base class for automatic union type generation through `MetaStruct` meta-class and field annotations 295 | * `Arr(type, count=None, size=None, stop_value=None):` parses and emits array of `types`, of optionally max `count` elements and `size` bytes total size 296 | * `Switch(default=None, fallback=None, options={}):` parses and emits a choice of types chosen through the `selector` field 297 | 298 | --- 299 | 300 | * `Ref(type, point, reference=os.SEEK_SET, adjustment=0, stream=None)`: parses and emits a value of `type` elsewhere in the stream at offset `point` 301 | * `Rebased(type, base=None):` parses and emits `type`, shifting the input position for absolute `Ref` references to `base` (default: file position on parse/emit time) 302 | * `Sized(type, limit=None, exact=False):` parses and emits `type`, limiting its size in the tream to `limit` bytes 303 | * `AlignTo(type, alignment, value?=b'\x00'):` parses and emits `type`, aligning stream to alignment bytes **after** 304 | * `AlignedTo(type, alignment, value?=b'\x00'):` parse and emits `type`, aligning stream to alignment bytes **before** 305 | * `Lazy(type, size):` parses and emits `type` lazily, returning a callable that will parse and return the type when 306 | * `Processed(type, parse, emit, with_context=False):` parses and emits `type`, processing it through `parse` and `emit` callables, respectively, optionally passing `context` 307 | * `Checked(type, check):` parses and emits `type`, calling `check(value)` and erroring if it returns `False` 308 | * `Mapped(type, mapping, default?=None):` parses and emits `type`, looking up the result in `mapping` 309 | 310 | License 311 | ======= 312 | 313 | BSD-2; see `LICENSE` for details. 314 | 315 | TODO 316 | ==== 317 | 318 | * Add `Maybe` and `Either` types 319 | * Fix `Arr` EOF-handling masking errors 320 | * Port more features over from `destruct` 321 | * More examples 322 | -------------------------------------------------------------------------------- /restruct.py: -------------------------------------------------------------------------------- 1 | import os 2 | from io import BytesIO 3 | import errno 4 | import math 5 | import types 6 | import struct 7 | import enum 8 | import collections 9 | import itertools 10 | from contextlib import contextmanager 11 | 12 | from typing import Generic as G, Union as U, TypeVar, Any, Callable, Sequence, Mapping, Optional as O, Tuple as TU 13 | 14 | 15 | ## Helpers 16 | 17 | def indent(s: str, count: int, start: bool = False) -> str: 18 | """ Indent all lines of a string. """ 19 | lines = s.splitlines() 20 | for i in range(0 if start else 1, len(lines)): 21 | lines[i] = ' ' * count + lines[i] 22 | return '\n'.join(lines) 23 | 24 | def stretch(b: bytes, count: int) -> bytes: 25 | b *= count // len(b) 26 | b += b[:count - len(b)] 27 | return b 28 | 29 | def format_value(value: Any, formatter: Callable[[Any], str], indentation: int = 0) -> str: 30 | """ Format containers to use the given formatter function instead of always repr(). """ 31 | if isinstance(value, (dict, collections.Mapping)): 32 | if value: 33 | fmt = '{{\n{}\n}}' 34 | values = [indent(',\n'.join('{}: {}'.format( 35 | format_value(k, formatter), 36 | format_value(v, formatter) 37 | ) for k, v in value.items()), 2, True)] 38 | else: 39 | fmt = '{{}}' 40 | values = [] 41 | elif isinstance(value, (list, set, frozenset)): 42 | l = len(value) 43 | is_set = isinstance(value, (set, frozenset)) 44 | if l > 3: 45 | fmt = '{{\n{}\n}}' if is_set else '[\n{}\n]' 46 | values = [indent(',\n'.join(format_value(v, formatter) for v in value), 2, True)] 47 | elif l > 0: 48 | fmt = '{{{}}}' if is_set else '[{}]' 49 | values = [', '.join(format_value(v, formatter) for v in value)] 50 | else: 51 | fmt = '{{}}' if is_set else '[]' 52 | values = [] 53 | elif isinstance(value, (bytes, bytearray)): 54 | fmt = '{}' 55 | values = [format_bytes(value)] 56 | else: 57 | fmt = '{}' 58 | values = [formatter(value)] 59 | return indent(fmt.format(*values), indentation) 60 | 61 | def format_bytes(bs: bytes) -> str: 62 | return '[' + ' '.join(hex(b)[2:].zfill(2) for b in bs) + ']' 63 | 64 | def format_path(path: Sequence[str]) -> str: 65 | s = '' 66 | first = True 67 | for p in path: 68 | sep = '.' 69 | if isinstance(p, int): 70 | p = '[' + str(p) + ']' 71 | sep = '' 72 | if sep and not first: 73 | s += sep 74 | s += p 75 | first = False 76 | return s 77 | 78 | def class_name(s: Any, module_whitelist: Sequence[str] = {'__main__', 'builtins', __name__}) -> str: 79 | if not isinstance(s, type): 80 | s = s.__class__ 81 | module = s.__module__ 82 | name = s.__qualname__ 83 | if module in module_whitelist: 84 | return name 85 | return module + '.' + name 86 | 87 | def friendly_name(s: Any) -> str: 88 | if hasattr(s, '__name__'): 89 | return s.__name__ 90 | return str(s) 91 | 92 | def process_sizes(s: Mapping[str, int], cb: Callable[[int, int], int]) -> Mapping[str, int]: 93 | sizes = {} 94 | for prev in s: 95 | for k, n in prev.items(): 96 | p = sizes.get(k, 0) 97 | if p is None or n is None: 98 | sizes[k] = None 99 | else: 100 | sizes[k] = cb(p, n) 101 | return sizes 102 | 103 | def min_sizes(*s: Mapping[str, int]) -> Mapping[str, int]: 104 | return process_sizes(s, min) 105 | 106 | def max_sizes(*s: Mapping[str, int]) -> Mapping[str, int]: 107 | return process_sizes(s, max) 108 | 109 | def add_sizes(*s: Mapping[str, int]) -> Mapping[str, int]: 110 | return process_sizes(s, lambda a, b: a + b) 111 | 112 | def ceil_sizes(s: Mapping[str, U[int, float]]) -> Mapping[str, int]: 113 | d = {} 114 | for k, v in s.copy().items(): 115 | if v is not None: 116 | d[k] = math.ceil(v) 117 | return d 118 | 119 | 120 | @contextmanager 121 | def seeking(fd: 'IO', pos: int, whence: int = os.SEEK_SET) -> None: 122 | oldpos = fd.tell() 123 | fd.seek(pos, whence) 124 | try: 125 | yield fd 126 | finally: 127 | fd.seek(oldpos, os.SEEK_SET) 128 | 129 | 130 | ## Bases 131 | 132 | class BitAlignment(enum.Enum): 133 | No = enum.auto() 134 | Fill = enum.auto() 135 | Yes = enum.auto() 136 | 137 | class IO: 138 | __slots__ = ('handle', 'bit_count', 'bit_val', 'bit_align', 'bit_wr') 139 | 140 | def __init__(self, handle, bit_align = BitAlignment.No) -> None: 141 | self.handle = handle 142 | self.bit_count = 8 143 | self.bit_val = None 144 | self.bit_align = bit_align 145 | self.bit_wr = False 146 | 147 | def get_bits(self, n: int) -> TU[int, int]: 148 | if n <= 0: 149 | return 0, 0 150 | 151 | if self.bit_count == 8: 152 | self.bit_val = self.handle.read(1)[0] 153 | self.bit_count = 0 154 | nb = min(n, 8 - self.bit_count) 155 | 156 | val = (self.bit_val >> self.bit_count) & ((1 << nb) - 1) 157 | self.bit_count += nb 158 | 159 | if nb > 0 and self.bit_count == 8 and self.bit_wr: 160 | self.flush_bits() 161 | return n - nb, val 162 | 163 | def put_bits(self, val: int, n: int) -> TU[int, int]: 164 | if n <= 0: 165 | return 0, 0 166 | 167 | if self.bit_count == 8: 168 | bv = self.handle.read(1) 169 | if bv: 170 | self.bit_val = bv[0] 171 | else: 172 | self.bit_val = 0 173 | self.handle.write(bytes([0])) 174 | self.bit_count = 0 175 | nb = min(n, 8 - self.bit_count) 176 | 177 | self.bit_val &= ~(((1 << nb) - 1) << self.bit_count) 178 | self.bit_val |= ((val >> (n - nb)) & ((1 << nb) - 1)) << self.bit_count 179 | self.bit_wr = True 180 | self.bit_count += nb 181 | 182 | if nb > 0 and self.bit_count == 8: 183 | self.flush_bits() 184 | return n - nb, val & ((1 << (n - nb)) - 1) 185 | 186 | def flush_bits(self): 187 | self.handle.seek(-1, os.SEEK_CUR) 188 | self.handle.write(bytes([self.bit_val])) 189 | self.bit_val = None 190 | self.bit_wr = False 191 | self.bit_count = 8 192 | 193 | def read(self, n: int = -1, *, bits: bool = False) -> U[bytes, int]: 194 | if bits: 195 | nl, v = self.get_bits(n) 196 | val = v << nl 197 | if nl >= 8: 198 | rounds = nl // 8 199 | nl -= 8 * rounds 200 | val |= int.from_bytes(self.handle.read(rounds), byteorder='big') << nl 201 | if nl > 0: 202 | _, v = self.get_bits(nl) 203 | val |= v 204 | return val 205 | if self.bit_count < 8: 206 | if self.bit_align == BitAlignment.No: 207 | raise ValueError('misaligned read') 208 | elif self.bit_align == BitAlignment.Yes: 209 | return self.read(n * 8, bits=True) 210 | elif self.bit_align == BitAlignment.Fill: 211 | self.bit_count = 8 212 | self.bit_val = None 213 | return self.handle.read(n) 214 | 215 | def write(self, b: U[bytes, int], *, bits: O[int] = None) -> None: 216 | if bits is not None: 217 | n, b = self.put_bits(b, bits) 218 | if n > 8: 219 | rounds = n // 8 220 | shift = 8 * rounds 221 | self.handle.write(((b >> (n - shift)) & ((1 << shift) - 1)).to_bytes(rounds, byteorder='big')) 222 | b &= (1 << (n - shift)) - 1 223 | n -= shift 224 | if n > 0: 225 | self.put_bits(b, n) 226 | return 227 | if self.bit_count < 8: 228 | if self.bit_align == BitAlignment.No: 229 | raise ValueError('misaligned write') 230 | elif self.bit_align == BitAlignment.Yes: 231 | return self.write(int.from_bytes(b, byteorder='big'), bits=len(b) * 8) 232 | elif self.bit_align == BitAlignment.Fill: 233 | self.flush_bits() 234 | return self.handle.write(b) 235 | 236 | def flush(self): 237 | if self.bit_align == BitAlignment.Fill and self.bit_count < 8: 238 | self.flush_bits() 239 | return self.handle.flush() 240 | 241 | def seek(self, n: int, whence: int = os.SEEK_SET) -> None: 242 | if self.bit_count < 8: 243 | self.flush_bits() 244 | 245 | if isinstance(n, float): 246 | bp = int((n % 1) * 8) 247 | n = int(n) 248 | else: 249 | bp = 0 250 | self.handle.seek(n, whence) 251 | if bp: 252 | self.get_bits(8 - bp) 253 | 254 | def tell(self) -> int: 255 | p = self.handle.tell() 256 | if self.bit_count < 8: 257 | p -= 1 258 | p += self.bit_count / 8 259 | return p 260 | 261 | @property 262 | def root(self): 263 | handle = self.handle 264 | while hasattr(handle, 'root'): 265 | handle = handle.root 266 | return handle 267 | 268 | @contextmanager 269 | def wrapped(self, handle): 270 | self.flush() 271 | old = self.handle 272 | self.handle = handle 273 | yield self 274 | self.handle = old 275 | 276 | class Stream: 277 | __slots__ = ('name', 'offset', 'dependents', 'pos') 278 | 279 | def __init__(self, name: str, dependents: Sequence['Stream'] = None) -> None: 280 | self.name = name 281 | self.offset = None 282 | self.dependents = dependents or [] 283 | self.pos = None 284 | 285 | def reset(self): 286 | self.offset = self.pos = None 287 | 288 | class Params: 289 | __slots__ = ('streams', 'default_stream', 'user') 290 | 291 | def __init__(self, streams: Sequence[Stream] = None): 292 | default = streams[0] if streams else Stream('default') 293 | self.streams = {s.name: s for s in (streams or [default, Stream('refs', [default])])} 294 | self.default_stream = default 295 | self.user = types.SimpleNamespace() 296 | 297 | def reset(self): 298 | for s in self.streams.values(): 299 | s.reset() 300 | 301 | class Context: 302 | __slots__ = ('root', 'value', 'path', 'stream_path', 'user', 'params') 303 | 304 | def __init__(self, root: 'Type', value: O[Any] = None, params: O[Params] = None) -> None: 305 | self.root = root 306 | self.value = value 307 | self.path = [] 308 | self.stream_path = [] 309 | self.params = params or Params() 310 | self.params.reset() 311 | self.user = self.params.user 312 | 313 | @contextmanager 314 | def enter(self, name: str, type: 'Type') -> None: 315 | self.path.append((name, type)) 316 | yield 317 | self.path.pop() 318 | 319 | @contextmanager 320 | def enter_stream(self, stream: str, io: O[IO] = None, pos: O[int] = None, reference = os.SEEK_SET) -> None: 321 | stream = self.params.streams[stream] 322 | if io: 323 | if pos is None: 324 | if stream.offset is None: 325 | stream.offset = self.stream_offset(stream) 326 | stream.pos = stream.offset 327 | pos = stream.pos 328 | with seeking(io.root, pos, reference) as s, io.wrapped(s) as f: 329 | self.stream_path.append(stream) 330 | yield f 331 | self.stream_path.pop() 332 | stream.pos = f.tell() 333 | else: 334 | self.stream_path.append(stream) 335 | yield io 336 | self.stream_path.pop() 337 | 338 | def stream_offset(self, stream: Stream) -> int: 339 | size = 0 340 | for s in stream.dependents: 341 | size += self.stream_offset(s) + self.stream_size(s) 342 | return size 343 | 344 | def stream_size(self, stream: Stream) -> O[int]: 345 | return sizeof(self.root, self.value, self, stream=stream.name) 346 | 347 | def format_path(self) -> str: 348 | return format_path(name for name, parser in self.path) 349 | 350 | class Error(Exception): 351 | __slots__ = ('path', 'stream_path') 352 | 353 | def __init__(self, context: Context, exception: Exception) -> None: 354 | path = context.format_path() 355 | if not isinstance(exception, Exception): 356 | exception = ValueError(exception) 357 | 358 | super().__init__('{}{}: {}'.format( 359 | ('[' + path + '] ') if path else '', class_name(exception), str(exception) 360 | )) 361 | self.exception = exception 362 | self.path = context.path.copy() 363 | self.stream_path = context.stream_path.copy() 364 | 365 | class Type: 366 | __slots__ = () 367 | 368 | def parse(self, io: IO, context: Context) -> Any: 369 | raise NotImplemented 370 | 371 | def emit(self, value: Any, io: IO, context: Context) -> None: 372 | raise NotImplemented 373 | 374 | def sizeof(self, value: O[Any], context: Context) -> O[int]: 375 | return None 376 | 377 | def offsetof(self, path: Sequence[U[int, str]], value: O[Any], context: Context) -> O[int]: 378 | return None 379 | 380 | def default(self, context: Context) -> O[Any]: 381 | return None 382 | 383 | 384 | ## Type helpers 385 | 386 | T = TypeVar('T', bound=Type) 387 | T2 = TypeVar('T2', bound=Type) 388 | 389 | 390 | ## Base types 391 | 392 | class Nothing(Type): 393 | __slots__ = () 394 | 395 | def parse(self, io: IO, context: Context) -> None: 396 | return None 397 | 398 | def emit(self, value: None, io: IO, context: Context) -> None: 399 | pass 400 | 401 | def sizeof(self, value: O[None], context: Context) -> O[int]: 402 | return 0 403 | 404 | def default(self, context: Context) -> None: 405 | return None 406 | 407 | def __repr__(self) -> str: 408 | return class_name(self) 409 | 410 | class Implied(Type): 411 | __slots__ = ('value',) 412 | 413 | def __init__(self, value: Any = None) -> None: 414 | self.value = value 415 | 416 | def parse(self, io: IO, context: Context) -> Any: 417 | return get_value(self.value, context) 418 | 419 | def emit(self, value: Any, io: IO, context: Context) -> None: 420 | pass 421 | 422 | def sizeof(self, value: O[Any], context: Context) -> int: 423 | return 0 424 | 425 | def default(self, context: Context) -> Any: 426 | return peek_value(self.value, context) 427 | 428 | def __repr__(self) -> str: 429 | return '+{!r}'.format(self.value) 430 | 431 | class Ignored(Type, G[T]): 432 | __slots__ = ('type', 'value') 433 | 434 | def __init__(self, type: T, value: O[Any] = None) -> None: 435 | self.type = type 436 | self.value = value 437 | 438 | def parse(self, io: IO, context: Context) -> None: 439 | parse(self.type, io, context) 440 | return None 441 | 442 | def emit(self, value: None, io: IO, context: Context) -> None: 443 | value = get_value(self.value, context) 444 | if value is None: 445 | value = default(self.type, context) 446 | return emit(self.type, value, io, context) 447 | 448 | def sizeof(self, value: None, context: Context) -> int: 449 | value = peek_value(self.value, context) 450 | if value is None: 451 | value = default(self.type, context) 452 | return _sizeof(self.type, value, context) 453 | 454 | def offsetof(self, path: Sequence[U[int, str]], value: O[None], context: Context) -> O[int]: 455 | value = peek_value(self.value, context) 456 | if value is None: 457 | value = default(self.type, context) 458 | return _offsetof(self.type, path, value, context) 459 | 460 | def default(self, context: Context) -> Any: 461 | return None 462 | 463 | def __repr__(self) -> str: 464 | return '-{!r}{}'.format( 465 | class_name(self).strip('<>'), 466 | '(' + repr(self.value) + ')' if self.value is not None else '', 467 | ) 468 | 469 | class Bits(Type): 470 | __slots__ = ('size', 'byteswap') 471 | 472 | def __init__(self, size: O[int] = None, byteswap: bool = False) -> None: 473 | self.size = size 474 | self.byteswap = byteswap 475 | 476 | def parse(self, io: IO, context: Context) -> int: 477 | size = get_value(self.size, context) 478 | value = io.read(size, bits=True) 479 | if get_value(self.byteswap, context): 480 | value <<= (8 - (size % 8)) % 8 481 | v = value.to_bytes(math.ceil(size / 8), byteorder='big') 482 | value = int.from_bytes(v, byteorder='little') 483 | return value 484 | 485 | def emit(self, value: int, io: IO, context: Context) -> None: 486 | size = get_value(self.size, context) 487 | if get_value(self.byteswap, context): 488 | value = int.from_bytes(value.to_bytes(math.ceil(size / 8), byteorder='big'), byteorder='little') 489 | value >>= (8 - (size % 8)) % 8 490 | io.write(value, bits=size) 491 | 492 | def sizeof(self, value: O[int], context: Context) -> O[int]: 493 | return peek_value(self.size, context) / 8 494 | 495 | def default(self, context: Context) -> int: 496 | return 0 497 | 498 | def __repr__(self) -> str: 499 | return '<{}{}>'.format( 500 | class_name(self), 501 | ('[' + str(self.size) + ']') if self.size is not None else '', 502 | ) 503 | 504 | class Data(Type): 505 | __slots__ = ('size',) 506 | 507 | def __init__(self, size: O[int] = None) -> None: 508 | self.size = size 509 | 510 | def parse(self, io: IO, context: Context) -> bytes: 511 | size = get_value(self.size, context) 512 | if size is None: 513 | size = -1 514 | data = io.read(size) 515 | if size >= 0 and len(data) != size: 516 | raise Error(context, 'Size mismatch!\n wanted: {} bytes\n found: {} bytes'.format( 517 | size, len(data) 518 | )) 519 | return data 520 | 521 | def emit(self, value: bytes, io: IO, context: Context) -> None: 522 | set_value(self.size, len(value), io, context) 523 | io.write(value) 524 | 525 | def sizeof(self, value: O[bytes], context: Context) -> O[int]: 526 | if value is not None: 527 | return len(value) 528 | return peek_value(self.size, context) 529 | 530 | def default(self, context: Context) -> bytes: 531 | return b'\x00' * (peek_value(self.size, context) or 0) 532 | 533 | def __repr__(self) -> str: 534 | return '<{}{}>'.format( 535 | class_name(self), 536 | ('[' + str(self.size) + ']') if self.size is not None else '', 537 | ) 538 | 539 | class Pad(Type): 540 | def __new__(self, size: int, value: bytes) -> Ignored[Data]: 541 | return Ignored(Data(size), value) 542 | 543 | class Fixed(Type, G[T]): 544 | __slots__ = ('pattern', 'type') 545 | 546 | def __init__(self, pattern: Any, type: O[T] = None) -> None: 547 | self.pattern = pattern 548 | self.type = type 549 | 550 | def parse(self, io: IO, context: Context) -> bytes: 551 | pattern = get_value(self.pattern, context) 552 | type = to_type(self.type or Data(len(pattern))) 553 | data = parse(type, io, context) 554 | if data != pattern: 555 | raise Error(context, 'Value mismatch!\n wanted: {}\n found: {}'.format( 556 | format_value(pattern, repr), format_value(data, repr), 557 | )) 558 | return data 559 | 560 | def emit(self, value: Any, io: IO, context: Context) -> None: 561 | set_value(self.pattern, value, io, context) 562 | type = to_type(self.type or Data(len(value))) 563 | return emit(type, value, io, context) 564 | 565 | def sizeof(self, value: O[Any], context: Context) -> O[int]: 566 | if value is None: 567 | value = peek_value(self.pattern, context) 568 | if self.type is None: 569 | if value is None: 570 | return None 571 | type = Data(len(value)) 572 | else: 573 | type = to_type(self.type) 574 | return _sizeof(type, value, context) 575 | 576 | def default(self, context: Context) -> Any: 577 | return peek_value(self.pattern, context) 578 | 579 | def __repr__(self) -> str: 580 | if self.type: 581 | type = to_type(self.type) 582 | return repr(type) + '!' + repr(self.pattern) 583 | else: 584 | return '!' + repr(self.pattern)[1:] 585 | 586 | 587 | E_co = TypeVar('E_co', bound=enum.Enum) 588 | 589 | class Enum(Type, G[E_co, T]): 590 | __slots__ = ('type', 'cls', 'exhaustive') 591 | 592 | def __init__(self, cls: E_co, type: T, exhaustive: bool = True) -> None: 593 | self.type = type 594 | self.cls = cls 595 | self.exhaustive = exhaustive 596 | 597 | def parse(self, io: IO, context: Context) -> U[E_co, T]: 598 | value = parse(self.type, io, context) 599 | try: 600 | return self.cls(value) 601 | except ValueError: 602 | if self.exhaustive: 603 | raise 604 | return value 605 | 606 | def emit(self, value: U[E_co, T], io: IO, context: Context) -> None: 607 | if isinstance(value, self.cls): 608 | value = value.value 609 | return emit(self.type, value, io, context) 610 | 611 | def sizeof(self, value: O[U[E_co, T]], context: Context) -> O[int]: 612 | if value is not None and isinstance(value, self.cls): 613 | value = value.value 614 | return _sizeof(self.type, value, context) 615 | 616 | def offsetof(self, path: Sequence[U[int, str]], value: O[U[E_co, T]], context: Context) -> O[int]: 617 | if value is not None and isinstance(value, self.cls): 618 | value = value.value 619 | return _offsetof(self.type, path, value, context) 620 | 621 | def default(self, context: Context) -> U[E_co, T]: 622 | return next(iter(self.cls.__members__.values())) 623 | 624 | def __repr__(self) -> str: 625 | return '<{}({}): {}>'.format(class_name(self), class_name(self.cls), repr(to_type(self.type)).strip('<>')) 626 | 627 | 628 | 629 | ## Modifier types 630 | 631 | class PartialAttr(Type, G[T]): 632 | 633 | def __init__(self, parent: 'Partial', name: str) -> None: 634 | self._parent = parent 635 | self._name = name 636 | self._type = None 637 | self._values = [] 638 | self._pvalues = [] 639 | 640 | def parse(self, io: IO, context: Context) -> T: 641 | offset = io.tell() 642 | value = parse(self._type, io, context) 643 | for _ in self._parent.types: 644 | self._values.append((offset, value)) 645 | return value 646 | 647 | def emit(self, value: T, io: IO, context: Context) -> None: 648 | offset = io.tell() 649 | for _ in self._parent.types: 650 | self._values.append((offset, value)) 651 | emit(self._type, value, io, context) 652 | 653 | def sizeof(self, value: O[T], context: Context) -> O[int]: 654 | for _ in self._parent.types: 655 | self._pvalues.append(value) 656 | return _sizeof(self._type, value, context) 657 | 658 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 659 | for _ in self._parent.types: 660 | self._pvalues.append(value) 661 | return _offsetof(self.type, path, value, context) 662 | 663 | def default(self, context: Context) -> T: 664 | value = default(self._type, context) 665 | for _ in self._parent.types: 666 | self._pvalues.append(value) 667 | return value 668 | 669 | def get_value(self, context: Context, peek: bool = False) -> T: 670 | if peek: 671 | _, value = self._values[-1] 672 | else: 673 | _, value = self._values.pop() 674 | return value 675 | 676 | def peek_value(self, context: Context, default=None) -> T: 677 | if self._pvalues: 678 | return self._pvalues.pop() 679 | if self._values: 680 | _, value = self._values[-1] 681 | return value 682 | return default 683 | 684 | def set_value(self, value: T, io: IO, context: Context) -> None: 685 | offset, _ = self._values.pop() 686 | with seeking(io, offset, os.SEEK_SET) as f: 687 | emit(self._type, value, f, context) 688 | 689 | def __matmul__(self, type: Any) -> Type: 690 | if isinstance(type, self.__class__): 691 | return type.__rmatmul__(self) 692 | return NotImplemented 693 | 694 | def __rmatmul__(self, type: Any) -> Type: 695 | if isinstance(type, Type): 696 | return self(type) 697 | return NotImplemented 698 | 699 | def __setattr__(self, n, v): 700 | if not n.startswith('_'): 701 | return setattr(self._type, n, v) 702 | return super().__setattr__(n, v) 703 | 704 | def __call__(self, type: Type) -> Type: 705 | self._type = type 706 | return self 707 | 708 | def __repr__(self) -> str: 709 | return ''.format(self._name, repr(self._type).strip('<>')) 710 | 711 | class Partial: 712 | __slots__ = ('types', 'attrs') 713 | 714 | def __init__(self): 715 | self.types = [] 716 | self.attrs = {} 717 | 718 | def __getattr__(self, name: str) -> PartialAttr: 719 | self.attrs[name] = PartialAttr(self, name) 720 | return self.attrs[name] 721 | 722 | def __matmul__(self, type: Any) -> Type: 723 | if isinstance(type, Type): 724 | return self(type) 725 | return NotImplemented 726 | 727 | def __rmatmul__(self, type: Any) -> Type: 728 | if isinstance(type, Type): 729 | return self(type) 730 | return NotImplemented 731 | 732 | def __call__(self, type: Type) -> Type: 733 | type = to_type(type) 734 | for n, v in self.attrs.items(): 735 | setattr(type, n, v) 736 | self.types.append(type) 737 | return type 738 | 739 | class Ref(Type, G[T]): 740 | __slots__ = ('type', 'point', 'reference', 'adjustment', 'stream') 741 | 742 | def __init__(self, type: Type, point: O[int] = None, reference: int = os.SEEK_SET, adjustment: U[int, Stream] = 0, stream: O[Stream] = None) -> None: 743 | self.type = type 744 | self.point = point 745 | self.reference = reference 746 | self.adjustment = adjustment 747 | self.stream = stream.name if stream else 'refs' 748 | 749 | if self.reference not in (os.SEEK_SET, os.SEEK_CUR, os.SEEK_END): 750 | raise ValueError('current reference must be any of [os.SEEK_SET, os.SEEK_CUR, os.SEEK_END]') 751 | 752 | def parse(self, io: IO, context: Context) -> T: 753 | reference = get_value(self.reference, context) 754 | adjustment = get_value(self.adjustment, context) 755 | if isinstance(adjustment, Stream): 756 | start = context.stream_offset(adjustment) 757 | if reference == os.SEEK_SET: 758 | adjustment = start 759 | elif reference == os.SEEK_CUR: 760 | adjustment = adjustment.pos 761 | elif reference == os.SEEK_END: 762 | adjustment = start + context.stream_size(adjustment) 763 | reference = os.SEEK_SET 764 | point = get_value(self.point, context) + adjustment 765 | with context.enter_stream(self.stream, io, point, reference) as f: 766 | return parse(self.type, f, context) 767 | 768 | def emit(self, value: T, io: IO, context: Context) -> None: 769 | reference = get_value(self.reference, context) 770 | adjustment = get_value(self.adjustment, context) 771 | if isinstance(adjustment, Stream): 772 | start = context.stream_offset(adjustment) 773 | if reference == os.SEEK_SET: 774 | adjustment = start 775 | elif reference == os.SEEK_CUR: 776 | adjustment = adjustment.pos 777 | elif reference == os.SEEK_END: 778 | adjustment = start + context.stream_size(adjustment) 779 | reference = os.SEEK_SET 780 | 781 | with context.enter_stream(self.stream, io) as f: 782 | point = f.tell() 783 | emit(self.type, value, f, context) 784 | 785 | if reference == os.SEEK_CUR: 786 | point -= io.tell() 787 | elif reference == os.SEEK_END: 788 | with seeking(io, 0, SEEK_END) as f: 789 | point -= f.tell() 790 | 791 | set_value(self.point, point - adjustment, io, context) 792 | 793 | def sizeof(self, value: O[T], context: Context) -> O[int]: 794 | with context.enter_stream(self.stream): 795 | return _sizeof(self.type, value, context) 796 | 797 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 798 | with context.enter_stream(self.stream): 799 | return _offsetof(self.type, path, value, context) 800 | 801 | def default(self, context: Context) -> T: 802 | return default(self.type, context) 803 | 804 | def __repr__(self) -> str: 805 | return '<&{} @ {}{!r}{})>'.format( 806 | repr(to_type(self.type)).strip('<>'), 807 | {os.SEEK_SET: '', os.SEEK_CUR: '+', os.SEEK_END: '-'}[self.reference], 808 | self.point, ('[' + repr(self.adjustment) + ']' if self.adjustment else ''), 809 | ) 810 | 811 | class RebasedFile: 812 | def __init__(self, file: IO, start: O[int] = None) -> None: 813 | self._file = file 814 | self._start = start or file.tell() 815 | 816 | def tell(self) -> int: 817 | return self._file.tell() - self._start 818 | 819 | def seek(self, offset: int, whence: int = os.SEEK_SET) -> None: 820 | if whence == os.SEEK_SET: 821 | offset += self._start 822 | return self._file.seek(offset, whence) 823 | 824 | def __getattr__(self, n: str) -> Any: 825 | return getattr(self._file, n) 826 | 827 | class Rebased(Type, G[T]): 828 | __slots__ = ('type', 'base') 829 | 830 | def __init__(self, type: Type, base: O[int] = None): 831 | self.type = type 832 | self.base = base 833 | 834 | def parse(self, io: IO, context: Context) -> T: 835 | base = get_value(self.base, context) 836 | with io.wrapped(RebasedFile(io.handle, base)) as f: 837 | return parse(self.type, f, context) 838 | 839 | def emit(self, value: T, io: IO, context: Context) -> None: 840 | base = get_value(self.base, context) 841 | with io.wrapped(RebasedFile(io.handle, base)) as f: 842 | return emit(self.type, value, f, context) 843 | 844 | def sizeof(self, value: O[T], context: Context) -> O[int]: 845 | return _sizeof(self.type, value, context) 846 | 847 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 848 | return _offsetof(self.type, path, value, context) 849 | 850 | def default(self, context: Context) -> T: 851 | return default(self.type, context) 852 | 853 | def __repr__(self) -> str: 854 | return repr(self.type) + ' @ ' + self.base 855 | 856 | class SizedFile: 857 | def __init__(self, file: IO, limit: int, exact: bool = False) -> None: 858 | self.root = self._file = file 859 | self._pos = 0 860 | self._limit = limit 861 | self._start = file.tell() 862 | 863 | def read(self, n: int = -1) -> bytes: 864 | remaining = max(0, self._limit - self._pos) 865 | if n < 0: 866 | n = remaining 867 | n = min(n, remaining) 868 | self._pos += n 869 | return self._file.read(n) 870 | 871 | def write(self, data: bytes) -> None: 872 | remaining = self._limit - self._pos 873 | if len(data) > remaining: 874 | raise ValueError('trying to write past limit by {} bytes'.format(len(data) - remaining)) 875 | self._pos += len(data) 876 | return self._file.write(data) 877 | 878 | def seek(self, offset: int, whence: int) -> None: 879 | if whence == os.SEEK_SET: 880 | pos = offset 881 | elif whence == os.SEEK_CUR: 882 | pos = self._start + self._pos + offset 883 | elif whence == os.SEEK_END: 884 | pos = self._start + self._limit - offset 885 | if pos < self._start: 886 | raise OSError(errno.EINVAL, os.strerror(errno.EINVAL), offset) 887 | self._pos = pos - self._start 888 | return self._file.seek(pos, os.SEEK_SET) 889 | 890 | def tell(self) -> int: 891 | return self._start + self._pos 892 | 893 | def __getattr__(self, n: str) -> Any: 894 | return getattr(self._file, n) 895 | 896 | class Sized(Type, G[T]): 897 | __slots__ = ('type', 'limit', 'exact', 'exact_write') 898 | 899 | def __init__(self, type: Type, limit: O[int] = None, exact: bool = False, exact_write: bool = False) -> None: 900 | self.type = type 901 | self.limit = limit 902 | self.exact = exact 903 | self.exact_write = exact_write 904 | 905 | def parse(self, io: IO, context: Context) -> T: 906 | exact = get_value(self.exact, context) 907 | limit = max(0, get_value(self.limit, context)) 908 | 909 | start = io.tell() 910 | with io.wrapped(SizedFile(io.handle, limit, exact)) as f: 911 | value = parse(self.type, f, context) 912 | 913 | if exact: 914 | io.seek(start + limit, os.SEEK_SET) 915 | return value 916 | 917 | def emit(self, value: T, io: IO, context: Context) -> None: 918 | exact_write = get_value(self.exact_write, context) 919 | limit = max(0, get_value(self.limit, context, peek=True)) 920 | 921 | start = io.tell() 922 | handle = SizedFile(io.handle, limit, True) if exact_write else io.handle 923 | with io.wrapped(handle) as f: 924 | ret = emit(self.type, value, f, context) 925 | 926 | if exact_write: 927 | io.seek(start + limit, os.SEEK_SET) 928 | else: 929 | limit = io.tell() - start 930 | set_value(self.limit, limit, io, context) 931 | return ret 932 | 933 | def sizeof(self, value: O[T], context: Context) -> O[int]: 934 | limit = peek_value(self.limit, context) 935 | if self.exact: 936 | return limit 937 | size = _sizeof(self.type, value, context) 938 | if size is None: 939 | return limit 940 | if limit is None: 941 | return size 942 | return min_sizes(*[size, to_size(limit, context)]) 943 | 944 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 945 | return _offsetof(self.type, path, value, context) 946 | 947 | def default(self, context: Context) -> T: 948 | return default(self.type, context) 949 | 950 | def __repr__(self) -> str: 951 | return '<{}: {!r} (limit={})>'.format(class_name(self), to_type(self.type), self.limit) 952 | 953 | class AlignTo(Type, G[T]): 954 | __slots__ = ('type', 'alignment', 'value') 955 | 956 | def __init__(self, type: T, alignment: int, value: bytes = b'\x00') -> None: 957 | self.type = type 958 | self.alignment = alignment 959 | self.value = value 960 | 961 | def parse(self, io: IO, context: Context) -> T: 962 | value = parse(self.type, io, context) 963 | adjustment = io.tell() % self.alignment 964 | if adjustment: 965 | io.seek(self.alignment - adjustment, os.SEEK_CUR) 966 | return value 967 | 968 | def emit(self, value: T, io: IO, context: Context) -> None: 969 | emit(self.type, value, io, context) 970 | adjustment = io.tell() % self.alignment 971 | if adjustment: 972 | io.write(self.value * (self.alignment - adjustment)) 973 | 974 | def sizeof(self, value: O[T], context: Context) -> O[int]: 975 | return None # TODO 976 | 977 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 978 | return _offsetof(self.type, path, value, context) 979 | 980 | def default(self, context: Context) -> T: 981 | return default(self.type, context) 982 | 983 | def __repr__(self) -> str: 984 | return '<{}: {!r} (n={})>'.format(class_name(self), to_type(self.type), self.alignment) 985 | 986 | class AlignedTo(Type, G[T]): 987 | __slots__ = ('child', 'alignment', 'value') 988 | 989 | def __init__(self, child: T, alignment: int, value: bytes = b'\x00') -> None: 990 | self.child = child 991 | self.alignment = alignment 992 | self.value = value 993 | 994 | def parse(self, io: IO, context: Context) -> T: 995 | adjustment = io.tell() % self.alignment 996 | if adjustment: 997 | io.seek(self.alignment - adjustment, os.SEEK_CUR) 998 | value = parse(self.child, io, context) 999 | return value 1000 | 1001 | def emit(self, value: T, io: IO, context: Context) -> None: 1002 | adjustment = io.tell() % self.alignment 1003 | if adjustment: 1004 | io.write(self.value * (self.alignment - adjustment)) 1005 | emit(self.child, value, io, context) 1006 | 1007 | def sizeof(self, value: O[T], context: Context) -> O[int]: 1008 | return None # TODO 1009 | 1010 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 1011 | return None # TODO 1012 | 1013 | def default(self, context: Context) -> T: 1014 | return default(self.type, context) 1015 | 1016 | def __repr__(self) -> str: 1017 | return '<{}: {!r} (n={})>'.format(class_name(self), self.child, self.alignment) 1018 | 1019 | class LazyEntry(G[T]): 1020 | __slots__ = ('type', 'io', 'pos', 'context', 'parsed') 1021 | 1022 | def __init__(self, type: T, io: IO, context: Context) -> None: 1023 | self.type = type 1024 | self.io = io 1025 | self.pos = self.io.tell() 1026 | self.context = context 1027 | self.parsed: O[T] = None 1028 | 1029 | def __call__(self) -> Any: 1030 | if self.parsed is None: 1031 | with seeking(self.io, self.pos) as f: 1032 | self.parsed = parse(self.type, f, self.context) 1033 | return self.parsed 1034 | 1035 | def __str__(self) -> str: 1036 | return '~~{}'.format(to_type(self.type)) 1037 | 1038 | def __repr__(self) -> str: 1039 | return '<{}: {!r}>'.format(class_name(self), to_type(self.type)) 1040 | 1041 | class Lazy(Type, G[T]): 1042 | __slots__ = ('type', 'size') 1043 | 1044 | def __init__(self, type: T, size: O[int] = None) -> None: 1045 | self.type = type 1046 | self.size = size 1047 | 1048 | def parse(self, io: IO, context: Context) -> T: 1049 | size = self.sizeof(None, context)[context.stream_path[-1].name] 1050 | if size is None: 1051 | raise ValueError('lazy type size must be known at parse-time') 1052 | entry = LazyEntry(to_type(self.type), io, context) 1053 | io.seek(size, os.SEEK_CUR) 1054 | return entry 1055 | 1056 | def emit(self, value: O[T], io: IO, context: Context) -> None: 1057 | emit(self.type, value(), io, context) 1058 | 1059 | def sizeof(self, value: O[T], context: Context) -> O[int]: 1060 | length = peek_value(self.size, context) 1061 | if length is not None: 1062 | return length 1063 | if value is not None: 1064 | value = value() 1065 | return _sizeof(self.type, value, context) 1066 | 1067 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 1068 | if value is not None: 1069 | value = value() 1070 | return _offsetof(self.type, path, value, context) 1071 | 1072 | def default(self, context: Context) -> T: 1073 | return default(self.type, context) 1074 | 1075 | def __str__(self) -> str: 1076 | return '~{}'.format(to_type(self.type)) 1077 | 1078 | def __repr__(self) -> str: 1079 | return '<{}: {!r}>'.format(class_name(self), to_type(self.type)) 1080 | 1081 | class Processed(Type, G[T, T2]): 1082 | __slots__ = ('type', 'do_parse', 'do_emit', 'with_context') 1083 | 1084 | def __init__(self, type: T, parse: Callable[[T], T2], emit: Callable[[T2], T], with_context: bool = False) -> None: 1085 | self.type = type 1086 | self.do_parse = parse 1087 | self.do_emit = emit 1088 | self.with_context = with_context 1089 | 1090 | def parse(self, io: IO, context: Context) -> T2: 1091 | value = parse(self.type, io, context) 1092 | if get_value(self.with_context, context): 1093 | return self.do_parse(value, context) 1094 | else: 1095 | return self.do_parse(value) 1096 | 1097 | def emit(self, value: T2, io: IO, context: Context) -> None: 1098 | if get_value(self.with_context, context): 1099 | raw = self.do_emit(value, context) 1100 | else: 1101 | raw = self.do_emit(value) 1102 | emit(self.type, raw, io, context) 1103 | 1104 | def sizeof(self, value: O[T2], context: Context) -> O[int]: 1105 | if value is not None: 1106 | if peek_value(self.with_context, context): 1107 | value = self.do_emit(value, context) 1108 | else: 1109 | value = self.do_emit(value) 1110 | return _sizeof(self.type, value, context) 1111 | 1112 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 1113 | if value is not None: 1114 | if peek_value(self.with_context, context): 1115 | value = self.do_emit(value, context) 1116 | else: 1117 | value = self.do_emit(value) 1118 | return _offsetof(self.type, path, value, context) 1119 | 1120 | def default(self, context: Context) -> T: 1121 | value = default(self.type, context) 1122 | if peek_value(self.with_context, context): 1123 | return self.do_parse(value, context) 1124 | else: 1125 | return self.do_parse(value) 1126 | 1127 | def __repr__(self) -> str: 1128 | return '<λ{}{!r} ->{} <-{}>'.format( 1129 | '+' if self.with_context else '', 1130 | to_type(self.type), self.do_parse.__name__, self.do_emit.__name__ 1131 | ) 1132 | 1133 | class Checked(Type, G[T]): 1134 | __slots__ = ('type', 'check', 'message') 1135 | 1136 | def __init__(self, type: T, check: Callable[[T], bool], message: O[str] = None) -> None: 1137 | self.type = type 1138 | self.check = check 1139 | self.message = message or 'check failed' 1140 | 1141 | def parse(self, io: IO, context: Context) -> T: 1142 | check = get_value(self.check, context) 1143 | value = parse(self.type, io, context) 1144 | if not check(value): 1145 | raise Error(context, self.message.format(value)) 1146 | return value 1147 | 1148 | def emit(self, value: T, io: IO, context: Context) -> None: 1149 | check = get_value(self.check, context) 1150 | if not check(value): 1151 | raise Error(context, self.message.format(value)) 1152 | return emit(self.type, value, io, context) 1153 | 1154 | def sizeof(self, value: O[T], context: Context) -> O[int]: 1155 | check = peek_value(self.check, context) 1156 | if value is not None and not check(value): 1157 | raise Error(context, self.message.format(value)) 1158 | return _sizeof(self.type, value, context) 1159 | 1160 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 1161 | check = peek_value(self.check, context) 1162 | if value is not None and not check(value): 1163 | raise Error(context, self.message.format(value)) 1164 | return _offsetof(self.type, path, value, context) 1165 | 1166 | def default(self, context: Context) -> T: 1167 | return default(self.type, context) 1168 | 1169 | def __repr__(self) -> str: 1170 | return ''.format(self.check, repr(self.type).strip('<>')) 1171 | 1172 | class Mapped(Type, G[T]): 1173 | def __new__(self, type: T, mapping: Mapping[T, T2], default: O[Any] = None) -> Processed: 1174 | reverse = {v: k for k, v in mapping.items()} 1175 | if default is not None: 1176 | mapping = collections.defaultdict(lambda: default, mapping) 1177 | reverse = collections.defaultdict(lambda: default, reverse) 1178 | return Processed(type, mapping.__getitem__, reverse.__getitem__) 1179 | 1180 | 1181 | ## Compound types 1182 | 1183 | class Generic(Type): 1184 | __slots__ = ('stack',) 1185 | 1186 | def __init__(self) -> None: 1187 | self.stack = [] 1188 | 1189 | def resolve(self, value) -> None: 1190 | if isinstance(value, Generic): 1191 | self.stack.append(value.stack[-1]) 1192 | else: 1193 | self.stack.append(value) 1194 | 1195 | def pop(self) -> None: 1196 | self.stack.pop() 1197 | 1198 | def __get_restruct_type__(self, ident: Any) -> Type: 1199 | return to_type(self.stack[-1]) 1200 | 1201 | def parse(self, io: IO, context: Context) -> Any: 1202 | if not self.stack: 1203 | raise Error(context, 'unresolved generic') 1204 | return parse(self.stack[-1], io, context) 1205 | 1206 | def emit(self, value: O[Any], io: IO, context: Context) -> None: 1207 | if not self.stack: 1208 | raise Error(context, 'unresolved generic') 1209 | return emit(self.stack[-1], value, io, context) 1210 | 1211 | def sizeof(self, value: O[Any], context: Context) -> O[int]: 1212 | if not self.stack: 1213 | return None 1214 | return _sizeof(self.stack[-1], value, context) 1215 | 1216 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 1217 | if not self.stack: 1218 | return None 1219 | return _offsetof(self.type, path, self.stack[-1], context) 1220 | 1221 | def default(self, context: Context) -> Any: 1222 | if not self.stack: 1223 | raise Error(context, 'unresolved generic') 1224 | return default(self.stack[-1], context) 1225 | 1226 | def get_value(self, context: Context, peek: bool = False) -> Any: 1227 | return self.stack[-1] 1228 | 1229 | def peek_value(self, context: Context) -> Any: 1230 | return self.stack[-1] 1231 | 1232 | def __repr__(self) -> str: 1233 | if self.stack: 1234 | return '<${}>'.format(repr(to_type(self.stack[-1])).strip('<>')) 1235 | return '<$unresolved>' 1236 | 1237 | def __deepcopy__(self, memo: Any) -> Any: 1238 | return self 1239 | 1240 | class GenericSelf(Generic): 1241 | def __repr__(self) -> str: 1242 | return ''.format((': ' + friendly_name(self.stack[-1])) if self.stack else '') 1243 | 1244 | class MetaSpec(collections.OrderedDict): 1245 | def __getattr__(self, item: Any) -> Any: 1246 | try: 1247 | return self[item] 1248 | except KeyError: 1249 | raise AttributeError(item) 1250 | 1251 | def __setattr__(self, item: Any, value: Any) -> Any: 1252 | if '__' in item: 1253 | super().__setattr__(item, value) 1254 | else: 1255 | self[item] = value 1256 | 1257 | class StructType(Type): 1258 | __slots__ = ('fields', 'cls', 'generics', 'union', 'partial', 'bound') 1259 | 1260 | def __init__(self, fields: Mapping[str, Type], cls: type, generics: Sequence[Generic] = [], union: bool = False, partial: bool = False, bound: Sequence[Any] = []) -> None: 1261 | self.fields = MetaSpec(fields) 1262 | self.cls = cls 1263 | self.generics = generics 1264 | self.union = union 1265 | self.partial = partial 1266 | self.bound = bound or [] 1267 | 1268 | def __getitem__(self, ty) -> Any: 1269 | if not isinstance(ty, tuple): 1270 | ty = (ty,) 1271 | 1272 | bound = self.bound[:] 1273 | bound.extend(ty) 1274 | if len(bound) > len(self.generics): 1275 | raise TypeError('too many generics arguments for {}: {}'.format( 1276 | self.__class__.__name__, len(bound) 1277 | )) 1278 | 1279 | subtype = self.__class__(self.fields, self.cls, self.generics, self.union, self.partial, bound=bound) 1280 | for i, b in enumerate(subtype.bound): 1281 | # re-bind Self bound 1282 | if b is self: 1283 | subtype.bound[i] = subtype 1284 | return subtype 1285 | 1286 | def parse(self, io: IO, context: Context) -> Any: 1287 | n = 0 1288 | pos = io.tell() 1289 | 1290 | c = self.cls.__new__(self.cls) 1291 | try: 1292 | with self.enter(): 1293 | for name, type in self.fields.items(): 1294 | with context.enter(name, type): 1295 | if type is None: 1296 | continue 1297 | if self.union: 1298 | io.seek(pos, os.SEEK_SET) 1299 | 1300 | val = parse(type, io, context) 1301 | 1302 | nbytes = io.tell() - pos 1303 | if self.union: 1304 | n = max(n, nbytes) 1305 | else: 1306 | n = nbytes 1307 | 1308 | setattr(c, name, val) 1309 | hook = 'on_parse_' + name 1310 | if hasattr(c, hook): 1311 | getattr(c, hook)(self.fields, context) 1312 | except Exception: 1313 | # Check EOF and allow if partial. 1314 | b = io.read(1) 1315 | if not self.partial or b: 1316 | if b: 1317 | io.seek(-1, os.SEEK_CUR) 1318 | raise 1319 | # allow EOF if partial 1320 | 1321 | io.seek(pos + n, os.SEEK_SET) 1322 | return c 1323 | 1324 | def emit(self, value: Any, io: IO, context: Context) -> None: 1325 | n = 0 1326 | pos = io.tell() 1327 | 1328 | with self.enter(): 1329 | for name, type in self.fields.items(): 1330 | with context.enter(name, type): 1331 | if self.union: 1332 | io.seek(pos, os.SEEK_SET) 1333 | 1334 | hook = 'on_emit_' + name 1335 | if hasattr(value, hook): 1336 | getattr(value, hook)(self.fields, context) 1337 | 1338 | field = getattr(value, name) 1339 | emit(type, field, io, context) 1340 | 1341 | nbytes = io.tell() - pos 1342 | if self.union: 1343 | n = max(n, nbytes) 1344 | else: 1345 | n = nbytes 1346 | 1347 | io.seek(pos + n, os.SEEK_SET) 1348 | 1349 | def sizeof(self, value: O[Any], context: Context) -> O[int]: 1350 | n = {} 1351 | 1352 | with self.enter(): 1353 | for name, type in self.fields.items(): 1354 | with context.enter(name, type): 1355 | if value: 1356 | field = getattr(value, name) 1357 | else: 1358 | field = None 1359 | 1360 | nbytes = _sizeof(type, field, context) 1361 | if self.union: 1362 | n = max_sizes(n, nbytes) 1363 | else: 1364 | n = add_sizes(n, nbytes) 1365 | 1366 | n = ceil_sizes(n) 1367 | return n 1368 | 1369 | def offsetof(self, path: Sequence[U[int, str]], value: O[Any], context: Context) -> O[int]: 1370 | n = {} 1371 | 1372 | fname, fpath = path[0], path[1:] 1373 | if fname not in self.fields: 1374 | raise ValueError('unknown field: {}'.format(fname)) 1375 | 1376 | with self.enter(): 1377 | for name, type in self.fields.items(): 1378 | with context.enter(name, type): 1379 | if value: 1380 | field = getattr(value, name) 1381 | else: 1382 | field = None 1383 | 1384 | if fname == name: 1385 | offset = _offsetof(type, fpath, field, context) 1386 | n = add_sizes(n, offset) 1387 | break 1388 | 1389 | if not self.union: 1390 | nbytes = _sizeof(type, field, context) 1391 | n = add_sizes(n, nbytes) 1392 | 1393 | return n 1394 | 1395 | @contextmanager 1396 | def enter(self): 1397 | for g, child in zip(self.generics, self.bound): 1398 | g.resolve(child) 1399 | yield 1400 | for g in self.generics: 1401 | g.pop() 1402 | 1403 | def default(self, context: Context) -> Any: 1404 | with self.enter(): 1405 | c = self.cls.__new__(self.cls) 1406 | for name, type in self.fields.items(): 1407 | with context.enter(name, type): 1408 | setattr(c, name, default(type, context)) 1409 | 1410 | return c 1411 | 1412 | def __str__(self) -> str: 1413 | return class_name(self.cls) 1414 | 1415 | def __repr__(self) -> str: 1416 | type = 'Union' if self.union else 'Struct' 1417 | if self.fields: 1418 | with self.enter(): 1419 | fields = '{\n' 1420 | for f, v in self.fields.items(): 1421 | fields += ' ' + f + ': ' + indent(format_value(to_type(v), repr), 2) + ',\n' 1422 | fields += '}' 1423 | else: 1424 | fields = '{}' 1425 | return '<{}({}) {}>'.format(type, class_name(self.cls), fields) 1426 | 1427 | class MetaStruct(type): 1428 | @classmethod 1429 | def __prepare__(mcls, name: str, bases: Sequence[Any], generics: Sequence[str] = [], partials: Sequence[str] = [], inject: bool = True, recursive: bool = False, **kwargs) -> dict: 1430 | attrs = collections.OrderedDict() 1431 | attrs.update({g: Generic() for g in generics}) 1432 | attrs.update({p: Partial() for p in partials}) 1433 | if inject: 1434 | attrs.update({c.__name__: c for c in __all_types__}) 1435 | if recursive: 1436 | attrs['Self'] = GenericSelf() 1437 | return attrs 1438 | 1439 | def __new__(mcls, name: str, bases: Sequence[Any], attrs: Mapping[str, Any], generics: Sequence[str] = [], partials: Sequence[str] = [], inject: bool = True, recursive: bool = False, **kwargs) -> Any: 1440 | # Inherit some properties from base types 1441 | gs = [] 1442 | bound = [] 1443 | if recursive: 1444 | gs.append(attrs.pop('Self')) 1445 | 1446 | fields = {} 1447 | for b in bases: 1448 | fields.update(getattr(b, '__annotations__', {})) 1449 | type = b.__restruct_type__ 1450 | gs.extend(type.generics) 1451 | bound.extend(type.bound) 1452 | if type.union: 1453 | kwargs['union'] = True 1454 | 1455 | for p in partials: 1456 | del attrs[p] 1457 | for g in generics: 1458 | gs.append(attrs.pop(g)) 1459 | if inject: 1460 | for c in __all_types__: 1461 | del attrs[c.__name__] 1462 | fields.update(attrs.get('__annotations__', {})) 1463 | 1464 | attrs['__slots__'] = attrs.get('__slots__', ()) + tuple(fields) 1465 | 1466 | c = super().__new__(mcls, name, bases, attrs) 1467 | type = StructType(fields, c, gs, bound=bound, **kwargs) 1468 | if recursive: 1469 | type.bound.insert(0, type) 1470 | c.__restruct_type__ = type 1471 | return c 1472 | 1473 | def __init__(cls, *args, **kwargs) -> Any: 1474 | return super().__init__(*args) 1475 | 1476 | def __getitem__(cls, ty) -> Any: 1477 | if not isinstance(ty, tuple): 1478 | ty = (ty,) 1479 | subtype = cls.__restruct_type__[ty] 1480 | new_name = '{}[{}]'.format(cls.__name__, ', '.join(friendly_name(r).strip('<>') for r in subtype.bound if r is not subtype)) 1481 | new = type(new_name, (cls,), cls.__class__.__prepare__(new_name, (cls,))) 1482 | new.__restruct_type__ = subtype 1483 | new.__slots__ = cls.__slots__ 1484 | new.__module__ = cls.__module__ 1485 | subtype.cls = new 1486 | return new 1487 | 1488 | def __repr__(cls) -> str: 1489 | return '<{}: {}>'.format( 1490 | 'union' if cls.__restruct_type__.union else 'struct', 1491 | class_name(cls), 1492 | ) 1493 | 1494 | class Struct(metaclass=MetaStruct, inject=False): 1495 | def __init__(self, **kwargs) -> None: 1496 | super().__init__() 1497 | 1498 | st = to_type(self) 1499 | with st.enter(): 1500 | for k, t in st.fields.items(): 1501 | if k not in kwargs: 1502 | v = default(t) 1503 | else: 1504 | v = kwargs.pop(k) 1505 | setattr(self, k, v) 1506 | 1507 | if kwargs: 1508 | raise TypeError('unrecognized fields: {}'.format(', '.join(kwargs))) 1509 | 1510 | def __iter__(self) -> Sequence[Any]: 1511 | return iter(self.__slots__) 1512 | 1513 | def __hash__(self) -> int: 1514 | return hash(tuple((k, getattr(self, k)) for k in self)) 1515 | 1516 | def __eq__(self, other) -> bool: 1517 | if type(self) != type(other): 1518 | return False 1519 | if self.__slots__ != other.__slots__: 1520 | return False 1521 | for k in self: 1522 | ov = getattr(self, k) 1523 | tv = getattr(other, k) 1524 | if ov != tv: 1525 | return False 1526 | return True 1527 | 1528 | def __fmt__(self, fieldfunc: Callable[[Any], str]) -> str: 1529 | args = [] 1530 | for k in self: 1531 | if k.startswith('_'): 1532 | continue 1533 | val = getattr(self, k) 1534 | val = format_value(val, fieldfunc, 2) 1535 | args.append(' {}: {}'.format(k, val)) 1536 | args = ',\n'.join(args) 1537 | # Format final value. 1538 | if args: 1539 | return '{} {{\n{}\n}}'.format(class_name(self), args) 1540 | else: 1541 | return '{} {{}}'.format(class_name(self)) 1542 | 1543 | def __str__(self) -> str: 1544 | return self.__fmt__(str) 1545 | 1546 | def __repr__(self) -> str: 1547 | return self.__fmt__(repr) 1548 | 1549 | class Union(Struct, metaclass=MetaStruct, union=True, inject=False): 1550 | def __setattr__(self, name, value) -> None: 1551 | super().__setattr__(name, value) 1552 | 1553 | io = BytesIO() 1554 | t = to_type(self) 1555 | try: 1556 | emit(t.fields[name], value, io=io) 1557 | except: 1558 | return 1559 | 1560 | for fname, ftype in t.fields.items(): 1561 | if fname == name: 1562 | continue 1563 | io.seek(0) 1564 | try: 1565 | fvalue = parse(ftype, io) 1566 | super().__setattr__(fname, fvalue) 1567 | except: 1568 | pass 1569 | 1570 | class Tuple(Type): 1571 | __slots__ = ('types',) 1572 | 1573 | def __init__(self, types: Sequence[Type]) -> None: 1574 | self.types = types 1575 | 1576 | def parse(self, io: IO, context: Context) -> Sequence[Any]: 1577 | value = [] 1578 | for i, type in enumerate(self.types): 1579 | type = to_type(type, i) 1580 | with context.enter(i, type): 1581 | value.append(parse(type, io, context)) 1582 | return tuple(value) 1583 | 1584 | def emit(self, value: Sequence[Any], io: IO, context: Context) -> None: 1585 | for i, (type, val) in enumerate(zip(self.types, value)): 1586 | type = to_type(type, i) 1587 | with context.enter(i, type): 1588 | emit(type, val, io, context) 1589 | 1590 | def sizeof(self, value: O[Sequence[Any]], context: Context) -> O[int]: 1591 | l = [] 1592 | if value is None: 1593 | value = [None] * len(self.types) 1594 | 1595 | for i, (type, val) in enumerate(zip(self.types, value)): 1596 | type = to_type(type, i) 1597 | with context.enter(i, type): 1598 | size = _sizeof(type, val, context) 1599 | l.append(size) 1600 | 1601 | return ceil_sizes(add_sizes(*l)) 1602 | 1603 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 1604 | idx, fpath = path[0], path[1:] 1605 | if not isinstance(idx, int): 1606 | raise ValueError('not a sequence index: {}'.format(idx)) 1607 | if idx >= len(self.types): 1608 | raise ValueError('sequence index out of bounds: {}'.format(idx)) 1609 | 1610 | l = [] 1611 | if value is None: 1612 | value = [None] * len(self.types) 1613 | 1614 | for i, (type, val) in enumerate(zip(self.types, value)): 1615 | type = to_type(type, i) 1616 | with context.enter(i, type): 1617 | if i == idx: 1618 | offset = _offsetof(type, fpath, val, context) 1619 | l.append(offset) 1620 | break 1621 | else: 1622 | size = _sizeof(type, val, context) 1623 | l.append(size) 1624 | 1625 | return ceil_sizes(add_sizes(*l)) 1626 | 1627 | def default(self, context: Context) -> Sequence[Any]: 1628 | value = [] 1629 | for i, type in enumerate(self.types): 1630 | type = to_type(type, i) 1631 | with context.enter(i, type): 1632 | value.append(default(type, context)) 1633 | return tuple(value) 1634 | 1635 | def __repr__(self) -> str: 1636 | return '<(' + ', '.join(repr(to_type(t)) for t in self.types) + ')>' 1637 | 1638 | 1639 | class Any(Type): 1640 | __slots__ = ('types') 1641 | 1642 | def __init__(self, types: Sequence[Type]) -> None: 1643 | self.types = types 1644 | 1645 | def parse(self, io: IO, context: Context) -> Any: 1646 | errors = [] 1647 | types = [] 1648 | start = io.tell() 1649 | 1650 | for i, type in enumerate(self.types): 1651 | io.seek(start, os.SEEK_SET) 1652 | type = to_type(type, i) 1653 | with context.enter(i, type): 1654 | try: 1655 | return parse(type, io, context) 1656 | except Exception as e: 1657 | if isinstance(e, Error): 1658 | e = e.exception 1659 | types.append(type) 1660 | errors.append(e) 1661 | 1662 | raise Error(context, 'Failed to parse using any of the following:\n' + '\n'.join( 1663 | ' - {!r} => {}: {}'.format(t, class_name(e), indent(str(e), 2)) 1664 | for (t, e) in zip(types, errors) 1665 | )) 1666 | 1667 | def emit(self, value: Any, io: IO, context: Context) -> None: 1668 | errors = [] 1669 | types = [] 1670 | start = io.tell() 1671 | 1672 | for i, type in enumerate(self.types): 1673 | io.seek(start, os.SEEK_SET) 1674 | type = to_type(type, i) 1675 | with context.enter(i, type): 1676 | try: 1677 | return emit(type, val, io, context) 1678 | except Exception as e: 1679 | types.append(type) 1680 | errors.append(e) 1681 | 1682 | raise Error(context, 'Failed to emit using any of the following:\n' + '\n'.join( 1683 | ' - {!r} => {}: {}'.format(t, class_name(e), indent(str(e), 2)) 1684 | for (t, e) in zip(types, errors) 1685 | )) 1686 | 1687 | def sizeof(self, value: O[Any], context: Context) -> O[int]: 1688 | return None 1689 | 1690 | def offsetof(self, path: Sequence[U[int, str]], value: O[Any], context: Context) -> O[int]: 1691 | return None 1692 | 1693 | def default(self, context: Context) -> O[Any]: 1694 | return None 1695 | 1696 | def __str__(self) -> str: 1697 | return 'Any[' + ', '.join(format_value(to_type(t, i), str) for i, t in enumerate(self.types)) + ']' 1698 | 1699 | def __repr__(self) -> str: 1700 | return '' 1701 | 1702 | 1703 | class Arr(Type, G[T]): 1704 | __slots__ = ('type', 'count', 'stop_value', 'separator') 1705 | 1706 | def __init__(self, type: T, count: O[int] = None, stop_value: O[Any] = None, separator: O[bytes] = None) -> None: 1707 | self.type = type 1708 | self.count = count 1709 | self.stop_value = stop_value 1710 | self.separator = separator 1711 | 1712 | def parse(self, io: IO, context: Context) -> Sequence[T]: 1713 | value = [] 1714 | 1715 | count = get_value(self.count, context) 1716 | stop_value = get_value(self.stop_value, context) 1717 | separator = get_value(self.separator, context) 1718 | 1719 | i = 0 1720 | while count is None or i < count: 1721 | if isinstance(self.type, list): 1722 | type = to_type(self.type[i], i) 1723 | else: 1724 | type = to_type(self.type, i) 1725 | 1726 | with context.enter(i, type): 1727 | eof = False 1728 | pos = io.tell() 1729 | if separator: 1730 | data = b'' 1731 | while True: 1732 | b = io.read(1) 1733 | if not b: 1734 | eof = True 1735 | break 1736 | data += b 1737 | if data[-len(separator):] == separator: 1738 | data = data[:-len(separator)] 1739 | break 1740 | eio = data 1741 | else: 1742 | eio = io 1743 | try: 1744 | elem = parse(type, eio, context) 1745 | except Exception: 1746 | # Check EOF. 1747 | if not eio or (io.tell() == pos and not io.read(1)): 1748 | break 1749 | io.seek(-1, os.SEEK_CUR) 1750 | raise 1751 | 1752 | if eof or elem == stop_value: 1753 | break 1754 | 1755 | value.append(elem) 1756 | i += 1 1757 | 1758 | return value 1759 | 1760 | def emit(self, value: Sequence[T], io: IO, context: Context) -> None: 1761 | set_value(self.count, len(value), io, context) 1762 | stop_value = get_value(self.stop_value, context) 1763 | separator = get_value(self.separator, context) 1764 | 1765 | if stop_value is not None: 1766 | value = value + [stop_value] 1767 | 1768 | start = io.tell() 1769 | for i, elem in enumerate(value): 1770 | if isinstance(self.type, list): 1771 | type = to_type(self.type[i], i) 1772 | else: 1773 | type = to_type(self.type, i) 1774 | 1775 | with context.enter(i, type): 1776 | emit(type, elem, io, context) 1777 | if separator and i < len(value) - 1: 1778 | io.write(separator) 1779 | 1780 | def sizeof(self, value: O[Sequence[T]], context: Context) -> int: 1781 | if value is None: 1782 | count = peek_value(self.count, context) 1783 | else: 1784 | count = len(value) 1785 | stop_value = peek_value(self.stop_value, context) 1786 | separator = peek_value(self.separator, context) 1787 | 1788 | if count is None: 1789 | return None 1790 | 1791 | l = [] 1792 | for i in range(count): 1793 | if isinstance(self.type, list): 1794 | type = to_type(self.type[i], i) 1795 | else: 1796 | type = to_type(self.type, i) 1797 | if value is not None: 1798 | val = value[i] 1799 | else: 1800 | val = None 1801 | size = _sizeof(type, val, context) 1802 | l.append(size) 1803 | 1804 | if stop_value is not None: 1805 | if isinstance(self.type, list): 1806 | type = to_type(self.type[count], count) 1807 | else: 1808 | type = to_type(self.type, count) 1809 | size = _sizeof(type, stop_value, context) 1810 | l.append(size) 1811 | 1812 | if separator: 1813 | l.append(to_size((count - 1) * len(separator), context)) 1814 | 1815 | return ceil_sizes(add_sizes(*l)) 1816 | 1817 | def offsetof(self, path: Sequence[U[int, str]], value: O[T], context: Context) -> O[int]: 1818 | idx, fpath = path[0], path[1:] 1819 | if not isinstance(idx, int): 1820 | raise ValueError('not a sequence index: {}'.format(idx)) 1821 | 1822 | if value is None: 1823 | count = peek_value(self.count, context) 1824 | else: 1825 | count = len(value) 1826 | stop_value = peek_value(self.stop_value, context) 1827 | separator = peek_value(self.separator, context) 1828 | 1829 | if count is not None and idx >= count: 1830 | raise ValueError('sequence index out of bounds: {}'.format(idx)) 1831 | 1832 | l = [] 1833 | for i in range(idx + 1): 1834 | if isinstance(self.type, list): 1835 | type = to_type(self.type[i], i) 1836 | else: 1837 | type = to_type(self.type, i) 1838 | if value is not None: 1839 | val = value[i] 1840 | else: 1841 | val = None 1842 | if idx == i: 1843 | offset = _offsetof(type, fpath, val, context) 1844 | l.append(offset) 1845 | break 1846 | else: 1847 | size = _sizeof(type, val, context) 1848 | l.append(size) 1849 | 1850 | if separator: 1851 | l.append(to_size((idx - 1) * len(separator), context)) 1852 | 1853 | return ceil_sizes(add_sizes(*l)) 1854 | 1855 | def default(self, context: Context) -> Sequence[T]: 1856 | return [] 1857 | 1858 | def __str__(self) -> str: 1859 | return str(to_type(self.type)) + (('[' + str(self.count) + ']') if self.count is not None else '[]') 1860 | 1861 | def __repr__(self) -> str: 1862 | return '<{}({!r}{}{})>'.format( 1863 | class_name(self), to_type(self.type), 1864 | ('[' + str(self.count) + ']') if self.count is not None else '', 1865 | (', stop: ' + repr(self.stop_value)) if self.stop_value is not None else '', 1866 | ) 1867 | 1868 | class Switch(Type): 1869 | __slots__ = ('options', 'selector', 'default_key', 'fallback') 1870 | 1871 | def __init__(self, default: O[Any] = None, fallback: O[T] = None, options: Mapping[Any, T] = None) -> None: 1872 | self.options = options or {} 1873 | self.selector = None 1874 | self.default_key = default 1875 | self.fallback = fallback 1876 | 1877 | def _get(self, sel) -> T: 1878 | if sel not in self.options and not self.fallback: 1879 | raise ValueError('Selector {} is invalid! [options: {}]'.format( 1880 | sel, ', '.join(repr(x) for x in self.options.keys()) 1881 | )) 1882 | if sel is not None and sel in self.options: 1883 | return self.options[sel] 1884 | else: 1885 | return self.fallback 1886 | 1887 | def peek_value(self, context: Context) -> O[T]: 1888 | selector = self.selector 1889 | if selector is not None: 1890 | selector = peek_value(self.selector, context, self.default_key) 1891 | if selector is not None: 1892 | return self._get(selector) 1893 | else: 1894 | return self.fallback 1895 | 1896 | def get_value(self, context: Context) -> T: 1897 | if self.selector is not None: 1898 | return self._get(get_value(self.selector, context)) 1899 | elif self.default_key is not None: 1900 | return self._get(self.default_key) 1901 | elif self.fallback is None: 1902 | raise ValueError('Selector not set!') 1903 | else: 1904 | return self.fallback 1905 | 1906 | def parse(self, io: IO, context: Context) -> Any: 1907 | return parse(self.get_value(context), io, context) 1908 | 1909 | def emit(self, value: Any, io: IO, context: Context) -> None: 1910 | return emit(self.get_value(context), value, io, context) 1911 | 1912 | def sizeof(self, value: O[Any], context: Context) -> O[int]: 1913 | type = self.peek_value(context) 1914 | if type is None: 1915 | return None 1916 | return _sizeof(type, value, context) 1917 | 1918 | def offsetof(self, path: Sequence[U[int, str]], value: O[Any], context: Context) -> O[int]: 1919 | type = self.peek_value(context) 1920 | if type is None: 1921 | return None 1922 | return _offsetof(type, path, value, context) 1923 | 1924 | def default(self, context: Context) -> O[Any]: 1925 | type = self.peek_value(context) 1926 | if type is None: 1927 | return None 1928 | return default(type, context) 1929 | 1930 | def __repr__(self) -> str: 1931 | return '<{}: {}>'.format(class_name(self), ', '.join(repr(k) + ': ' + repr(v) for k, v in self.options.items())) 1932 | 1933 | 1934 | ## Primitive types 1935 | 1936 | class Int(Type): 1937 | __slots__ = ('bits', 'signed', 'order') 1938 | 1939 | def __init__(self, bits: O[int] = None, order: str = 'le', signed: bool = True) -> None: 1940 | self.bits = bits 1941 | self.signed = signed 1942 | self.order = order 1943 | 1944 | def parse(self, io: IO, context: Context) -> int: 1945 | bits = get_value(self.bits, context) 1946 | order = get_value(self.order, context) 1947 | signed = get_value(self.signed, context) 1948 | if bits is not None: 1949 | bs = io.read(bits // 8) 1950 | if len(bs) != bits // 8: 1951 | raise ValueError('short read') 1952 | else: 1953 | bs = io.read() 1954 | return int.from_bytes(bs, byteorder='little' if order == 'le' else 'big', signed=signed) 1955 | 1956 | def emit(self, value: int, io: IO, context: Context) -> None: 1957 | bits = get_value(self.bits, context) 1958 | order = get_value(self.order, context) 1959 | signed = get_value(self.signed, context) 1960 | if bits is None: 1961 | bits = 8 * math.ceil(value.bit_length() // 8) 1962 | bs = value.to_bytes(bits // 8, byteorder='little' if order == 'le' else 'big', signed=signed) 1963 | io.write(bs) 1964 | 1965 | def sizeof(self, value: O[int], context: Context) -> int: 1966 | bits = peek_value(self.bits, context) 1967 | if bits is None: 1968 | return None 1969 | return bits // 8 1970 | 1971 | def default(self, context: Context) -> int: 1972 | return 0 1973 | 1974 | def __repr__(self) -> str: 1975 | return '<{}{}({}, {})>'.format( 1976 | 'U' if not self.signed else '', 1977 | class_name(self), self.bits, self.order 1978 | ) 1979 | 1980 | class UInt(Type): 1981 | def __new__(cls, *args, **kwargs) -> Int: 1982 | return Int(*args, signed=False, **kwargs) 1983 | 1984 | class Bool(Type, G[T]): 1985 | def __new__(self, type: T = UInt(8), true_value: T = 1, false_value: T = 0) -> Mapped: 1986 | return Mapped(type, {true_value: True, false_value: False}) 1987 | 1988 | class Float(Type): 1989 | __slots__ = ('bits',) 1990 | 1991 | FORMATS = { 1992 | 32: 'f', 1993 | 64: 'd', 1994 | } 1995 | 1996 | def __init__(self, bits: int = 32) -> None: 1997 | self.bits = bits 1998 | if self.bits not in self.FORMATS: 1999 | raise ValueError('unsupported bit count for float: {}'.format(bits)) 2000 | 2001 | def parse(self, io: IO, context: Context) -> float: 2002 | bits = get_value(self.bits, context) 2003 | bs = io.read(bits // 8) 2004 | return struct.unpack(self.FORMATS[bits], bs)[0] 2005 | 2006 | def emit(self, value: float, io: IO, context: Context) -> None: 2007 | bits = get_value(self.bits, context) 2008 | bs = struct.pack(self.FORMATS[bits], value) 2009 | io.write(bs) 2010 | 2011 | def sizeof(self, value: O[float], context: Context) -> int: 2012 | bits = peek_value(self.bits, context) 2013 | if bits is None: 2014 | return None 2015 | return bits // 8 2016 | 2017 | def default(self, context: Context) -> float: 2018 | return 0.0 2019 | 2020 | def __repr__(self) -> str: 2021 | return '<{}({})>'.format(class_name(self), self.bits) 2022 | 2023 | class Str(Type): 2024 | __slots__ = ('length', 'type', 'encoding', 'terminator', 'exact', 'length_type', 'length_unit') 2025 | 2026 | def __init__(self, length: O[int] = None, type: str = 'c', encoding: str = 'utf-8', terminator: O[bytes] = None, exact: bool = False, length_unit: int = 1, length_type: Type = UInt(8)) -> None: 2027 | self.length = length 2028 | self.type = type 2029 | self.encoding = encoding 2030 | self.terminator = terminator or b'\x00' * length_unit 2031 | self.exact = exact 2032 | self.length_unit = length_unit 2033 | self.length_type = length_type 2034 | 2035 | if self.type not in ('raw', 'c', 'pascal'): 2036 | raise ValueError('string type must be any of [raw, c, pascal]') 2037 | 2038 | def parse(self, io: IO, context: Context) -> str: 2039 | length = get_value(self.length, context) 2040 | length_unit = get_value(self.length_unit, context) 2041 | type = get_value(self.type, context) 2042 | exact = get_value(self.exact, context) 2043 | encoding = get_value(self.encoding, context) 2044 | terminator = get_value(self.terminator, context) 2045 | 2046 | if type == 'pascal': 2047 | read_length = parse(self.length_type, io, context) 2048 | if length is not None: 2049 | read_length = min(read_length, length) 2050 | raw = io.read(read_length * length_unit) 2051 | elif type in ('raw', 'c'): 2052 | read_length = 0 2053 | raw = bytearray() 2054 | for i in itertools.count(start=1): 2055 | if length is not None and i > length: 2056 | break 2057 | c = io.read(length_unit) 2058 | read_length += 1 2059 | if not c: 2060 | if type == 'raw': 2061 | break 2062 | raise ValueError('expected terminator') 2063 | if type == 'c' and c == terminator: 2064 | break 2065 | raw.extend(c) 2066 | 2067 | if exact and length is not None: 2068 | if read_length > length: 2069 | raise ValueError('exact length specified but read length ({}) > given length ({})'.format(read_length, length)) 2070 | left = length - read_length 2071 | if exact and left: 2072 | io.read(left * length_unit) 2073 | 2074 | return raw.decode(encoding) 2075 | 2076 | def emit(self, value: str, io: IO, context: Context) -> None: 2077 | length = get_value(self.length, context) 2078 | length_unit = get_value(self.length_unit, context) 2079 | type = get_value(self.type, context) 2080 | exact = get_value(self.exact, context) 2081 | encoding = get_value(self.encoding, context) 2082 | terminator = get_value(self.terminator, context) 2083 | 2084 | raw = value.encode(encoding) 2085 | 2086 | write_length = (len(value) + (len(terminator) if type == 'c' else 0)) // length_unit 2087 | if type == 'pascal': 2088 | emit(self.length_type, write_length, io, context) 2089 | io.write(raw) 2090 | elif type in ('c', 'raw'): 2091 | io.write(raw) 2092 | if type == 'c': 2093 | io.write(terminator) 2094 | 2095 | if length is not None: 2096 | if write_length > length: 2097 | raise ValueError('exact length specified but write length ({}) > given length ({})'.format(write_length, length)) 2098 | left = length - write_length 2099 | if exact and left: 2100 | io.write(b'\x00' * (left * length_unit)) 2101 | 2102 | if not exact: 2103 | set_value(self.length, write_length, io, context) 2104 | 2105 | def sizeof(self, value: O[str], context: Context) -> O[int]: 2106 | length = peek_value(self.length, context) 2107 | length_unit = peek_value(self.length_unit, context) 2108 | type = peek_value(self.type, context) 2109 | exact = peek_value(self.exact, context) 2110 | encoding = peek_value(self.encoding, context) 2111 | terminator = peek_value(self.terminator, context) 2112 | 2113 | if exact and length is not None: 2114 | l = length * length_unit 2115 | elif value is not None: 2116 | l = len(value.encode(encoding)) 2117 | if type == 'c': 2118 | l += len(terminator) 2119 | else: 2120 | return None 2121 | 2122 | if type == 'pascal': 2123 | size_len = _sizeof(self.length_type, l, context) 2124 | if size_len is None: 2125 | return None 2126 | l = add_sizes(to_size(l, context), size_len) 2127 | 2128 | return l 2129 | 2130 | def offsetof(self, path: Sequence[U[int, str]], value: O[str], context: Context) -> O[int]: 2131 | idx, fpath = path[0], path[1:] 2132 | if not isinstance(idx, int): 2133 | raise ValueError('not a sequence index: {}'.format(idx)) 2134 | 2135 | length = peek_value(self.length, context) 2136 | length_unit = peek_value(self.length_unit, context) 2137 | type = peek_value(self.type, context) 2138 | exact = peek_value(self.exact, context) 2139 | encoding = peek_value(self.encoding, context) 2140 | terminator = peek_value(self.terminator, context) 2141 | 2142 | if type == 'pascal': 2143 | if exact and length is not None: 2144 | l = length * length_unit 2145 | elif value is not None: 2146 | l = len(value.encode(encoding)) 2147 | if type == 'c': 2148 | l += len(terminator) 2149 | else: 2150 | l = None 2151 | off = _sizeof(self.length_type, l, context) 2152 | else: 2153 | off = {} 2154 | 2155 | return add_sizes(to_size(idx * length_unit, context), off) 2156 | 2157 | def default(self, context: Context) -> str: 2158 | return '' 2159 | 2160 | def __repr__(self) -> str: 2161 | return '<{}{}({}{})>'.format(self.type.capitalize(), class_name(self), '=' if self.exact else '', self.length) 2162 | 2163 | 2164 | 2165 | ## Main functions 2166 | 2167 | def to_io(value: Any) -> IO: 2168 | if isinstance(value, IO): 2169 | return value 2170 | if value is None: 2171 | value = BytesIO() 2172 | if isinstance(value, (bytes, bytearray)): 2173 | value = BytesIO(value) 2174 | return IO(value) 2175 | 2176 | def to_type(spec: Any, ident: O[Any] = None) -> Type: 2177 | if isinstance(spec, Type): 2178 | return spec 2179 | if isinstance(spec, (list, tuple)): 2180 | return Tuple(spec) 2181 | elif hasattr(spec, '__restruct_type__'): 2182 | return spec.__restruct_type__ 2183 | elif hasattr(spec, '__get_restruct_type__'): 2184 | return spec.__get_restruct_type__(ident) 2185 | elif callable(spec): 2186 | return spec(ident) 2187 | 2188 | raise ValueError('Could not figure out specification from argument {}.'.format(spec)) 2189 | 2190 | def get_value(t: Type, context: Context, peek: bool = False) -> Any: 2191 | if isinstance(t, (Generic, PartialAttr)): 2192 | return t.get_value(context, peek) 2193 | return t 2194 | 2195 | def peek_value(t: Type, context: Context, default=None) -> Any: 2196 | if isinstance(t, (Generic, PartialAttr)): 2197 | return t.peek_value(context, default) 2198 | return t 2199 | 2200 | def set_value(t: Type, value: Any, io: IO, context: Context) -> None: 2201 | if isinstance(t, PartialAttr): 2202 | t.set_value(value, io, context) 2203 | 2204 | def to_size(v: Any, context: Context) -> Mapping[str, int]: 2205 | if not isinstance(v, dict): 2206 | stream = context.stream_path[-1] if context.stream_path else context.params.default_stream 2207 | v = {stream.name: v} 2208 | return v 2209 | 2210 | def parse(spec: Any, io: IO, context: O[Context] = None, params: O[Params] = None) -> Any: 2211 | type = to_type(spec) 2212 | io = to_io(io) 2213 | context = context or Context(type, params=params) 2214 | at_start = not context.path 2215 | try: 2216 | return type.parse(io, context) 2217 | except Error: 2218 | raise 2219 | except Exception as e: 2220 | if at_start: 2221 | raise Error(context, e) 2222 | else: 2223 | raise 2224 | 2225 | def emit(spec: Any, value: Any, io: O[IO] = None, context: O[Context] = None, params: O[Params] = None) -> None: 2226 | type = to_type(spec) 2227 | io = to_io(io) 2228 | ctx = context or Context(type, value, params=params) 2229 | try: 2230 | type.emit(value, io, ctx) 2231 | return io.handle 2232 | except Error: 2233 | raise 2234 | except Exception as e: 2235 | if not context: 2236 | raise Error(ctx, e) 2237 | else: 2238 | raise 2239 | 2240 | def _sizeof(spec: Any, value: O[Any], context: Context) -> Mapping[str, O[int]]: 2241 | type = to_type(spec) 2242 | return to_size(type.sizeof(value, context), context) 2243 | 2244 | def _offsetof(spec: Any, path: Sequence[U[int, str]], value: O[Any], context: Context) -> Mapping[str, O[int]]: 2245 | if not path: 2246 | return to_size(0, context) 2247 | type = to_type(spec) 2248 | return to_size(type.offsetof(path, value, context), context) 2249 | 2250 | def sizeof(spec: Any, value: O[Any] = None, context: O[Context] = None, params: O[Params] = None, stream: O[Str] = None) -> O[int]: 2251 | type = to_type(spec) 2252 | ctx = context or Context(type, value, params=params) 2253 | try: 2254 | sizes = _sizeof(type, value, ctx) 2255 | except Exception as e: 2256 | raise Error(ctx, e) 2257 | 2258 | if stream: 2259 | return sizes.get(stream, 0) 2260 | else: 2261 | n = 0 2262 | for v in sizes.values(): 2263 | if v is None: 2264 | return None 2265 | n += v 2266 | return n 2267 | 2268 | def offsetof(spec: Any, path: Sequence[U[int, str]], value: O[Any] = None, context: O[Context] = None, params: O[Params] = None, stream: O[Str] = None) -> O[int]: 2269 | type = to_type(spec) 2270 | ctx = context or Context(type, value, params=params) 2271 | try: 2272 | offsets = _offsetof(type, path, value, ctx) 2273 | except Exception as e: 2274 | raise Error(ctx, e) 2275 | 2276 | if not stream: 2277 | stream = ctx.params.default_stream 2278 | else: 2279 | stream = ctx.params.streams[stream] 2280 | 2281 | off = offsets.get(stream.name, 0) 2282 | if off is None: 2283 | return None 2284 | return ctx.stream_offset(stream) + off 2285 | 2286 | def default(spec: Any, context: O[Context] = None, params: O[Params] = None) -> O[Any]: 2287 | type = to_type(spec) 2288 | ctx = context or Context(type, params=params) 2289 | try: 2290 | return type.default(ctx) 2291 | except Error: 2292 | raise 2293 | except Exception as e: 2294 | if not context: 2295 | raise Error(ctx, e) 2296 | else: 2297 | raise 2298 | 2299 | 2300 | __all_types__ = { 2301 | # Base types 2302 | Nothing, Bits, Data, Implied, Ignored, Pad, Fixed, Generic, 2303 | # Modifier types 2304 | Ref, Rebased, Sized, AlignTo, AlignedTo, Lazy, Processed, Checked, Mapped, 2305 | # Compound types 2306 | StructType, MetaStruct, Struct, Union, Tuple, Any, Arr, Switch, Enum, 2307 | # Primitive types 2308 | Bool, Int, UInt, Float, Str, 2309 | } 2310 | __all__ = [c.__name__ for c in __all_types__ | { 2311 | # Bases 2312 | IO, Context, Params, Error, Type, 2313 | # Functions 2314 | parse, emit, sizeof, offsetof, default, 2315 | }] 2316 | --------------------------------------------------------------------------------