├── .gitignore ├── CHANGES ├── COPYING ├── README.md ├── api.lua ├── ast.lua ├── ast ├── Bitpack.lua ├── IO.lua ├── List.lua ├── Name.lua ├── Node.lua ├── Number.lua ├── Repeat.lua ├── Root.lua └── Table.lua ├── compat1x.lua ├── cursor.lua ├── frexp.lua ├── init.lua ├── io.lua ├── io ├── a.lua ├── b.lua ├── bigendian.lua ├── c.lua ├── defaults.lua ├── endianness.lua ├── f.lua ├── hostendian.lua ├── i.lua ├── littleendian.lua ├── m.lua ├── p.lua ├── pu.lua ├── s.lua ├── seekb.lua ├── seekf.lua ├── seekto.lua ├── u.lua ├── x.lua └── z.lua ├── lexer.lua ├── test.lua ├── test ├── basic.lua ├── common.lua ├── compat1x.lua ├── error.lua ├── fp-bigendian.lua ├── fp-littleendian.lua ├── frexp.lua ├── perf.lua ├── regression.lua └── struct-test-gen.lua ├── vstruct-2.0.1-1.rockspec ├── vstruct-2.0.2-1.rockspec ├── vstruct-2.1.0-1.rockspec ├── vstruct-2.1.1-1.rockspec └── vstruct-2.2.0-1.rockspec /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.sublime-* 3 | -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | 2.2.1 [WIP] 2 | ----------- 3 | Backreference support expanded to field widths. 4 | `vstruct.sizeof()` function added. 5 | `vstruct.records()` now available as a method on compiled formats. 6 | 7 | 2.2 8 | --- 9 | Backreference support added. 10 | 11 | 2.1.1 12 | ----- 13 | Test and documentation improvements. 14 | 15 | 2.1.0 16 | ----- 17 | Lua 5.3 support (thanks @deepakjois!). 18 | 19 | 2.0.2 20 | ----- 21 | LuaRocks metadata fixes. 22 | 23 | 2.0.1 24 | ----- 25 | Internal optimizations resulting in significantly improved performance when operating on string cursors rather than fds. 26 | Dropped support for LuaDist. 27 | Added support for LuaRocks. 28 | 29 | 2.0 30 | --- 31 | *** THIS VERSION BREAKS BACKWARDS COMPATIBILITY *** 32 | Many internal structural improvements. 33 | unpack/pack replaced with read/write in API. 34 | Changed the behaviour of the third argument to read(). 35 | Changed read() and write() on compiled format strings from functions to methods. 36 | Added vstruct.records() iterator. 37 | Added vstruct.register() and format string splicing. 38 | Completely removed the code generator and replaced it with an AST walker. 39 | Improved test coverage. 40 | Improved error reporting for invalid arguments to API functions. 41 | 42 | 1.1.4 43 | ----- 44 | Fix a bug where specific non-integer values passed to `p`, `i`, and `u` will either crash the library or write corrupt data. 45 | Improved handling for errors in the unit tests. 46 | Documentation updates. 47 | 48 | 1.1.3 49 | ----- 50 | `x` format extended to permit padding with a specific value: "x8,15" writes 8 bytes of 0x0F. 51 | Improved error reporting for compile and runtime errors. 52 | Tests for error reporting. 53 | README is now in Github Flavoured Markdown format. 54 | Documentation updates. 55 | 56 | 1.1.2 57 | ----- 58 | 5.2 support 59 | Remove use of module() 60 | 61 | 1.1.1 62 | ----- 63 | Bug fixes in the test suite. 64 | Endianness detection now works properly in luajit. 65 | Autodetect endianness on library load. 66 | 67 | 1.1 68 | --- 69 | Test suite upgrades. 70 | LuaDist compatibility. 71 | Assorted bugfixes. 72 | Widechar support for 'z' format. 73 | Change in 'p' format to make it consistent with the other formats. 74 | Change in bitpacks to read MSB first. 75 | 76 | 1.1 beta 1 77 | ---------- 78 | New test set (contributed by Sanooj) which randomly generates new test cases. 79 | New parser and lexer, with greater extensibility and better error reporting. 80 | Bitpack support. 81 | 82 | 1.0.2 83 | ----- 84 | Bugfix release. 85 | 86 | 1.0.1 87 | ----- 88 | Bugfix release. 89 | 90 | 1.0 final 91 | --------- 92 | Bugfixes to read error handling 93 | Ability to return unpacked values rather than tables 94 | 95 | 1.0 beta 4 96 | ---------- 97 | Added the ability to say 's' with no width to read until EOF. 98 | Modified struct.unpack and struct.pack to return the number of bytes 99 | read/written as a second value. Note that this is not the same as the r/w 100 | pointer delta if seeks are involved. 101 | 102 | 1.0 beta 3 103 | ---------- 104 | Lots of bugfixing and general cleanup 105 | Improved error reporting 106 | API name changes 107 | 108 | 1.0 beta 2 109 | ---------- 110 | Added the counted string format "c". 111 | Added float and double support, courtesy of Peter "Corsix" Cawley. 112 | Updated the test framework. 113 | Fixed a bug in format m that could result in data loss when manipulating fields 114 | of 7 bytes or more width. 115 | 116 | 117 | 1.0 beta 1 118 | ---------- 119 | Released to the world. 120 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright © 2011-2020 Rebecca "ToxicFrog" Kelly 2 | FP module copyright © 2008 Peter "Corsix" Cawley 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | this software and associated documentation files (the "Software"), to deal in 6 | the Software without restriction, including without limitation the rights to 7 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | the Software, and to permit persons to whom the Software is furnished to do so, 9 | subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Contents ## 2 | 3 | 1. [Overview](#1-overview) 4 | 2. [Warnings](#2-warnings) 5 | 1. Numeric Precision 6 | 2. Known Incompatibilities 7 | 3. [Setup](#3-setup) 8 | 1. Installation 9 | 2. Testing 10 | 3. Loading 11 | 4. Backwards Compatibility 12 | 4. [API](#4-api) 13 | 1. Variables 14 | 2. Functions 15 | 5. [Format string syntax](#5-format-string-syntax) 16 | 1. Repeat markers 17 | 2. Tables 18 | 3. Names 19 | 4. Bitpacks 20 | 5. Backreferences 21 | 6. [Data items](#6-data-items) 22 | 1. Controlling endianness 23 | 2. Seeking 24 | 3. Reading and writing 25 | 7. [Adding new formats](#7-adding-new-formats) 26 | 8. [Credits](#8-credits) 27 | 28 | 29 | 30 | ## 1. Overview ## 31 | 32 | VStruct is a library for Lua 5.1-5.4 and LuaJIT 2. It provides functions for manipulating binary data, in particular for reading binary files or byte buffers into Lua values and for writing Lua values back into files or buffers. Supported data types include: 33 | 34 | * signed and unsigned integers of arbitrary size 35 | * fixed and floating point numbers 36 | * fixed-size, length-prefixed, and null-terminated strings 37 | * booleans and bitmasks 38 | * bit-packed integers, booleans and bitmasks 39 | 40 | In addition, the library supports seeking, alignment, and byte order controls, repetition, grouping of data into tables, and named fields. 41 | 42 | 43 | 44 | ## 2. Warnings ## 45 | 46 | ### 2.1 Numeric Precision ### 47 | 48 | When reading and writing numeric formats, vstruct is inherently limited by lua's number format, which is by default the IEEE 754 double. What this means in practice is that formats `i`, `u`, `c`, and `p` (see [Data Items](#6-data-items)) may be subject to data loss, if they contain more than 52 significant bits. (The same is true of numeric constants declared in Lua itself, of course, and other libraries which store values in lua numbers). In other words - be careful when manipulating data, especially 64-bit ints, that may not fit in Lua's native types. 49 | 50 | Formats not listed above are not subject to this limitation, as they either do not use Lua numbers at all, or do so only in ways that are guaranteed to be lossless. 51 | 52 | 53 | ### 2.2 Known Incompatibilities ### 54 | 55 | Lua 5.0 is not supported, as vstruct makes heavy use of features introduced in 5.1. 56 | 57 | LuaJIT 2.0b7 has a bug in the code generator that affects vstruct. This is fixed in 2.0b9. If you need to run vstruct in 2.0b7, you will need to disable JIT compilation for the `m.read` function: 58 | 59 | jit.off(require("vstruct.io.m").read, true) 60 | 61 | The `strict` module raises an error when using vstruct outside of LuaJIT, as it checks for the presence of LuaJIT by checking if the global variable `jit` is defined (resulting in a "variable not declared" error). This can be worked around by setting `jit = false` before loading vstruct. 62 | 63 | 64 | ## 3. Setup ## 65 | 66 | ### 3.1 Installation ### 67 | 68 | #### From LuaRocks #### 69 | 70 | vstruct can be installed from the [LuaRocks](https://luarocks.org/) package management system as follows. 71 | 72 | ``` 73 | luarocks install vstruct 74 | ``` 75 | 76 | #### From Source #### 77 | 78 | vstruct is a pure-lua module, and as such requires no seperate build step; it can be installed as-is. It automatically detects which version of Lua you're using it and enables compatibility shims as needed, so one installation can be shared by multiple versions of Lua/LuaJIT. 79 | 80 | The initializer is `vstruct/init.lua`, so it should be installed in a way that means `require "vstruct"` will load that file, and that `require "vstruct.foo"` will load `vstruct/foo.lua`. In a default install of Lua 5.1 on Linux, you can do this simply by copying the `vstruct/` directory from the vstruct distribution into any of these directories: 81 | 82 | * `/usr/local/share/lua/5.1/` 83 | * `/usr/share/lib/5.1/` 84 | * `/usr/local/lib/lua/5.1/` 85 | * `/usr/lib/lua/5.1/` 86 | 87 | This also works for Lua 5.2, replacing the `5.1` in each path with `5.2`. 88 | 89 | If you install it elsewhere, you can accomplish this by modifying `package.path` to include the following entries (assuming that `$LIBDIR` is the directory *containing* `vstruct/`): 90 | 91 | * `$LIBDIR/?.lua` 92 | * `$LIBDIR/?/init.lua` 93 | 94 | Note that installing it to `./` will *not* work unless you also add an entry for `./?/init.lua` to `package.path`; by default, lua will look for `./?.lua` but not `./?/init.lua`. 95 | 96 | 97 | ### 3.2 Testing ### 98 | 99 | vstruct comes with a number of builtin tests. To run them, simply invoke vstruct/test.lua: 100 | 101 | lua vstruct/test.lua 102 | 103 | If any of the tests fail, it will report them on standard output and then raise an error; if all of the tests pass, it will exit cleanly. 104 | 105 | If vstruct is properly installed, you can also load the tests as a library: 106 | 107 | lua -lvstruct.test 108 | require "vstruct.test" 109 | 110 | With the same behaviour - it will load cleanly if the tests succeeded and raise an error if any of them failed. 111 | 112 | 113 | ### 3.3 Loading ### 114 | 115 | vstruct makes itself available under the name `vstruct`. Note that, in accordance with current module conventions, it does *not* assign the module to a global; you must assign the return value of `require` yourself: 116 | 117 | local vstruct = require "vstruct" 118 | print(vstruct._VERSION) 119 | 120 | vstruct does make one global modification when loaded: if the function `math.trunc` is not already defined, it will install its own definition. 121 | 122 | 123 | ### 3.4 Backwards Compatibility ### 124 | 125 | vstruct 2.x is not backwards compatible with 1.x versions, and code using 1.x APIs will not work with it. Converting 1.x code to use 2.x is very easy - just replace calls to `unpack`, `unpackvals`, and `pack` with `read`, `readvals`, and `write`, and if you are using custom formats, rename their `width`, `pack*`, and `unpack*` functions to `size`, `write`, and `read`. 126 | 127 | If you do need to run legacy code and can't, for whatever reason, update it to use the 2.x API or install an older version of vstruct, there is a compatibility module available. `require "vstruct.compat1x"` and it will be loaded, making the 1.x API available through a translation layer. By default, it will emit a warning on stderr every time a legacy function is called; to disable this, set `vstruct.WARN = false` after loading it. 128 | 129 | 130 | ## 4. API ## 131 | 132 | vstruct, once loaded, exports a number of variables and functions, all of them available in the table returned by `require "vstruct"`. 133 | 134 | In this section, a *format string* means the string used to describe a binary format, controlling how data is read or written. The syntax for format strings is described in [section 5](#5-format-string-syntax); the semantics in [section 6](#6-data-items). They are not be confused with the format strings used by `string.format`. 135 | 136 | 137 | ### 4.1 Error Handling ### 138 | 139 | In vstruct, all errors are *hard errors*: they are raised with `error` and must be caught with `pcall`. The vstruct API functions will always either return a valid result or raise an error. Barring bugs in vstruct itself, the only errors one is likely to encounter involve invalid arguments to the API - wrong types, a syntactically invalid format string, or a data table that doesn't match up with the format string. The one potential surprise is seeking past the start or end of a file; in the standard `io` library, this is a soft error (returning `nil,error`), while in vstruct this will immediately raise. 140 | 141 | There are also some conditions which are *not* currently treated as errors: 142 | 143 | * Seeking past the end of the file. This will extend the file when writing; it is an error when reading only if you subsequently attempt to read data without seeking back into the bounds of the file. 144 | * Lossy numeric conversion. In particular, passing a float where an int is expected will truncate the float, and trying to read a number with more bits of precision than Lua supports will give you as many bits as it can manage. 145 | * Passing numbers where strings are expected, or numeric strings where numbers are expected, is permitted *in some cases* due to Lua's automatic coercion between strings and numbers. This may not always be the case, so I recommend not relying on this. 146 | * One can often substitute any boolean true value when `true` is expected, or nil when `false` is expected. As above, this is not guaranteed to work and is not to be relied on. 147 | 148 | 149 | ### 4.2 Variables ### 150 | 151 | These variables are used to control various settings of the library at runtime. At present there is only one public setting, but others may be added in the future. 152 | 153 | -------- 154 | 155 | vstruct.cache = (true|false|nil) 156 | 157 | Enables or disables caching of compiled format strings. This can improve performance in the common case where you are re-using the same format strings many times; if, on the other hand, you are generating lots of different format strings, this will increase memory usage - perhaps significantly - for no performance benefit. 158 | 159 | If true, enables caching. If false, existing cache entries will be used, but new ones will not be created. If nil, the cache is entirely disabled. 160 | 161 | The default is nil. 162 | 163 | 164 | ### 4.3 Functions ### 165 | 166 | vstruct.cursor(string) 167 | 168 | Wraps a string so that it can be used as a file. The returned object ('cur') supports cur:seek, cur:read(num_bytes) and cur:write(string), with the same behaviours as the file methods of the same names. In general, vstruct will attempt to automatically wrap strings if they are passed to it where a file is expected (and unwrap them before returning them); this function is primarily useful when more control over the process is required. 169 | 170 | To access the wrapped string, use cur.str; to determine where the read/write pointer is, use cur.pos. 171 | 172 | -------- 173 | 174 | vstruct.sizeof(fmt) 175 | 176 | Takes a format string and returns the on-disk size of the data it describes, in bytes. If the size cannot be determined solely from the string (for example, it contains backreferences or variable-width fields), or if it performs seeks (even if the seek distance is known in advance), returns `nil`. 177 | 178 | -------- 179 | 180 | vstruct.read(fmt, , [data]) 181 | 182 | `read` takes a format string and a buffer or file to read from, and returns a table of unpacked data items. The items will appear in the table with the same order and/or names that they had in the format string. 183 | 184 | The `data` argument is an optional table. If present, `read` will not create a new table, but will store any values read into this table in-place, and return the table. Existing entries in the table will not be cleared (but might be overwritten by named entries). Numbered entries will always be appended, not overwritten: 185 | 186 | t = { 0, 0, 0 } 187 | vstruct.read("3*u1 x:u1", buf, t) -- t == { 0, 0, 0, 1, 2, 3; x = 4 } 188 | 189 | -------- 190 | 191 | vstruct.readvals(fmt, , [data]) 192 | 193 | Equivalent to `vstruct.read` in every way, except it calls `unpack` (or `table.unpack` in 5.2) before returning. This is a convenience function allowing you to write this: 194 | 195 | count = vstruct.readvals(fmt, fd) 196 | x,y,z = vstruct.readvals(fmt, fd) 197 | 198 | Instead of this: 199 | 200 | count = vstruct.read(fmt, fd)[1] 201 | x,y,z = table.unpack(vstruct.read(fmt, fd)) 202 | 203 | -------- 204 | 205 | vstruct.write(fmt, [fd], data) 206 | 207 | `write` takes a format string and a table of data and writes the contents of the table. If the `fd` argument is present, it will write the data directly to it using standard file io methods; if `fd` is a string, it will wrap it with `vstruct.cursor` first. 208 | 209 | `write` returns the written data in string format, if `fd` was omitted or was a string to wrap; otherwise it returns `fd` as originally passed to the function. 210 | 211 | The structure of the `data` table is expected to be the same as the structure that would be created by a call to `read` with the same format string; in effect, you should be able to take the result of `read` and pass it to `write` unaltered (and get the same data back), and vice versa. 212 | 213 | -------- 214 | 215 | vstruct.explode(int) 216 | vstruct.implode(table) 217 | 218 | `explode` converts a bitmask (in integer format) into a list of booleans, and implode does the converse. In such lists, list[1] is the least significant bit, and list[n] the most significant. 219 | 220 | vstruct.explode(0xF0) -- { 0, 0, 0, 0, 1, 1, 1, 1 } 221 | vstruct.impode { 0, 0, 0, 0, 1, 1, 1, 1 } -- 0xF0 222 | 223 | -------- 224 | 225 | vstruct.compile([name,] format) 226 | 227 | `compile` takes a format string and runs it through the compiler and code generator, but does not actually pack or unpack anything. Instead, it returns a *format object* with the following fields: 228 | 229 | * `format.source` - the original format string 230 | * `format:read(fd, [data])` - equivalent to `vstruct.read(format.source, fd, data)` 231 | * `format:write(fd, data)` - equivalent to `vstruct.write(format.source, fd, data)` 232 | * `format:records(fd, unpacked)` - equivalent to `vstruct.records(format.source, fd, unpacked)` 233 | * `format:sizeof()` - equivalent to `vstruct.sizeof(format.source)` 234 | 235 | In effect, the following code: 236 | 237 | obj = vstruct.compile(fmt) 238 | d = obj:read(fd) 239 | obj:write(fd, d) 240 | 241 | Is equivalent to: 242 | 243 | d = vstruct.read(fmt, fd) 244 | vstruct.write(fmt, fd, d) 245 | 246 | If `name` is specified, it additionally registers the format string it just compiled under `name`, allowing it to referenced in future format strings as `&name`; see [Splices](#55-splices) for details. 247 | 248 | -------- 249 | 250 | vstruct.records(format, , [unpack]) 251 | 252 | Given a format string, and a data source to read records of that format from, `records` returns an iterator over all of those records. It terminates when there is no more data to be read (i.e. fd:read(0) returns nil rather than ""). If there is an incomplete record at the end of the data source, it will error. 253 | 254 | It is roughly analogous to: 255 | 256 | for _,record in ipairs(vstruct.read(NUM_RECORDS .. "*" .. format, fd)) do 257 | ... 258 | end 259 | 260 | Except that it doesn't require you to know the number of records in advance, and doesn't read all of the records into memory at once. 261 | 262 | If `unpack` is true, it will additionally call `unpack()` (`table.unpack()` in 5.2) on each record and return the results of the unpacking, so that you can (for example) easily read a list of coordinates with: 263 | 264 | for x,y,z in vstruct.records("f8 f8 f8", fd, true) do ... 265 | 266 | If `unpack` is false or unspecified, it behaves the same as `vstruct.read`. 267 | 268 | 269 | ## 5. Format string syntax ## 270 | 271 | A format string consists of a series of *format items*. A format item is: 272 | 273 | * a data item, seek control, or endianness control (see [Data Items](#6-data-items)) 274 | * a constant repeat marker `N*` followed by a format item, or a sequence of format items enclosed in `(` and `)` 275 | * a variable repeat marker `#foo*` followed by a format item, or a sequence of format items enclosed in `(` and `)` 276 | * a table `{ ... }` enclosing any number of format items 277 | * a name `foo:` followed by a data item or table 278 | * a bitpack `[S| ... ]` enclosing any number of *bitpack-capable* format items 279 | * a splice `&name` 280 | * a comment, starting with `--` and ending at the next newline 281 | 282 | These are explained in detail in the rest of this section, apart from data items, seek controls, and endianness controls, which are a sufficiently lengthy topic that they have [a section of their own](#6-data-items). 283 | 284 | In general, whitespace may be omitted where the result is unambiguous, and when present, the amount and type of whitespace is irrelevant. Comments are considered to be whitespace. 285 | 286 | 287 | ### 5.1 Repeat markers ### 288 | 289 | A repeat marker consists of a *repeat count*, followed by a `*`, followed by a format item (or a group of such items enclosed in parentheses). The following item is repeated a number of times equal to the count. For example, these three format strings: 290 | 291 | "u4 u4 u4 u4" 292 | "{ u2 b1 } { u2 b1 }" 293 | "u2 u2 u4 u2 u2 u4 u2 u2 u4 m2" 294 | 295 | Can be expressed more concisely as these: 296 | 297 | "4*u4" 298 | "2*{ u2 b1 }" 299 | "3*(u2 u2 u4) m2" 300 | 301 | Repeat counts can be either numeric literals or [backreferences](#56-backreferences) 302 | 303 | ### 5.2 Tables ### 304 | 305 | A table consists of any number of format items enclosed in curlybraces. When reading, the items contained in the table will be packed into their own subtable in the output; when writing, items contained in the table will be searched for in a subtable. For example, this format string: 306 | 307 | "{ u4 u4 u4 } { { b1 b1 } s8 }" 308 | 309 | Describes the following table (or something similarly structured): 310 | 311 | { 312 | { 1, 2, 3 }; 313 | { 314 | { true, false }; 315 | "test"; 316 | }; 317 | }; 318 | 319 | Note the outer table; `read` returns all read values in a table by default, and `write` expects the values it writes to be contained in one. 320 | 321 | Within a format string, tables may be nested arbitrarily. 322 | 323 | 324 | ### 5.3 Names ### 325 | 326 | A name consists of a valid Lua identifier, or sequence of such identifiers separated with `.`, followed by a `:`. It must be followed by a data item or a table. The following item will be stored in/retrieved from a field with the given name, rather than being read/written sequentially as is the default. 327 | 328 | For example, this table: 329 | 330 | { 331 | coords = { x=1, y=2, z=3 }; 332 | } 333 | 334 | Could be expressed by either of these format strings: 335 | 336 | "coords:{ x:u4 y:u4 z:u4 }" 337 | "coords:{} coords.x:u4 coords.y:u4 coords.z:u4" 338 | 339 | 340 | ### 5.4 Bitpacks ### 341 | 342 | A bitpack consists of multiple data items packed into a single item (typically an int) with no regard for byte-alignment - for example, something that stores three five-bit ints and a one-bit boolean in a single 16-bit int. 343 | 344 | These are expressed in vstruct as `'[' size '|' items ']'`, where `size` is the size of the entire bitpack in *bytes* and `items` is a sequence of data items with their sizes in *bits*. The above example would be expressed thus: 345 | 346 | "[2| u5 u5 u5 b1 ]" 347 | 348 | Bitpacks read their contents MSB to LSB; in the above example, the boolean is the least significant bit of the enclosing 2-byte int. 349 | 350 | The declared total size of the bitpack, and the sizes of the individual items inside it, *must* match; if they do not, the format string will not compile. If there are bits inside the pack you are not interested in, use the `x` (skip/pad) format to ignore them. 351 | 352 | Despite being conceptually packed into ints, bitpacks are not subject to numeric precision limitations (although the data items inside them might be, if they are large enough); a 100-byte bitpack will be handled just as accurately as a 1-byte one. 353 | 354 | Reading and writing of bitpacks respects *byte* endianness; in little-endian mode, the least significant *byte* is expected to come first. Bits are always MSB first, LSB last. That is to say, given the following bytes on disk: 355 | 356 | 01 02 357 | 358 | It will be read in the following manners in big- and little-endian modes 359 | 360 | "> [2| 4*u4 ]" -- { 0, 1, 0, 2 } 361 | "< [2| 4*u4 ]" -- { 0, 2, 0, 1 } 362 | 363 | At present, only formats `b`, `u`, `i`, `x` and `m` are supported inside bitpacks. 364 | 365 | 366 | ### 5.5 Splices ### 367 | 368 | Splices let you concisely refer to other format strings, provided that those others have been registered ahead of time using the optional `name` argument to `vstruct.compile`. A splice `&foo` is equivalent to including the contents of the format string registered as `foo` at the point where the splice appears; thus, the following two calls are equivalent: 369 | 370 | vstruct.compile("coord", "x:u4 y:u4 z:u4") 371 | vstruct.read("name:z128 position:{ x:u4 y:u4 z:u4 }") 372 | vstruct.read("name:z128 position:{ &coord }") 373 | 374 | 375 | ### 5.6 Backreferences ### 376 | 377 | In addition to numeric literals, vstruct supports *backreferences*, allowing the value from an earlier read to be used directly in the format string, instead of needing to perform one read, then construct a new format string based on the results. Backreferences are supported for repeat counts and field widths; they are *not* currently supported for bitpacks. 378 | 379 | A backreference consists of a valid Lua identifier, or sequence of such identifiers separated by `.`, prefixed with a `#`. When encountered on read or write, the value of the named field will be used. If that field has not been read yet, or if it does not hold a numeric type, an error is thrown. 380 | 381 | Using backreferences allows you to replace this code: 382 | 383 | local header = vstruct.read(fd, "offset:u4 count:u4") 384 | local toc = vstruct.read(fd, "@"..header.offset.." "..header.count.."*&toc_entry") 385 | 386 | With the much more readable: 387 | 388 | local toc = vstruct.read(fd, [[ 389 | header:{ offset:u4 count:u4 } 390 | @#header.offset #header.count * &toc_entry 391 | ]]) 392 | 393 | Note that when writing, it is (as always) up to the caller to make sure the data structure is internally consistent; if you use the above code to read in a TOC, add or remove entries from it, and then write it back out, make sure you also update `toc.header.count` accordingly! 394 | 395 | 396 | ## 6. Data Items ## 397 | 398 | This section describes the individual field types that make up the bulk of a format string, as well as the seek and endianness controls available. All data items consist of a single letter or punctuation character, optionally followed by one or more comma-separated numbers indicating the size of the corresponding field (or other parameters specific to the type). 399 | 400 | By convention, in this section, upper-case single letters represent decimal numbers to be filled in when the format string is written. In particular, *S* is consistently used to mean the size, in bytes, of a field, and *A* an address or offset in the packed data. 401 | 402 | Sizes and offsets can be either numeric literals or [backreferences](#56-backreferences), although the latter is usually only useful for seeking and, sometimes, string formats. 403 | 404 | 405 | ### 6.1 Controlling Endianness ### 406 | 407 | At any given moment, when reading or writing, vstruct is in either *big-endian* or *little-endian* mode. These affect the order in which it expects bytes to appear for formats `f`, `u`, `i`, and `m`, for the initial string length in `c`, and for the bytes making up a bitpack. In big-endian mode it expects the most significant byte to occur first; in little-endian mode, the least sigificant byte. (There is at present no support for more esoteric modes like middle-endian.) 408 | 409 | Each operation starts in *host-endian* mode - the endianness is set to match that of the host system. It can subsequently be controlled with the following characters in the format string: 410 | 411 | -------- 412 | 413 | < 414 | 415 | Sets the endianness to little-endian (eg, Intel processors) 416 | 417 | -------- 418 | 419 | > 420 | 421 | Sets the endianness to big-endian (eg, PPC processors) 422 | 423 | -------- 424 | 425 | = 426 | 427 | Resets the endianness to match that of the host system 428 | 429 | The means by which host-endianness is detected is, at present, implementation specific. Under luaJIT, it uses the ffi library (specifically, the ffi.abi function). Under standard lua, it uses string.dump on an empty function and checks the bytecode header. If neither of these is available, it emits a warning and assumes little-endian. 430 | 431 | If you have a lua implementation for which this approach fails (either crashing, or getting the wrong answer), please file a bug report. 432 | 433 | 434 | ### 6.2 Seeking ### 435 | 436 | vstruct supports seeking in the underlying buffer or file; these operations will translate into a call to the :seek() method. Note that attempting to use these when reading from or writing to a non-seekable stream (such as stdout) will generate a runtime error. In that case, use `x` (the skip/pad format) instead. 437 | 438 | -------- 439 | 440 | @A 441 | 442 | Seek to absolute address `A`. 443 | 444 | -------- 445 | 446 | +A 447 | 448 | Seek forwards `A` bytes. 449 | 450 | -------- 451 | 452 | -A 453 | 454 | Seek backwards `A` bytes. 455 | 456 | -------- 457 | 458 | aS 459 | 460 | Align to word size `S` (seek to the next address which is a multiple of `S`). If the current address is `S`-aligned already, this is a no-op. 461 | 462 | 463 | ### 6.3 Reading and Writing ### 464 | 465 | The following items perform actual reading and writing of data. 466 | 467 | -------- 468 | 469 | bS -- Boolean. 470 | 471 | - Read: as `uS`, but returns true if the result is non-zero and false otherwise. 472 | - Write: as `uS` with input 1 if true and 0 otherwise. 473 | 474 | Note that when writing, the output for true is integer 1, not all bits 1. 475 | 476 | -------- 477 | 478 | cS -- Length-prefixed ("counted") string. 479 | 480 | - Read: `uS` to determine the length of the string `S'`, followed by `sS'`. 481 | - Write: the length of the string as `uS`, followed by the string itself. 482 | 483 | The counted string is a common idiom where a string is immediately prefixed with its length, as in: 484 | 485 | size_t len; 486 | char[] str; 487 | 488 | The counted string format can be used to easily read and write these. The size provided is the size of the `len` field, which is treated as an unsigned int. Only the string itself is returned (when reading) or required (when writing). Internally, this is implemented as `u` followed by `s`; consequently, it is affected by endianness. 489 | 490 | -------- 491 | 492 | fS -- IEEE 754 floating point. 493 | 494 | - Read/Write: a float, double, or quad. 495 | 496 | Valid sizes are 4 (float) 8 (double) and 16 (quad). Note that quads have more precision than the default lua number format (double), and thus may not read exactly unless you're using a custom lua build. 497 | 498 | Affected by endianness. 499 | 500 | -------- 501 | 502 | iS -- Signed integer. 503 | 504 | - Read/Write: a signed integer with `S` bytes of precision. 505 | 506 | When writing, floating point values will be truncated, not rounded. 507 | 508 | Affected by endianness. 509 | 510 | -------- 511 | 512 | mS -- Bitmask. 513 | 514 | - Read: as `uS`, but explodes the result into a list of booleans, one per bit. 515 | - Write: implodes the input value, then writes it as `uS`. 516 | 517 | In effect, a `u` that automatically calls `vstruct.implode/explode`; unlike `u`, however, it can operate on fields of arbitrarily large size without loss of precision, regardless of what numeric type lua is using. 518 | 519 | Affected by endianness. 520 | 521 | -------- 522 | 523 | pS,F -- Signed fixed point. 524 | puS,F -- Unsigned fixed point. 525 | 526 | `S` is, as usual, the size of the entire field in *bytes*. `F` is the number of *bits* of fractional precision. Thus, a 24.8 fixed point number (24 bits integer, 8 bits fraction, 32 bits total) would be written as `"p4,8"`. 527 | 528 | - Read/Write: a `S`-byte fixed point number with `F` bits of fractional precision. 529 | 530 | When writing, values which cannot be precisely expressed in the given precision will be truncated, not rounded. 531 | 532 | Affected by endianness. 533 | 534 | -------- 535 | 536 | sS -- Fixed-length string. 537 | 538 | `S` is optional. 539 | 540 | - Read: reads exactly `S` bytes and returns them as a string. If `S` is omitted, reads until EOF. 541 | - Write: writes exactly `S` bytes; if the given string is too long, it will be truncated, and if too short, it will be nul-padded. If `S` is omitted, it is considered equal to the length of the string (i.e. it will write the contents of the string without truncation or padding). 542 | 543 | -------- 544 | 545 | uS -- Unsigned integer. 546 | 547 | - Read/Write: an unsigned integer with `S` bytes of precision. 548 | 549 | On write, non-integer values will be truncated. The behaviour of negative numbers is unspecified, but will probably not do what you want; in particular, it usually does not produce the same results as `i`, and may become an error in future versions. 550 | 551 | Affected by endianness. 552 | 553 | -------- 554 | 555 | xS,V -- Skip/pad. 556 | 557 | `,V` is optional, and defaults to 0 if omitted. 558 | 559 | - Read: read and discard the next `S` bytes. 560 | - Write: write `S` bytes with value `V`. Within a bitpack, the only valid values for `V` are 0 or 1. 561 | 562 | This format does not consume input data (when writing) or produce output values (when reading). However, unlike the seek controls (`@+-a`), it can be used even when the input or output does not support seeking (e.g. when reading from a pipe or socket). 563 | 564 | -------- 565 | 566 | zS,C -- Nul terminated/nul padded string. 567 | 568 | `S` and `,C` are both optional. The `,` is mandatory if `C` is present. 569 | 570 | `S`, if present, is the length of the string (in *bytes*, not characters). 571 | 572 | `C`, if present, is the size *in bytes* of individual characters in the string. The default is 1. It is important when operating on wide-character strings to specify `C` correctly, so that sequences like "00 66 00 67" are not incorrectly interpreted as ending the string. 573 | 574 | - Read: reads exactly `S` bytes, and returns everything up to the first nul *character*. If `S` is omitted, reads until it encounters a nul. The nul is read, but not included in the returned string. 575 | - Write: writes exactly `S` bytes. If the input is shorter than `S`, nul pads the output; if longer, truncates it to `S - C` bytes and nul terminates it. If `S` is omitted entirely, the string is written out in full and nul terminated. 576 | 577 | When nul terminating, or looking for a nul character to detect the end of the string, `C` zero bytes, `C`-aligned relative to the start of the string, are used. In particular, this means that the following 6-byte string: 578 | 579 | 6600 0066 0000 580 | 581 | Will be one byte long (plus one byte termination) under `"z6,1"`, but four bytes long (plus two bytes termination) under `"z6,2"` - the second and third bytes make up 0000, nul, but since they are not character-aligned, they are instead read as parts of the two characters `6600` and `0066`. 582 | 583 | 584 | 585 | ## 7. Adding new IO operations ## 586 | 587 | If you want to add support for a new data type, or modify or replace an existing one, this is how you do it. To see how the current ones are implemented, look at the files in vstruct/io/; any new formats added will use the same API. 588 | 589 | 590 | ### 7.1 How IO operations are loaded ### 591 | 592 | When vstruct first sees an operation - say `"p2,8"` - it first breaks it down into two parts - the `op` ("p") and the `args` (2,8). It then attempts to load a handler for this operation using: 593 | 594 | require("vstruct.io."..op) 595 | 596 | Typically, this will load the file `vstruct/io/.lua` - in our above example, `vstruct/io/p.lua`. The easiest way to install new operations, then, is just to put them in the `vstruct/io/` directory. 597 | 598 | `package.preload` can also be used; for example, `package.preload["vstruct.io.p"]` can be used to override the version of `p` that comes with vstruct. 599 | 600 | 601 | ### 7.2 How they are used ### 602 | 603 | When loaded, the handler for an IO operation returns a table containing some or all of the following functions. Note that `...` here means the arguments as given in the format string - `2,8` in the above example. 604 | 605 | -------- 606 | 607 | hasdata() 608 | 609 | Returns true if, when writing, this format consumes a value from the table of input. Formats that actually read and write data will typically return true; formats that merely adjust the output stream or internal vstruct state (eg, seek and endianness controls) or those that generate the data on the fly (skip/pad) will return false. 610 | 611 | -------- 612 | 613 | size(...) 614 | 615 | This function is called both at compile time and at runtime. Its arguments are the size specifiers from the format string, with the following preprocessing: 616 | 617 | - numeric parameters are passed through `tonumber()` 618 | - backreference parameters are passed as `true` at compile time and as the numeric value the backreference resolved to at runtime 619 | - zero-length parameters are passed as `nil` 620 | - all other parameters are passed as strings, unmodified 621 | 622 | In practice, this almost always means that the first parameter will either be entirely absent, a constant numeric size, or `true` at compile time and a numeric size at runtime. 623 | 624 | The function must return the exact amount of data, in bytes, that this format will consume from the input if `read` is called, or the exact amount it will append to the output if `write` is called. If this cannot yet be determined (for example, the `c` format, or if backreferences are involved), if it changes the position of the read/write pointer (for example, seek commands), or if does anything else that might interfere with vstruct's own use of the file handle, it should return nil. 625 | 626 | Formats that neither change the state of the file handle nor read or write data should return 0; this is the case, for example, for the endianness controls. 627 | 628 | The default implementation asserts that a size (either constant or backreference) was specified, then returns the specified size or nil, respectively. 629 | 630 | -------- 631 | 632 | write(fd, data, ...) 633 | 634 | This is called with the file descriptor (which may not be backed by an actual file, but will conform to the Lua file API), a data value to pack, and tail arguments based on the size specifiers in the format string (as with `size()`). 635 | 636 | If `size()` returned a value earlier, this *must* ignore `fd`, pack `data` into a string, and return the string - the caller will handle writing the string to the fd in an efficient manner. If it did not, this function may freely choose either to return a packed string, or to manipulate `fd` directly (in which case it should return nil). 637 | 638 | Some operators, of course, must manipulate `fd` directly by their very nature (such as seek controls); when possible, however, one should endeavour to pre-calculate sizes and return packed strings. 639 | 640 | -------- 641 | 642 | read(fd, buffer, ...) 643 | 644 | This is called with the same arguments as `write()`, except that if `size()` returned a value earlier, `buffer` will be a string of exactly that many bytes (containing the bytestream to be unpacked); otherwise `buffer` will be nil. 645 | 646 | If `size()` returned a value earlier, `read` *must* ignore `fd` and return the value represented by the contents of `buffer`. If `size()` returned nil, `read` must manipulate `fd` directly to get the data it needs, and ignore `buffer`. In either case it must return the unpacked value. 647 | 648 | -------- 649 | 650 | writebits(bit, data, ...) 651 | 652 | This is called when the operation is performed inside a bitpack. Data should be written bit-by-bit, MSB first; to write a bit `B` (which must be 0 or 1), call `bit(B)`. `bit()` does not presently accept multiple arguments to write multiple bits at once. Other arguments are identical to `write()`. 653 | 654 | -------- 655 | 656 | readbits(bit, ...) 657 | 658 | The converse of `writebits`. Each call to `bit()` returns the next bit, MSB first. Other arguments are identical to `read()`. 659 | 660 | 661 | 662 | ## 8. Credits ## 663 | 664 | While most of the library code was written by me (Bex Kelly), the existence of this library owes itself to many others. 665 | 666 | * The original inspiration came from Roberto Ierusalimschy's "struct" library and Luiz Henrique de Figueiredo's "lpack" library, as well as the "struct" available in Python. 667 | * The floating point code was contributed by Peter Cawley ("corsix") on lua-l, as was support for Lua 5.2. 668 | * sanooj, from #lua on freenode, has done so much testing and bug reporting that at this point he's practically a co-author; the 'struct-test-gen' module in test/ is his work, and has aided in detected many bugs. 669 | * The overall library design and interface are the result of much discussion with rici, sanooj, Keffo, snogglethorpe, Spark, kozure, Vornicus, McMartin, and probably several others I've forgotten about on IRC (#lua on freenode and #code on nightstar). 670 | * Lua 5.3 compatibility, and LuaRocks support, was contributed by deepakjois 671 | * Finally, without Looking Glass Studios to make System Shock, and Team TSSHP (in particular Jim "hairyjim" Cameron) to reverse engineer it, I wouldn't have had a reason to write this library in the first place. 672 | -------------------------------------------------------------------------------- /api.lua: -------------------------------------------------------------------------------- 1 | local vstruct = require "vstruct" 2 | local cursor = require "vstruct.cursor" 3 | local ast = require "vstruct.ast" 4 | 5 | local lua52 = tonumber(_VERSION:match"%d+%.%d+") >= 5.2 6 | local loadstring = lua52 and load or loadstring 7 | local _unpack = table.unpack or unpack 8 | 9 | local api = {} 10 | vstruct.registry = {} 11 | 12 | function api.check_arg(caller, index, value, typename, check) 13 | if not check then 14 | check = function(v) return type(v) == typename end 15 | end 16 | 17 | return check(value) 18 | or error(string.format( 19 | "bad argument #%d to '%s' (%s expected, got %s)", 20 | index, 21 | caller, 22 | typename, 23 | type(value))) 24 | end 25 | 26 | local function is_fd(fd) 27 | local e,r = pcall(function() 28 | return fd and (fd.read or fd.write) 29 | end) 30 | return e and r 31 | end 32 | 33 | local function wrap_fd(fd) 34 | if type(fd) == "string" then 35 | return cursor(fd) 36 | end 37 | return fd 38 | end 39 | 40 | local function unwrap_fd(fd) 41 | if getmetatable(fd) == cursor then 42 | fd:flush() 43 | return fd.str 44 | end 45 | return fd 46 | end 47 | 48 | function api.read(ast, fd, data) 49 | fd = wrap_fd(fd) 50 | 51 | api.check_arg("vstruct.read", 2, fd, "file or string", is_fd) 52 | if data ~= nil then 53 | api.check_arg("vstruct.read", 3, data, "table") 54 | end 55 | 56 | return ast.ast:read(fd, data or {}) 57 | end 58 | 59 | function api.write(ast, fd, data) 60 | if fd and not data then 61 | data,fd = fd,nil 62 | end 63 | fd = wrap_fd(fd or "") 64 | 65 | api.check_arg("vstruct.write", 2, fd, "file or string", is_fd) 66 | api.check_arg("vstruct.write", 3, data, "table") 67 | 68 | local result = ast.ast:write(fd, data) 69 | return unwrap_fd(fd) 70 | end 71 | 72 | function api.records(ast, fd, unpacked) 73 | fd = wrap_fd(fd or "") 74 | api.check_arg("vstruct.records", 2, fd, "file or string", is_fd) 75 | if unpacked ~= nil then 76 | api.check_arg("vstruct.records", 3, unpacked, "boolean") 77 | end 78 | return function() 79 | if fd:read(0) then 80 | if unpacked then 81 | return _unpack(ast:read(fd)) 82 | else 83 | return ast:read(fd) 84 | end 85 | end 86 | end 87 | end 88 | 89 | 90 | function api.sizeof(ast) 91 | return ast.ast.size 92 | end 93 | 94 | local cache = {} 95 | 96 | function api.compile(name, format) 97 | local obj,root 98 | 99 | if vstruct.cache ~= nil and cache[format] then 100 | obj = cache[format] 101 | root = obj.ast 102 | else 103 | root = ast.parse(format) 104 | obj = { 105 | source = format; 106 | ast = root; 107 | read = api.read; 108 | write = api.write; 109 | records = api.records; 110 | sizeof = api.sizeof; 111 | } 112 | 113 | if vstruct.cache == true then 114 | cache[format] = obj 115 | end 116 | end 117 | 118 | if name then 119 | vstruct.registry[name] = root 120 | end 121 | 122 | return obj 123 | end 124 | 125 | return api 126 | -------------------------------------------------------------------------------- /ast.lua: -------------------------------------------------------------------------------- 1 | -- Abstract Syntax Tree module for vstruct 2 | -- This module implements the parser for vstruct format definitions. It is a 3 | -- fairly simple recursive-descent parser that constructs an AST using Lua 4 | -- tables, and then generates lua source from it. 5 | 6 | -- See ast/*.lua for the implementations of various node types in the AST, 7 | -- and see lexer.lua for the implementation of the lexer. 8 | 9 | local vstruct = require "vstruct" 10 | local lexer = require "vstruct.lexer" 11 | 12 | local ast = {} 13 | local cache = {} 14 | 15 | -- load the implementations of the various AST node types 16 | for _,node in ipairs { "IO", "List", "Name", "Table", "Repeat", "Root", "Bitpack", "Number" } do 17 | ast[node] = require ((...).."."..node) 18 | end 19 | 20 | -- given a source string, compile it 21 | -- returns a table containing pack and unpack functions and the original 22 | -- source - see README#vstruct.compile for a description. 23 | -- 24 | -- if (vstruct.cache) is non-nil, will return the cached version, if present 25 | -- if (vstruct.cache) is true, will create a new cache entry, if needed 26 | function ast.parse(source) 27 | local lex = lexer(source) 28 | local root = ast.List() 29 | 30 | for node in (function() return ast.next(lex) end) do 31 | root:append(node) 32 | end 33 | 34 | return ast.Root(root) 35 | end 36 | 37 | -- used by the rest of the parser to report syntax errors 38 | function ast.error(lex, expected) 39 | error("vstruct: parsing format string at "..lex.where()..": expected "..expected..", got "..lex.peek().type) 40 | end 41 | 42 | -- Everything below this line is internal to the recursive descent parser 43 | 44 | function ast.io(lex) 45 | local io = ast.raw_io(lex) 46 | if io.hasvalue then 47 | return ast.Name(nil, io) 48 | else 49 | return io 50 | end 51 | end 52 | 53 | function ast.raw_io(lex) 54 | local name = lex.next().text 55 | local next = lex.peek() 56 | 57 | if next and next.type == "number" and not lex.whitespace() then 58 | return ast.IO(name, lex.next().text) 59 | else 60 | return ast.IO(name, nil) 61 | end 62 | end 63 | 64 | function ast.key(lex) 65 | local name = lex.next().text 66 | local next = lex.peek() 67 | 68 | if next.type == "io" then 69 | local io = ast.raw_io(lex) 70 | if not io.hasvalue then 71 | ast.error(lex, "value (io specifier or table) - format '"..name.."' has no value") 72 | end 73 | return ast.Name(name, io) 74 | elseif next.type == "{" then 75 | return ast.Name(name, ast.raw_table(lex)) 76 | else 77 | ast.error(lex, "value (io specifier or table)") 78 | end 79 | end 80 | 81 | function ast.next(lex) 82 | local tok = lex.peek() 83 | 84 | if tok.type == "EOF" then 85 | return nil 86 | end 87 | 88 | if tok.type == '(' then 89 | return ast.group(lex) 90 | 91 | elseif tok.type == '{' then 92 | return ast.table(lex) 93 | 94 | elseif tok.type == '[' then 95 | return ast.bitpack(lex) 96 | 97 | elseif tok.type == "io" then 98 | return ast.io(lex) 99 | 100 | elseif tok.type == "key" then 101 | return ast.key(lex) 102 | 103 | elseif tok.type == "number" then 104 | return ast.repetition(lex) 105 | 106 | elseif tok.type == "control" then 107 | return ast.control(lex) 108 | 109 | elseif tok.type == "splice" then 110 | return ast.splice(lex) 111 | 112 | else 113 | ast.error(lex, "'(', '{', '[', name, number, control, or io specifier") 114 | end 115 | end 116 | 117 | function ast.next_until(lex, type) 118 | return function() 119 | local tok = lex.peek() 120 | 121 | if tok.type == 'EOF' then 122 | ast.error(lex, type) 123 | end 124 | 125 | if tok.type == type then 126 | return nil 127 | end 128 | 129 | return ast.next(lex) 130 | end 131 | end 132 | 133 | function ast.splice(lex) 134 | local name = lex.next().text 135 | 136 | local root = vstruct.registry[name] 137 | if not root then 138 | error("vstruct: attempt to splice in format '"..name.."', which is not registered") 139 | end 140 | 141 | return root[1] 142 | end 143 | 144 | function ast.repetition(lex) 145 | local count = ast.Number(lex.next().text) 146 | ast.require(lex, "*"); 147 | return ast.Repeat(count, ast.next(lex)) 148 | end 149 | 150 | function ast.group(lex) 151 | ast.require(lex, '(') 152 | 153 | local group = ast.List() 154 | group.tag = "group" 155 | 156 | for next in ast.next_until(lex, ')') do 157 | group:append(next) 158 | end 159 | 160 | ast.require(lex, ')') 161 | return group 162 | end 163 | 164 | function ast.table(lex) 165 | return ast.Name(nil, ast.raw_table(lex)) 166 | end 167 | 168 | function ast.raw_table(lex) 169 | ast.require(lex, '{') 170 | 171 | local group = ast.Table() 172 | 173 | for next in ast.next_until(lex, '}') do 174 | group:append(next) 175 | end 176 | 177 | ast.require(lex, '}') 178 | return group 179 | end 180 | 181 | function ast.bitpack(lex) 182 | ast.require(lex, "[") 183 | 184 | local bitpack = ast.Bitpack(tonumber(ast.require(lex, "number").text)) 185 | 186 | ast.require(lex, "|") 187 | 188 | for next in ast.next_until(lex, ']') do 189 | bitpack:append(next) 190 | end 191 | 192 | ast.require(lex, "]") 193 | bitpack:finalize() 194 | return bitpack 195 | end 196 | 197 | function ast.require(lex, type) 198 | local t = lex.next() 199 | 200 | if t.type ~= type then 201 | ast.error(lex, type) 202 | end 203 | 204 | return t 205 | end 206 | 207 | return ast 208 | 209 | --[[ 210 | 211 | format -> commands 212 | 213 | command -> repeat | bitpack | group | named | value | control | splice 214 | 215 | repeat -> count '*' command 216 | bitpack -> '[' NUMBER '|' commands ']' 217 | group -> '(' commands ')' 218 | 219 | named -> NAME ':' value 220 | value -> table | primitive 221 | table -> '{' commands '}' 222 | 223 | count -> '#' NAME | NUMBER 224 | splice -> '&' NAME 225 | 226 | primitive -> ATOM NUMBERS 227 | 228 | control -> SEEK NUMBER | ENDIANNESS 229 | 230 | --]] 231 | -------------------------------------------------------------------------------- /ast/Bitpack.lua: -------------------------------------------------------------------------------- 1 | local io = require "vstruct.io" 2 | local Node = require "vstruct.ast.Node" 3 | local Bitpack = Node:copy() 4 | local unpack = table.unpack or unpack 5 | 6 | -- return an iterator over the individual bits in buf 7 | local function biterator(buf) 8 | local e = io("endianness", "get") 9 | 10 | local data = { buf:byte(1,-1) } 11 | local bit = 7 12 | local byte = e == "big" and 1 or #data 13 | local delta = e == "big" and 1 or -1 14 | 15 | return function() 16 | local v = math.floor(data[byte]/(2^bit)) % 2 17 | 18 | bit = (bit - 1) % 8 19 | 20 | if bit == 7 then -- we just wrapped around 21 | byte = byte + delta 22 | end 23 | 24 | return v 25 | end 26 | end 27 | 28 | local function bitpacker(buf, size) 29 | for i=1,size do 30 | buf[i] = 0 31 | end 32 | 33 | local e = io("endianness", "get") 34 | 35 | local bit = 7 36 | local byte = e == "big" and 1 or size 37 | local delta = e == "big" and 1 or -1 38 | 39 | return function(b) 40 | buf[byte] = buf[byte] + b * 2^bit 41 | 42 | bit = (bit - 1) % 8 43 | 44 | if bit == 7 then -- we just wrapped around 45 | byte = byte + delta 46 | end 47 | end 48 | end 49 | 50 | function Bitpack:__init(size) 51 | self.size = 0 52 | self.total_size = size 53 | end 54 | 55 | function Bitpack:finalize() 56 | self.size = self.size/8 -- children are getting added with size in bits, not bytes 57 | assert(self.size, "bitpacks cannot contain variable-width fields") 58 | assert(self.size == self.total_size, "bitpack contents do not match bitpack size: "..self.size.." ~= "..self.total_size) 59 | end 60 | 61 | function Bitpack:read(fd, data) 62 | local buf = fd:read(self.size) 63 | self:readbits(biterator(buf), data) 64 | end 65 | 66 | function Bitpack:write(fd, ctx) 67 | local buf = {} 68 | self:writebits(bitpacker(buf, self.size), ctx) 69 | fd:write(string.char(unpack(buf))) 70 | end 71 | 72 | return Bitpack 73 | -------------------------------------------------------------------------------- /ast/IO.lua: -------------------------------------------------------------------------------- 1 | local io = require "vstruct.io" 2 | local unpack = table.unpack or unpack 3 | local Number = require "vstruct.ast.Number" 4 | local Node = require "vstruct.ast.Node" 5 | local IO = Node:copy() 6 | 7 | local function is_backref(arg) 8 | return arg:match('^#[%a_][%w_.]*$') 9 | end 10 | 11 | local function str2args(args) 12 | local argv = { n = 0; has_backrefs = false; } 13 | if args then 14 | local args = args.."," 15 | 16 | for arg in args:gmatch("([^,]*),") do 17 | if #arg == 0 then arg = nil 18 | elseif tonumber(arg) then arg = tonumber(arg) 19 | elseif is_backref(arg) then 20 | arg = Number(arg) 21 | argv.has_backrefs = true 22 | end 23 | argv.n = argv.n +1 24 | argv[argv.n] = arg 25 | end 26 | end 27 | return argv 28 | end 29 | 30 | function IO:__init(name, args) 31 | self.name = name 32 | self.argv = str2args(args) 33 | self.size = io(name, "size", self:get_argv()) 34 | self.hasvalue = io(name, "hasvalue", self:get_argv()) 35 | end 36 | 37 | function IO:read(fd, data) 38 | local buf 39 | 40 | if self.size and self.size > 0 then 41 | buf = fd:read(self.size) 42 | assert(buf and #buf == self.size, "attempt to read past end of buffer in format "..self.name) 43 | end 44 | 45 | return io(self.name, "read", fd, buf, self:get_argv(data)) 46 | end 47 | 48 | function IO:readbits(bits, data) 49 | return io(self.name, "readbits", bits, self:get_argv(data)) 50 | end 51 | 52 | function IO:write(fd, ctx) 53 | local buf = io(self.name, "write", fd, ctx.data, self:get_argv_ctx(ctx)) 54 | if buf then 55 | fd:write(buf) 56 | end 57 | end 58 | 59 | function IO:writebits(bits, ctx) 60 | local buf = io(self.name, "writebits", bits, ctx.data, self:get_argv_ctx(ctx)) 61 | if buf then 62 | fd:write(buf) 63 | end 64 | end 65 | 66 | function IO:get_argv(data) 67 | -- Usually the contents were determined at compile-time and we can just 68 | -- unpack it as is. 69 | if not self.argv.has_backrefs then 70 | return unpack(self.argv, 1, self.argv.n) 71 | end 72 | 73 | -- If backreferences were involved, we have to try to resolve them. 74 | local buf = {} 75 | for i=1,self.argv.n do 76 | if self.argv[i] then 77 | buf[i] = self.argv[i]:get(data) 78 | end 79 | end 80 | 81 | return unpack(buf, 1, self.argv.n) 82 | end 83 | 84 | -- Get fully resolved argv from a write context rather than from the actual 85 | -- data structure. 86 | function IO:get_argv_ctx(ctx) 87 | if ctx.parent then 88 | return self:get_argv(ctx.parent.data) 89 | else 90 | return self:get_argv(ctx.data) 91 | end 92 | end 93 | 94 | return IO 95 | -------------------------------------------------------------------------------- /ast/List.lua: -------------------------------------------------------------------------------- 1 | local Node = require "vstruct.ast.Node" 2 | 3 | return Node:copy() 4 | -------------------------------------------------------------------------------- /ast/Name.lua: -------------------------------------------------------------------------------- 1 | local Node = require "vstruct.ast.Node" 2 | local Name = Node:copy() 3 | 4 | local function put(data, key, val) 5 | if not key then 6 | data[#data+1] = val 7 | else 8 | local data = data 9 | for name in key:gmatch("([^%.]+)%.") do 10 | if data[name] == nil then 11 | data[name] = {} 12 | end 13 | data = data[name] 14 | end 15 | data[key:match("[^%.]+$")] = val 16 | end 17 | end 18 | 19 | -- Return a new subcontext containing only the data referenced by the key. 20 | -- `parent` points to the parent context, so that backreferences can be resolved. 21 | local function get(ctx, key) 22 | local val 23 | if not key then 24 | val = ctx.data[ctx.n] 25 | ctx.n = ctx.n + 1 26 | else 27 | local data = ctx.data 28 | for name in key:gmatch("([^%.]+)%.") do 29 | if data[name] == nil then 30 | val = nil 31 | break 32 | end 33 | data = data[name] 34 | end 35 | val = data[key:match("[^%.]+$")] 36 | end 37 | 38 | assert(val ~= nil, "vstruct: bad input while writing: no value for key "..tostring(key or ctx.n-1)) 39 | return { data = val, n = 1, parent = ctx } 40 | end 41 | 42 | function Name:__init(key, child) 43 | self.child = child 44 | self.size = child.size 45 | self.key = key 46 | end 47 | 48 | function Name:read(fd, data) 49 | return put(data, self.key, self.child:read(fd, data)) 50 | end 51 | 52 | function Name:readbits(bits, data) 53 | return put(data, self.key, self.child:readbits(bits, data)) 54 | end 55 | 56 | function Name:write(fd, ctx) 57 | self.child:write(fd, get(ctx, self.key)) 58 | end 59 | 60 | function Name:writebits(bits, ctx) 61 | self.child:writebits(bits, get(ctx, self.key)) 62 | end 63 | 64 | return Name 65 | -------------------------------------------------------------------------------- /ast/Node.lua: -------------------------------------------------------------------------------- 1 | local Node = {} 2 | Node.__index = Node 3 | Node.__call = function(self, ...) 4 | return self:new(...) 5 | end 6 | 7 | function Node:copy() 8 | local obj = setmetatable({}, self) 9 | obj.__index = obj 10 | return obj 11 | end 12 | 13 | function Node:new(...) 14 | local obj = self:copy() 15 | obj:__init(...) 16 | return obj 17 | end 18 | 19 | function Node:__init() 20 | self.size = 0 21 | end 22 | 23 | function Node:append(node) 24 | self[#self+1] = node 25 | if node.size and self.size then 26 | self.size = self.size + node.size 27 | else 28 | self.size = nil 29 | end 30 | end 31 | 32 | function Node:execute(env) 33 | for i,child in ipairs(self) do 34 | child:execute(env) 35 | end 36 | end 37 | 38 | function Node:read(fd, data) 39 | for i,child in ipairs(self) do 40 | child:read(fd, data) 41 | end 42 | end 43 | 44 | function Node:readbits(bits, data) 45 | for i,child in ipairs(self) do 46 | child:readbits(bits, data) 47 | end 48 | end 49 | 50 | function Node:write(fd, ctx) 51 | for i,child in ipairs(self) do 52 | child:write(fd, ctx) 53 | end 54 | end 55 | 56 | function Node:writebits(bits, ctx) 57 | for i,child in ipairs(self) do 58 | child:writebits(bits, ctx) 59 | end 60 | end 61 | 62 | return Node 63 | -------------------------------------------------------------------------------- /ast/Number.lua: -------------------------------------------------------------------------------- 1 | -- A node that holds either a number, or a reference to an already-read, named 2 | -- field which contains a number. 3 | local Node = require "vstruct.ast.Node" 4 | local Number = Node:copy() 5 | 6 | -- Partially based on get() in Name.lua. Does not support numeric indices, only 7 | -- string ones. 8 | local function get(data, key) 9 | local val 10 | for name in key:gmatch("([^%.]+)%.") do 11 | if data[name] == nil then 12 | val = nil 13 | break 14 | end 15 | data = data[name] 16 | end 17 | val = data[key:match("[^%.]+$")] 18 | 19 | assert(val ~= nil, 'vstruct: backreferenced field "'..key..'" has not been read yet') 20 | assert(type(val) == 'number', 'vstruct: backreferenced field "'..key..'" is not a numeric type') 21 | return val 22 | end 23 | 24 | function Number:__init(text) 25 | if text:match('^#') then 26 | self.key = text:sub(2,-1) 27 | else 28 | self.value = assert(tonumber(text), 'vstruct: numeric constant "'..text..'" is not a number') 29 | end 30 | end 31 | 32 | -- This does not support the regular read/write interface; the only thing you 33 | -- can do with it is get its contained number. 34 | -- If it holds a backreference and data is nil -- which should happen only at 35 | -- compile time -- it returns true to indicate that it contains something but 36 | -- cannot resolve it yet. If it holds a backreference and data is not-nil but 37 | -- also does not contain an appropriate field, it throws. 38 | function Number:get(data) 39 | if self.value then 40 | return self.value 41 | elseif not data then 42 | return true 43 | else 44 | return get(data, self.key) 45 | end 46 | end 47 | 48 | return Number 49 | 50 | -------------------------------------------------------------------------------- /ast/Repeat.lua: -------------------------------------------------------------------------------- 1 | local Node = require "vstruct.ast.Node" 2 | 3 | local Repeat = Node:copy() 4 | 5 | function Repeat:__init(count, child) 6 | self.child = child 7 | self.count = count 8 | if count.value and child.size then 9 | self.size = count:get(nil) * child.size 10 | else 11 | -- Child has runtime-deferred size, or count is a backreference 12 | self.size = nil 13 | end 14 | end 15 | 16 | function Repeat:read(fd, data) 17 | for i=1,self.count:get(data) do 18 | self.child:read(fd, data) 19 | end 20 | end 21 | 22 | function Repeat:readbits(bits, data) 23 | for i=1,self.count:get(data) do 24 | self.child:readbits(bits, data) 25 | end 26 | end 27 | 28 | function Repeat:write(fd, data) 29 | for i=1,self.count:get(data.data) do 30 | self.child:write(fd, data) 31 | end 32 | end 33 | 34 | function Repeat:writebits(bits, data) 35 | for i=1,self.count:get(data.data) do 36 | self.child:writebits(bits, data) 37 | end 38 | end 39 | 40 | return Repeat 41 | -------------------------------------------------------------------------------- /ast/Root.lua: -------------------------------------------------------------------------------- 1 | local io = require "vstruct.io" 2 | local Node = require "vstruct.ast.Node" 3 | 4 | local Root = Node:copy() 5 | 6 | function Root:__init(children) 7 | self[1] = children 8 | self.size = children.size 9 | end 10 | 11 | function Root:read(fd, data) 12 | io("endianness", "host") 13 | self[1]:read(fd, data) 14 | return data 15 | end 16 | 17 | function Root:write(fd, data) 18 | io("endianness", "host") 19 | self[1]:write(fd, { data = data, n = 1 }) 20 | end 21 | 22 | return Root 23 | -------------------------------------------------------------------------------- /ast/Table.lua: -------------------------------------------------------------------------------- 1 | local Node = require "vstruct.ast.Node" 2 | 3 | local Table = Node:copy() 4 | 5 | function Table:execute(env) 6 | env.push() 7 | Node.execute(self, env) 8 | env.pop() 9 | end 10 | 11 | function Table:read(fd, data) 12 | local t = {} 13 | Node.read(self, fd, t) 14 | return t 15 | end 16 | 17 | function Table:readbits(bits, data) 18 | local t = {} 19 | Node.readbits(self, bits, t) 20 | return t 21 | end 22 | 23 | -- We inherit write() and writebits() from Node unmodified - 24 | -- our parent takes care of passing the right context in. 25 | 26 | return Table 27 | -------------------------------------------------------------------------------- /compat1x.lua: -------------------------------------------------------------------------------- 1 | -- vstruct 1.x compatibility module 2 | 3 | local vstruct = require "vstruct" 4 | 5 | vstruct.WARN = true 6 | 7 | local function warning(f, level) 8 | return function(...) 9 | if vstruct.WARN then 10 | local info = debug.getinfo(level or 2) 11 | io.stderr:write(info.short_src..":"..info.currentline..": vstruct: call to legacy API\n") 12 | end 13 | return f(...) 14 | end 15 | end 16 | 17 | vstruct.unpack = warning(vstruct.read) 18 | vstruct.unpackvals = warning(vstruct.readvals) 19 | vstruct.pack = warning(vstruct.write) 20 | 21 | do 22 | local _compile = vstruct.compile 23 | function vstruct.compile(...) 24 | local obj = _compile(...) 25 | obj.unpack = warning(function(...) return obj:read(...) end) 26 | obj.pack = warning(function(...) return obj:write(...) end) 27 | return obj 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /cursor.lua: -------------------------------------------------------------------------------- 1 | -- cursor - a wrapper for strings that makes them look like files 2 | -- exports: seek read write 3 | -- read only supports numeric amounts 4 | 5 | local cursor = {} 6 | 7 | -- like fseek 8 | -- seeking past the end of the string is permitted 9 | -- reads will return EOF, writes will fill in the intermediate space with nuls 10 | -- seeking past the start of the string is a soft error 11 | function cursor:seek(whence, offset) 12 | self:flush() 13 | whence = whence or "cur" 14 | offset = offset or 0 15 | 16 | if whence == "set" then 17 | self.pos = offset 18 | elseif whence == "cur" then 19 | self.pos = self.pos + offset 20 | elseif whence == "end" then 21 | self.pos = #self.str + offset 22 | else 23 | error "bad argument #1 to seek" 24 | end 25 | 26 | if self.pos < 0 then 27 | self.pos = 0 28 | return nil,"attempt to seek prior to start of file" 29 | end 30 | 31 | return self.pos 32 | end 33 | 34 | -- read n bytes from the current position 35 | -- reads longer than the string can satisfy return as much as it can 36 | -- reads while the position is at the end return nil,"eof" 37 | function cursor:read(n) 38 | self:flush() 39 | 40 | if self.pos >= #self.str then 41 | return nil,"eof" 42 | end 43 | 44 | if n == "*a" then 45 | n = #self.str 46 | end 47 | 48 | local buf = self.str:sub(self.pos+1, self.pos + n) 49 | self.pos = math.min(self.pos + n, #self.str) 50 | 51 | return buf 52 | end 53 | 54 | -- write the contents of the buffer at the current position, overwriting 55 | -- any data already present 56 | -- if the write pointer is past the end of the string, also fill in the 57 | -- intermediate space with nuls 58 | -- Internally, this just appends it to an internal buffer which is added to 59 | -- the string when needed. 60 | function cursor:write(buf) 61 | table.insert(self.buf, buf) 62 | return self 63 | end 64 | 65 | function cursor:flush() 66 | if #self.buf == 0 then 67 | return 68 | end 69 | 70 | -- Pad end with \0 if we're writing past end of file 71 | if self.pos > #self.str then 72 | self.str = self.str .. string.char(0):rep(self.pos - #self.str) 73 | end 74 | 75 | -- Concatenate queued writes 76 | local buf = table.concat(self.buf) 77 | 78 | -- Append or splice into the string as needed 79 | self.str = self.str:sub(1, self.pos) 80 | .. buf 81 | .. self.str:sub(self.pos + #buf + 1, -1) 82 | 83 | self.pos = self.pos + #buf 84 | self.buf = {} 85 | end 86 | 87 | cursor.__index = cursor 88 | 89 | setmetatable(cursor, { 90 | __call = function(self, source) 91 | assert(type(source) == "string", "invalid first argument to cursor()") 92 | return setmetatable( 93 | { str = source, pos = 0, buf = {} }, 94 | cursor) 95 | end; 96 | }) 97 | 98 | return cursor 99 | -------------------------------------------------------------------------------- /frexp.lua: -------------------------------------------------------------------------------- 1 | -- math.frexp() replacement for Lua 5.3 when compiled without LUA_COMPAT_MATHLIB. 2 | -- The C approach is just to type-pun the float, but we can't do that here short 3 | -- of stupid loadstring() tricks, which would be both architecture and version 4 | -- dependent and a maintenance headache at best. So instead we use math. 5 | 6 | local abs,floor,log = math.abs,math.floor,math.log 7 | local log2 = log(2) 8 | 9 | return function(x) 10 | if x == 0 then return 0.0,0.0 end 11 | local e = floor(log(abs(x)) / log2) 12 | if e > 0 then 13 | -- Why not x / 2^e? Because for large-but-still-legal values of e this 14 | -- ends up rounding to inf and the wheels come off. 15 | x = x * 2^-e 16 | else 17 | x = x / 2^e 18 | end 19 | -- Normalize to the range [0.5,1) 20 | if abs(x) >= 1.0 then 21 | x,e = x/2,e+1 22 | end 23 | return x,e 24 | end 25 | -------------------------------------------------------------------------------- /init.lua: -------------------------------------------------------------------------------- 1 | -- vstruct, the versatile struct library 2 | 3 | local vstruct = {} 4 | package.loaded.vstruct = vstruct 5 | 6 | vstruct._NAME = "vstruct" 7 | vstruct._VERSION = "2.2.0" 8 | vstruct._M = vstruct 9 | 10 | vstruct.cursor = require "vstruct.cursor" 11 | local api = require "vstruct.api" 12 | local ast = require "vstruct.ast" 13 | 14 | local _unpack = unpack or table.unpack 15 | 16 | -- cache control for the parser 17 | -- true: cache is read/write (new formats will be cached, old ones retrieved) 18 | -- false: cache is read-only 19 | -- nil: cache is disabled 20 | vstruct.cache = true 21 | 22 | -- detect system endianness on startup 23 | require "vstruct.io" ("endianness", "probe") 24 | 25 | -- this is needed by some IO formats as well as init itself 26 | -- FIXME: should it perhaps be a vstruct internal function rather than 27 | -- installing it in math? 28 | if not math.trunc then 29 | function math.trunc(n) 30 | if n < 0 then 31 | return math.ceil(n) 32 | else 33 | return math.floor(n) 34 | end 35 | end 36 | end 37 | 38 | -- turn an int into a list of booleans 39 | -- the length of the list will be the smallest number of bits needed to 40 | -- represent the int 41 | function vstruct.explode(int, size) 42 | size = size or 0 43 | api.check_arg("vstruct.explode", 1, int, "number") 44 | api.check_arg("vstruct.explode", 2, size, "number") 45 | 46 | local mask = {} 47 | while int ~= 0 or #mask < size do 48 | table.insert(mask, int % 2 ~= 0) 49 | int = math.trunc(int/2) 50 | end 51 | return mask 52 | end 53 | 54 | -- turn a list of booleans into an int 55 | -- the converse of explode 56 | function vstruct.implode(mask, size) 57 | api.check_arg("vstruct.implode", 1, mask, "table") 58 | size = size or #mask 59 | api.check_arg("vstruct.implode", 2, size, "number") 60 | 61 | local int = 0 62 | for i=size,1,-1 do 63 | int = int*2 + ((mask[i] and 1) or 0) 64 | end 65 | return int 66 | end 67 | 68 | -- Return the size on disk of the structure described by the format string. 69 | -- If it can't be determined statically, returns nil. 70 | function vstruct.sizeof(fmt) 71 | api.check_arg("vstruct.sizeof", 1, fmt, "string") 72 | return api.compile(nil, fmt):sizeof() 73 | end 74 | 75 | -- Given a format string, a buffer or file, and an optional third argument, 76 | -- read data from the buffer or file according to the format string 77 | function vstruct.read(fmt, ...) 78 | api.check_arg("vstruct.read", 1, fmt, "string") 79 | return api.compile(nil, fmt):read(...) 80 | end 81 | 82 | function vstruct.readvals(...) 83 | return _unpack(vstruct.read(...)) 84 | end 85 | 86 | -- Given a format string, an optional file-like, and a table of data, 87 | -- write data into the file-like (or create and return a string of packed data) 88 | -- according to the format string 89 | function vstruct.write(fmt, ...) 90 | api.check_arg("vstruct.write", 1, fmt, "string") 91 | return api.compile(nil, fmt):write(...) 92 | end 93 | 94 | -- Given a format string, compile it and return a table containing the original 95 | -- source and the read/write functions derived from it. 96 | function vstruct.compile(name, fmt) 97 | api.check_arg("vstruct.compile", 1, name, "string") 98 | if fmt then 99 | api.check_arg("vstruct.compile", 2, fmt, "string") 100 | return api.compile(name, fmt) 101 | end 102 | return api.compile(nil, name) 103 | end 104 | 105 | -- Takes the same arguments as vstruct.unpack() 106 | -- returns an iterator over the input, repeatedly calling read until it runs 107 | -- out of data 108 | function vstruct.records(fmt, fd, unpacked) 109 | api.check_arg("vstruct.records", 1, fmt, "string") 110 | if unpacked ~= nil then 111 | api.check_arg("vstruct.records", 3, unpacked, "boolean") 112 | end 113 | 114 | local t = api.compile(nil, fmt) 115 | if type(fd) == "string" then 116 | fd = vstruct.cursor(fd) 117 | end 118 | 119 | return t:records(fd, unpacked) 120 | end 121 | 122 | -- Returns an array containing the results of vstruct.records, with an optional 123 | -- starting index. 124 | function vstruct.array(fmt, fd, n) 125 | n = n or 1 126 | local array = {} 127 | for record in vstruct.records(fmt, fd) do 128 | array[n] = record 129 | n = n+1 130 | end 131 | return array 132 | end 133 | 134 | return vstruct 135 | -------------------------------------------------------------------------------- /io.lua: -------------------------------------------------------------------------------- 1 | -- The loader for individual IO formats for vstruct. 2 | -- Returns the "io function", which is called with: 3 | -- * an IO type, such as 'x' or 'c' 4 | -- * an IO operation, such as 'write' or 'size' 5 | -- * some (type x operation) specific other arguments 6 | -- upon which it will attempt to load the handler for that operation from the 7 | -- module vstruct.io. and call it. 8 | 9 | local defaults = require "vstruct.io.defaults" 10 | local mt = { __index = defaults } 11 | 12 | local function iorequire(format) 13 | local r,v = pcall(require, "vstruct.io."..format) 14 | 15 | if not r then 16 | error("vstruct: no support for format '"..format.."':\n"..tostring(v)) 17 | end 18 | 19 | setmetatable(v, mt) 20 | 21 | return v 22 | end 23 | 24 | local controlnames = { 25 | seekf = "+"; 26 | seekb = "-"; 27 | seekto = "@"; 28 | bigendian = ">"; 29 | littleendian= "<"; 30 | hostendian = "="; 31 | } 32 | 33 | for name,symbol in pairs(controlnames) do 34 | package.preload["vstruct.io."..symbol] = function() return iorequire(name) end 35 | end 36 | 37 | return function(format, method, ...) 38 | local fmt = iorequire(format) 39 | 40 | assert(fmt[method], "No support for method '"..tostring(method).."' in IO module '"..format.."'") 41 | 42 | return fmt[method](...) 43 | end 44 | -------------------------------------------------------------------------------- /io/a.lua: -------------------------------------------------------------------------------- 1 | -- align-to 2 | 3 | local io = require "vstruct.io" 4 | local a = {} 5 | 6 | function a.hasvalue() 7 | return false 8 | end 9 | 10 | function a.size(w) 11 | assert(w, "format requires a size") 12 | return nil 13 | end 14 | 15 | function a.read(fd, _, align) 16 | local cur = fd:seek() 17 | 18 | if cur % align ~= 0 then 19 | fd:seek("cur", align - (cur % align)) 20 | end 21 | end 22 | 23 | a.write = a.read 24 | 25 | return a 26 | -------------------------------------------------------------------------------- /io/b.lua: -------------------------------------------------------------------------------- 1 | -- boolean 2 | 3 | local io = require "vstruct.io" 4 | local b = {} 5 | 6 | function b.read(_, buf) 7 | return (buf:match("%Z") and true) or false 8 | end 9 | 10 | function b.readbits(bit, size) 11 | local n = 0 12 | for i=1,size do 13 | n = n + bit() 14 | end 15 | return n > 0 16 | end 17 | 18 | function b.write(_, data, size) 19 | return io("u", "write", nil, data and 1 or 0, size) 20 | end 21 | 22 | function b.writebits(bit, data, size) 23 | for i=1,size do 24 | bit(data and 1 or 0) 25 | end 26 | end 27 | 28 | return b 29 | -------------------------------------------------------------------------------- /io/bigendian.lua: -------------------------------------------------------------------------------- 1 | -- > set endianness: big 2 | 3 | local io = require "vstruct.io" 4 | local be = {} 5 | 6 | function be.hasvalue() 7 | return false 8 | end 9 | 10 | function be.size(n) 11 | assert(n == nil, "'>' is an endianness control, and does not have size") 12 | return 0 13 | end 14 | 15 | function be.read() 16 | io("endianness", "big") 17 | end 18 | 19 | be.write = be.read 20 | 21 | return be 22 | -------------------------------------------------------------------------------- /io/c.lua: -------------------------------------------------------------------------------- 1 | -- counted strings 2 | 3 | local io = require "vstruct.io" 4 | local c = {} 5 | 6 | function c.size(w) 7 | assert(w, "format requires a size") 8 | return nil 9 | end 10 | 11 | function c.read(fd, _, size) 12 | assert(size) 13 | local buf = fd:read(size) 14 | local len = io("u", "read", nil, buf, size) 15 | if len == 0 then 16 | return "" 17 | end 18 | return fd:read(len) 19 | end 20 | 21 | function c.write(fd, data, size) 22 | return io("u", "write", nil, #data, size) 23 | .. io("s", "write", nil, data) 24 | end 25 | 26 | return c 27 | -------------------------------------------------------------------------------- /io/defaults.lua: -------------------------------------------------------------------------------- 1 | -- default implementations for some of the IO sub-operations - if a format 2 | -- doesn't provide its own, these will be used 3 | 4 | local defaults = {} 5 | 6 | -- verifies the size given, returns the number of bytes that will actually 7 | -- be read or written. Default assumes that size is mandatory and exactly 8 | -- equals the amount that will be RWd. 9 | -- If the format doesn't do any, should return 0. If the amount cannot be 10 | -- determined ahead of time (for example, format c) should return nil. 11 | -- If the caller specified a constant size, it will be passsed in here; if they 12 | -- specified a runtime-determined size (via backreferences), it will be passed 13 | -- in at runtime, but will be `true` at compile time. Implementations should 14 | -- respect both. 15 | function defaults.size(n) 16 | assert(n, "format requires a size") 17 | return tonumber(n) 18 | end 19 | 20 | function defaults.validate() 21 | return true 22 | end 23 | 24 | function defaults.hasvalue() 25 | return true 26 | end 27 | 28 | return defaults 29 | -------------------------------------------------------------------------------- /io/endianness.lua: -------------------------------------------------------------------------------- 1 | -- the actual endianness controls. These should not be used directly, but 2 | -- are instead invoked by the <=> formats (bigendian, littleendian, and 3 | -- hostendian) to do the actual work. 4 | 5 | -- FIXME: endianness flag should be operation-local rather than VM-local; at 6 | -- present writeread operations are atomic, but if in the future they are 7 | -- not - for example, if an io is implemented that uses coroutines - the current 8 | -- implementation will fuck us right up. 9 | 10 | local e = {} 11 | 12 | local endianness; 13 | 14 | function e.hasvalue() 15 | return false 16 | end 17 | 18 | function e.big() 19 | endianness = "big" 20 | return endianness 21 | end 22 | 23 | function e.little() 24 | endianness = "little" 25 | return endianness 26 | end 27 | 28 | -- select whichever endianness the host system uses 29 | local bigendian 30 | function e.probe() 31 | -- if we're running in luajit, we can just query the FFI library 32 | if jit then 33 | bigendian = require("ffi").abi("be") 34 | 35 | -- if we're not, we dump an empty function and see if the first byte is nul 36 | -- or not. HACK HACK HACK - this is unlikely to work in anything but the 37 | -- reference implementation. 38 | elseif string.dump then 39 | bigendian = string.byte(string.dump(function() end), 7) == 0x00 40 | 41 | -- if neither jit nor string.dump is available, we guess wildly that it's 42 | -- a little-endian system (and emit a warning) 43 | else 44 | io.stderr:write("[vstruct] Warning: can't determine endianness of host system, assuming litle-endian\n") 45 | bigendian = false 46 | end 47 | end 48 | 49 | function e.host() 50 | if bigendian then 51 | return e.big() 52 | else 53 | return e.little() 54 | end 55 | end 56 | 57 | function e.get() 58 | return endianness 59 | end 60 | 61 | return e 62 | -------------------------------------------------------------------------------- /io/f.lua: -------------------------------------------------------------------------------- 1 | -- IEEE floating point floats, doubles and quads 2 | 3 | local struct = require "vstruct" 4 | local io = require "vstruct.io" 5 | local unpack = table.unpack or unpack 6 | local frexp = math.frexp or require "vstruct.frexp" 7 | 8 | local sizes = { 9 | [4] = {1, 8, 23}; 10 | [8] = {1, 11, 52}; 11 | [16] = {1, 15, 112}; 12 | } 13 | 14 | local function reader(data, size_exp, size_fraction) 15 | local fraction, exponent, sign 16 | local endian = io("endianness", "get") == "big" and ">" or "<" 17 | 18 | -- Split the unsigned integer into the 3 IEEE fields 19 | local bits = struct.read(endian.." m"..#data, data)[1] 20 | local fraction = struct.implode({unpack(bits, 1, size_fraction)}, size_fraction) 21 | local exponent = struct.implode({unpack(bits, size_fraction+1, size_fraction+size_exp)}, size_exp) 22 | local sign = bits[#bits] and -1 or 1 23 | 24 | -- special case: exponent is all 1s 25 | if exponent == 2^size_exp-1 then 26 | -- significand is 0? +- infinity 27 | if fraction == 0 then 28 | return sign * math.huge 29 | 30 | -- otherwise it's NaN 31 | else 32 | return 0/0 33 | end 34 | end 35 | 36 | -- restore the MSB of the significand, unless it's a subnormal number 37 | if exponent ~= 0 then 38 | fraction = fraction + (2 ^ size_fraction) 39 | else 40 | exponent = 1 41 | end 42 | 43 | -- remove the exponent bias 44 | exponent = exponent - 2 ^ (size_exp - 1) + 1 45 | 46 | -- Decrease the size of the exponent rather than make the fraction (0.5, 1] 47 | exponent = exponent - size_fraction 48 | 49 | return sign * (fraction * 2.0^exponent) 50 | end 51 | 52 | local function writer(value, size_exp, size_fraction) 53 | local fraction, exponent, sign 54 | local size = (size_exp + size_fraction + 1)/8 55 | local endian = io("endianness", "get") == "big" and ">" or "<" 56 | local bias = 2^(size_exp-1)-1 57 | 58 | if value < 0 59 | or 1/value == -math.huge then -- handle the case of -0 60 | sign = true 61 | value = -value 62 | else 63 | sign = false 64 | end 65 | 66 | -- special case: value is infinite 67 | if value == math.huge then 68 | exponent = bias+1 69 | fraction = 0 70 | 71 | -- special case: value is NaN 72 | elseif value ~= value then 73 | exponent = bias+1 74 | fraction = 2^(size_fraction-1) 75 | 76 | --special case: value is 0 77 | elseif value == 0 then 78 | exponent = -bias 79 | fraction = 0 80 | 81 | else 82 | fraction,exponent = frexp(value) 83 | 84 | -- subnormal number 85 | if exponent+bias <= 1 then 86 | fraction = fraction * 2^(size_fraction+(exponent+bias)-1) 87 | exponent = -bias 88 | 89 | else 90 | -- remove the most significant bit from the fraction and adjust exponent 91 | fraction = fraction - 0.5 92 | exponent = exponent - 1 93 | 94 | -- turn the fraction into an integer 95 | fraction = fraction * 2^(size_fraction+1) 96 | end 97 | end 98 | 99 | 100 | -- add the exponent bias 101 | exponent = exponent + bias 102 | 103 | local bits = struct.explode(fraction) 104 | local bits_exp = struct.explode(exponent) 105 | for i=1,size_exp do 106 | bits[size_fraction+i] = bits_exp[i] 107 | end 108 | bits[size_fraction+size_exp+1] = sign 109 | 110 | return struct.write(endian.."m"..size, {bits}) 111 | end 112 | 113 | local f = {} 114 | 115 | function f.size(n) 116 | assert(n == 4 or n == 8 or n == 16 117 | , "format 'f' only supports sizes 4 (float), 8 (double) and 16 (quad)") 118 | 119 | return n 120 | end 121 | 122 | function f.read(_, buf, size) 123 | return reader(buf, unpack(sizes[size], 2)) 124 | end 125 | 126 | function f.write(_, data, size) 127 | return writer(data, unpack(sizes[size], 2)) 128 | end 129 | 130 | return f 131 | -------------------------------------------------------------------------------- /io/hostendian.lua: -------------------------------------------------------------------------------- 1 | -- = set endianness to same as host system 2 | 3 | local io = require "vstruct.io" 4 | local he = {} 5 | 6 | function he.hasvalue() 7 | return false 8 | end 9 | 10 | function he.size(n) 11 | assert(n == nil, "'=' is an endianness control, and does not have size") 12 | return 0 13 | end 14 | 15 | function he.read() 16 | io("endianness", "host") 17 | end 18 | 19 | he.write = he.read 20 | 21 | return he 22 | -------------------------------------------------------------------------------- /io/i.lua: -------------------------------------------------------------------------------- 1 | -- signed integers 2 | 3 | local io = require "vstruct.io" 4 | local i = {} 5 | 6 | function i.read(fd, buf, size) 7 | local n = io("u", "read", fd, buf, size) 8 | 9 | if n >= 2^(size*8-1) then 10 | return n - 2^(size*8) 11 | end 12 | 13 | return n 14 | end 15 | 16 | function i.readbits(bit, size) 17 | local n = io("u", "readbits", bit, size) 18 | 19 | if n >= 2^(size-1) then 20 | return n - 2^size 21 | end 22 | 23 | return n 24 | end 25 | 26 | function i.write(_, data, size) 27 | data = math.trunc(data) 28 | 29 | if data < 0 then 30 | data = data + 2^(size*8) 31 | end 32 | 33 | return io("u", "write", _, data, size) 34 | end 35 | 36 | function i.writebits(bit, data, size) 37 | if data < 0 then 38 | data = data + 2^size 39 | end 40 | 41 | return io("u", "writebits", bit, data, size) 42 | end 43 | 44 | return i 45 | -------------------------------------------------------------------------------- /io/littleendian.lua: -------------------------------------------------------------------------------- 1 | -- < set endianness: little 2 | 3 | local io = require "vstruct.io" 4 | local le = {} 5 | 6 | function le.hasvalue() 7 | return false 8 | end 9 | 10 | function le.size(n) 11 | assert(n == nil, "'<' is an endianness control, and does not have size") 12 | return 0 13 | end 14 | 15 | function le.read() 16 | io("endianness", "little") 17 | end 18 | 19 | le.write = le.read 20 | 21 | return le 22 | -------------------------------------------------------------------------------- /io/m.lua: -------------------------------------------------------------------------------- 1 | -- bitmasks 2 | 3 | local struct = require "vstruct" 4 | local io = require "vstruct.io" 5 | local unpack = table.unpack or unpack 6 | local m = {} 7 | 8 | function m.read(_, buf, size) 9 | local mask = {} 10 | local e = io("endianness", "get") 11 | 12 | local sof,eof,step 13 | if e == "big" then 14 | sof,eof,step = #buf,1,-1 15 | else 16 | sof,eof,step = 1,#buf,1 17 | end 18 | 19 | for i=sof,eof,step do 20 | local byte = buf:byte(i) 21 | for i=1,8 do 22 | mask[#mask+1] = (byte % 2 == 1) and true or false 23 | byte = math.floor(byte/2) 24 | end 25 | end 26 | 27 | return mask 28 | end 29 | 30 | function m.readbits(bit, size) 31 | local mask = {} 32 | for i=1,size do 33 | mask[i] = bit() == 1 and true or false 34 | end 35 | return mask 36 | end 37 | 38 | -- bitmask 39 | -- we use a string here because using an unsigned will lose data on bitmasks 40 | -- wider than lua's native number format 41 | function m.write(fd, data, size) 42 | local buf = "" 43 | local e = io("endianness", "get") 44 | 45 | for i=1,size*8,8 do 46 | local bits = { unpack(data, i, i+7) } 47 | local byte = string.char(struct.implode(bits, 8)) 48 | if e == "big" then 49 | buf = byte..buf 50 | else 51 | buf = buf..byte 52 | end 53 | end 54 | return io("s", "write", fd, buf, size) 55 | end 56 | 57 | function m.writebits(bit, data, size) 58 | for i=1,size do 59 | bit(data[i] and 1 or 0) 60 | end 61 | end 62 | 63 | return m 64 | -------------------------------------------------------------------------------- /io/p.lua: -------------------------------------------------------------------------------- 1 | -- signed fixed point 2 | -- format is pTOTAL_SIZE,FRACTIONAL_SIZE 3 | -- Fractional size is in bits, total size in bytes. 4 | -- FIXME: this should support bitpacks 5 | 6 | local io = require "vstruct.io" 7 | local p = {} 8 | 9 | function p.size(size, frac) 10 | assert(size, "format requires a size") 11 | assert(frac, "format requires a fractional-part size") 12 | if tonumber(size) and tonumber(frac) then 13 | -- Check only possible if both values were specified at compile time 14 | assert(size*8 >= frac, "fixed point number has more fractional bits than total bits") 15 | end 16 | 17 | return size 18 | end 19 | 20 | function p.read(fd, buf, size, frac) 21 | return io("i", "read", fd, buf, size)/(2^frac) 22 | end 23 | 24 | function p.write(fd, data, size, frac) 25 | return io("i", "write", fd, data * 2^frac, size) 26 | end 27 | 28 | return p 29 | -------------------------------------------------------------------------------- /io/pu.lua: -------------------------------------------------------------------------------- 1 | -- signed fixed point 2 | -- format is pTOTAL_SIZE,FRACTIONAL_SIZE 3 | -- Fractional size is in bits, total size in bytes. 4 | -- FIXME: this should support bitpacks 5 | 6 | local io = require "vstruct.io" 7 | local pu = {} 8 | 9 | function pu.size(size, frac) 10 | assert(size, "format requires a size") 11 | assert(frac, "format requires a fractional-part size") 12 | if tonumber(size) and tonumber(frac) then 13 | -- Check only possible if both values were specified at compile time 14 | assert(size*8 >= frac, "fixed point number has more fractional bits than total bits") 15 | end 16 | 17 | return size 18 | end 19 | 20 | function pu.read(fd, buf, size, frac) 21 | return io("u", "read", fd, buf, size)/(2^frac) 22 | end 23 | 24 | function pu.write(fd, data, size, frac) 25 | return io("u", "write", fd, data * 2^frac, size) 26 | end 27 | 28 | return pu 29 | -------------------------------------------------------------------------------- /io/s.lua: -------------------------------------------------------------------------------- 1 | -- fixed length strings 2 | 3 | local io = require "vstruct.io" 4 | local s = {} 5 | 6 | function s.size(w) 7 | return tonumber(w) 8 | end 9 | 10 | function s.read(fd, buf, size) 11 | if buf then 12 | assert(#buf == size, "sanity failure: length of buffer does not match length of string format") 13 | return buf 14 | end 15 | 16 | return fd:read(size or '*a') 17 | end 18 | 19 | function s.write(_, data, size) 20 | size = size or #data 21 | if size > #data then 22 | data = data..string.rep("\0", size - #data) 23 | end 24 | return data:sub(1,size) 25 | end 26 | 27 | return s 28 | -------------------------------------------------------------------------------- /io/seekb.lua: -------------------------------------------------------------------------------- 1 | -- - seek backward a constant amount 2 | 3 | local seek = {} 4 | 5 | function seek.hasvalue() 6 | return false 7 | end 8 | 9 | function seek.size(w) 10 | assert(w, "format requires a size") 11 | return nil 12 | end 13 | 14 | function seek.read(fd, _, offset) 15 | assert(fd:seek("cur", -offset)) 16 | end 17 | seek.write = seek.read 18 | 19 | return seek 20 | -------------------------------------------------------------------------------- /io/seekf.lua: -------------------------------------------------------------------------------- 1 | -- + seek forward a constant amount 2 | 3 | local seek = {} 4 | 5 | function seek.hasvalue() 6 | return false 7 | end 8 | 9 | function seek.size(w) 10 | assert(w, "format requires a size") 11 | return nil 12 | end 13 | 14 | function seek.read(fd, _, offset) 15 | assert(fd:seek("cur", offset)) 16 | end 17 | seek.write = seek.read 18 | 19 | return seek 20 | -------------------------------------------------------------------------------- /io/seekto.lua: -------------------------------------------------------------------------------- 1 | -- @ seek to a constant offset 2 | 3 | local seek = {} 4 | 5 | function seek.hasvalue() 6 | return false 7 | end 8 | 9 | function seek.size(w) 10 | assert(w, "format requires a size") 11 | return nil 12 | end 13 | 14 | function seek.read(fd, _, offset) 15 | assert(fd:seek("set", offset)) 16 | end 17 | seek.write = seek.read 18 | 19 | return seek 20 | -------------------------------------------------------------------------------- /io/u.lua: -------------------------------------------------------------------------------- 1 | -- unsigned ints 2 | 3 | local io = require "vstruct.io" 4 | local u = {} 5 | 6 | function u.read(_, buf) 7 | local n = 0 8 | local e = io("endianness", "get") 9 | 10 | local sof,eof,step 11 | if e == "big" then 12 | sof,eof,step = 1,#buf,1 13 | else 14 | sof,eof,step = #buf,1,-1 15 | end 16 | 17 | for i=sof,eof,step do 18 | n = n * 256 + buf:byte(i,i) 19 | end 20 | 21 | return n 22 | end 23 | 24 | function u.readbits(bit, size) 25 | local n = 0 26 | for i=1,size do 27 | n = n * 2 + bit() 28 | end 29 | return n 30 | end 31 | 32 | function u.write(_, data, size) 33 | local s = "" 34 | local e = io("endianness", "get") 35 | data = math.trunc(data) 36 | 37 | for i=1,size do 38 | if e == "big" then 39 | s = string.char(data % 256) .. s 40 | else 41 | s = s .. string.char(data % 256) 42 | end 43 | data = math.trunc(data/256) 44 | end 45 | 46 | return s 47 | end 48 | 49 | function u.writebits(bit, data, size) 50 | for i=size-1,0,-1 do 51 | bit(math.floor(data/2^i) % 2) 52 | end 53 | end 54 | 55 | return u 56 | -------------------------------------------------------------------------------- /io/x.lua: -------------------------------------------------------------------------------- 1 | -- skip/pad 2 | -- unlike the seek controls @+- or the alignment control a, x will never call 3 | -- seek, and instead uses write '\0' or read-and-ignore - this means it is 4 | -- safe to use on streams. 5 | 6 | local io = require "vstruct.io" 7 | local x = {} 8 | 9 | function x.hasvalue() 10 | return false 11 | end 12 | 13 | function x.read(fd, buf, size) 14 | io("s", "read", fd, buf, size) 15 | return nil 16 | end 17 | 18 | function x.readbits(bit, size) 19 | for i=1,size do 20 | bit() 21 | end 22 | end 23 | 24 | function x.writebits(bit, _, size, val) 25 | val = val or 0 26 | assert(val == 0 or val == 1, "invalid value to `x` format in bitpack: 0 or 1 required, got "..val) 27 | for i=1,size do 28 | bit(val or 0) 29 | end 30 | end 31 | 32 | function x.write(fd, data, size, val) 33 | return string.rep(string.char(val or 0), size) 34 | end 35 | 36 | return x 37 | -------------------------------------------------------------------------------- /io/z.lua: -------------------------------------------------------------------------------- 1 | -- null-terminated strings 2 | 3 | local io = require "vstruct.io" 4 | local z = {} 5 | 6 | function z.size(size, csize) 7 | return tonumber(size) 8 | end 9 | 10 | -- null terminated string 11 | -- w==nil is write string as is + termination 12 | -- w>0 is write exactly w bytes, truncating/padding and terminating 13 | 14 | function z.write(_, data, size, csize) 15 | csize = csize or 1 16 | size = size or #data+csize 17 | 18 | assert(size % csize == 0, "string length is not a multiple of character size") 19 | 20 | -- truncate to field size 21 | if #data >= size then 22 | data = data:sub(1, size-csize) 23 | end 24 | 25 | return io("s", "write", _, data..("\0"):rep(csize), size) 26 | end 27 | 28 | -- null-terminated string 29 | -- if w is omitted, reads up to and including the first nul, and returns everything 30 | -- except that nul; WARNING: SLOW 31 | -- otherwise, reads exactly w bytes and returns everything up to the first nul 32 | function z.read(fd, buf, size, csize) 33 | csize = csize or 1 34 | nul = ("\0"):rep(csize) 35 | 36 | -- read exactly that many characters, then strip the null termination 37 | if size then 38 | local buf = io("s", "read", fd, buf, size) 39 | local len = 0 40 | 41 | -- search the string for the null terminator. If charsize > 1, just 42 | -- finding nul isn't good enough - it needs to be aligned on a character 43 | -- boundary. 44 | repeat 45 | len = buf:find(nul, len+1, true) 46 | until len == nil or (len-1) % csize == 0 47 | 48 | return buf:sub(1,(len or 0)-1) 49 | end 50 | 51 | -- this is where it gets ugly: the size wasn't specified, so we need to 52 | -- read (csize) bytes at a time looking for the null terminator 53 | local chars = {} 54 | local c = fd:read(csize) 55 | while c and c ~= nul do 56 | chars[#chars+1] = c 57 | c = fd:read(csize) 58 | end 59 | 60 | return table.concat(chars) 61 | end 62 | 63 | return z 64 | -------------------------------------------------------------------------------- /lexer.lua: -------------------------------------------------------------------------------- 1 | -- Implements the lexer for vstruct format strings. 2 | -- Returns a function which, given a source string, returns a table of lexer 3 | -- operations closed over that source. 4 | 5 | local lexis = {} 6 | 7 | local function lexeme(name) 8 | return function(pattern) 9 | lexis[#lexis+1] = { name=name, pattern="^"..pattern } 10 | end 11 | end 12 | 13 | lexeme (false) "%s+" -- whitespace 14 | lexeme (false) "%-%-[^\n]*" -- comments 15 | lexeme "key" "([%a_][%w_.]*):" 16 | lexeme "io" "([-+@<>=])" 17 | lexeme "io" "([%a_]+)" 18 | lexeme "number" "([%d.,]+)" -- may be multiple comma-separated numbers 19 | lexeme "number" "(#[%a_][%w_.]*)" 20 | lexeme "splice" "&(%S+)" 21 | lexeme "{" "%{" 22 | lexeme "}" "%}" 23 | lexeme "(" "%(" 24 | lexeme ")" "%)" 25 | lexeme "*" "%*" 26 | lexeme "[" "%[" 27 | lexeme "]" "%]" 28 | lexeme "|" "%|" 29 | 30 | return function(source) 31 | local orig = source 32 | local index = 1 33 | local hadwhitespace = false 34 | 35 | local function where() 36 | return ("character %d ('%s')"):format(index, source:sub(1,4)) 37 | end 38 | 39 | local function find_match() 40 | for _,lexeme in ipairs(lexis) do 41 | if source:match(lexeme.pattern) then 42 | return lexeme,select(2, source:find(lexeme.pattern)) 43 | end 44 | end 45 | error (("Lexical error in format string at %s."):format(where())) 46 | end 47 | 48 | local function eat_whitespace() 49 | local function aux() 50 | if #source == 0 then return end 51 | local match,size = find_match() 52 | 53 | if not match.name then 54 | hadwhitespace = true 55 | source = source:sub(size+1, -1) 56 | index = index + size 57 | return aux() 58 | end 59 | end 60 | hadwhitespace = false 61 | return aux() 62 | end 63 | 64 | local function whitespace() 65 | return hadwhitespace 66 | end 67 | 68 | local function next() 69 | eat_whitespace() 70 | 71 | if #source == 0 then return { text = nil, type = "EOF" } end 72 | 73 | local lexeme,size,text = find_match() 74 | 75 | source = source:sub(size+1, -1) 76 | index = index+size 77 | 78 | return { text = text, type = lexeme.name } 79 | end 80 | 81 | local function peek() 82 | eat_whitespace() 83 | 84 | if #source == 0 then return { text = nil, type = "EOF" } end 85 | 86 | local lexeme,size,text = find_match() 87 | 88 | return { text = text, type = lexeme.name } 89 | end 90 | 91 | return { 92 | next = next; 93 | peek = peek; 94 | where = where; 95 | whitespace = whitespace; 96 | tokens = function() return next end; 97 | } 98 | end 99 | 100 | -------------------------------------------------------------------------------- /test.lua: -------------------------------------------------------------------------------- 1 | -- top-level test module for vstruct 2 | -- run with 'lua test.lua' or, if vstruct is installed, with 3 | -- 'lua -lvstruct.test -e ""' 4 | 5 | -- Runs a "sensible default" set of tests against the vstruct module. Exits 6 | -- cleanly if all of them passed; if any failed, reports the failed tests 7 | -- on stdout and then raises an error. 8 | 9 | if arg and arg[1] == "DISABLE_FREXP" then 10 | table.remove(arg, 1) 11 | math.frexp = nil 12 | end 13 | 14 | local ok,test = pcall(require, "vstruct.test.common") 15 | 16 | -- maybe we aren't installed, and just need to deduce a custom package.path 17 | -- from the location of this file 18 | -- if arg is undefined, we were loaded with require and can draw no conclusions 19 | if ok then 20 | -- loaded successfully, no error handling needed 21 | elseif arg then 22 | -- see if we can figure out where we were loaded from 23 | local libdir = arg[0]:gsub("[^/\\]+$", "").."../" 24 | package.path = package.path..";"..libdir.."?.lua;"..libdir.."?/init.lua" 25 | 26 | -- clear "failure" entries in package.path 27 | package.loaded["vstruct.test.common"] = nil 28 | package.loaded["vstruct"] = nil 29 | 30 | -- retry 31 | test = require "vstruct.test.common" 32 | else 33 | error(test) 34 | end 35 | 36 | require "vstruct.test.basic" 37 | require "vstruct.test.compat1x" 38 | require "vstruct.test.fp-bigendian" 39 | require "vstruct.test.fp-littleendian" 40 | require "vstruct.test.error" 41 | require "vstruct.test.frexp" 42 | require "vstruct.test.regression" 43 | 44 | if arg and #arg > 0 then 45 | require "vstruct.test.struct-test-gen" 46 | else 47 | arg = { "NROF_TESTS=2^10", "read", "write" } 48 | require "vstruct.test.struct-test-gen" 49 | end 50 | 51 | if test.report() > 0 then 52 | error("Some test cases failed; see preceding output for details.") 53 | end 54 | -------------------------------------------------------------------------------- /test/basic.lua: -------------------------------------------------------------------------------- 1 | -- "basic" test module for vstruct 2 | -- Tests a variety of non-floating-point usage. 3 | -- Does not test floating point operations - see test/fp-*endian.lua for those, 4 | -- since there are quite a lot of them. 5 | 6 | local vstruct = require "vstruct" 7 | local test = require "vstruct.test.common" 8 | 9 | local x = test.x 10 | local T = test.autotest 11 | 12 | test.group "basic tests" 13 | 14 | T("true", "> b8", x"0000 0000 0000 0001", true) 15 | T("false", "> b8", x"0000 0000 0000 0000", false) 16 | 17 | T("unsigned", "< u3", x"FE FF FF", 2^24-2) 18 | T("signed", "< i3", x"FE FF FF", -2) 19 | 20 | T("c str", "z", "foobar\0baz", "foobar", "foobar\0") 21 | T("padded str", "z10", "foobar\0baz", "foobar", "foobar\0\0\0\0") 22 | 23 | T("fixed str", "s4", "foobar", "foob", "foob") 24 | 25 | T("counted str","< c4", x"06000000".."foobar", "foobar") 26 | 27 | T("bitmask", "> m1", x"FA", {{ false, true, false, true, true, true, true, true }}) 28 | T("bitmask", "> m2", x"FA 78", {{ false, false, false, true; true, true, true, false; false, true, false, true; true, true, true, true }}) 29 | T("bitmask", "< m2", x"78 FA", {{ false, false, false, true; true, true, true, false; false, true, false, true; true, true, true, true }}) 30 | 31 | T("skip/pad", "x4u1", x"00 00 00 00 02", 2) 32 | T("skip/pad value", "x4,255u1", x"FF FF FF FF 02", 2) 33 | 34 | T("seek @", "@2 u1x2", x"00 00 02 00 00", 2) 35 | T("seek +", "+2 u1x2", x"00 00 02 00 00", 2) 36 | T("seek -", "+4-2 u1x2",x"00 00 02 00 00", 2) 37 | 38 | T("little-endian", "< u2", x"00 01", 256) 39 | T("big-endian", "> u2", x"00 01", 1) 40 | 41 | if test.bigendian() then 42 | T("host-endian","= u2", x"01 00", 256) 43 | T("endianness leak (1)", "< u2", x"00 01", 256) 44 | T("endianness leak (2)", "u2", x"00 01", 1) 45 | else 46 | T("host-endian","= u2", x"01 00", 1) 47 | T("endianness leak (1)", "> u2", x"00 01", 1) 48 | T("endianness leak (2)", "u2", x"00 01", 256) 49 | end 50 | 51 | T("bitpack [>b]", "> [2| x15 b1 ]", x"00 01", true) 52 | T("bitpack [u]", "> [2| 4*u4 ]", x"12 34", {1,2,3,4}) 54 | T("bitpack [i]", "> [2| 4*i4 ]", x"12 EF", {1,2,-2,-1}) 56 | T("bitpack [m]", "> [2| m5 m3 x8 ]", x"12 00", {{false,false,false,true,false},{false,true,false}}) 58 | T("bitpack [ [2| x8,1 x8,0 ]", x"FF 00", {}) 60 | 61 | T("fixed point >", "> p2,8", x"40 80", 64.5) 62 | T("fixed point <", "< p2,8", x"40 80", -127.75) 63 | 64 | T("repetition", "> 4*u1", x"01 02 03 04", { 1, 2, 3, 4 }) 65 | T("groups", "> 2*(u1 i1)", x"01 FF 02 FE", { 1, -1, 2, -2 }) 66 | T("tables", "> 2*{ u1 i1 }", x"01 FF 02 FE", { { 1, -1 }, { 2, -2 } }) 67 | T("names", "> coords:{ x:u1 y:u1 } coords.z:u1", x"01 02 03", { coords = { x = 1, y = 2, z = 3 } }) 68 | 69 | T("backref-repeat", "count:u1 #count*u1", x"04 80 FF 11 22", { count = 4; 128, 255, 17, 34 }) 70 | T("backref-seek", "offset:u1 @#offset u1", 71 | x"02 00 03 00 00", { offset = 2; 3 }, x"02 00 03") 72 | T("backref-str", "< size:u2 str:s#size", 73 | "\x10\x00abcdefghijklmnopqrstuvwxyz", { size = 16; str = "abcdefghijklmnop"; }, 74 | "\x10\x00abcdefghijklmnop") 75 | 76 | T("scoping", "outer:{ size:u1 s#size inner:{ size:u1 s#size } } outer.inner.size:u1", 77 | "\x0Aouter-text\x0Cinner-string\x0C", 78 | { outer = { size = 10; "outer-text"; inner = { size = 12; "inner-string" }}}) 79 | 80 | T("nesting", "header:{ size:u1 offset:u1 } @#header.offset s#header.size", 81 | "\x04\x06 text", { header = { size = 4; offset = 6; }; "text" }, 82 | "\x04\x06\x00\x00\x00\x00text") 83 | 84 | T("UCS-2 z", "> z,2", x"0061 0062 0000 FFFF", "\0a\0b", x"0061 0062 0000") 85 | T("UCS-2 z8", "> z8,2", x"0061 0062 0000 FFFF", "\0a\0b", x"0061 0062 0000 0000") 86 | 87 | T("repeated repeat", "2*2*u1", x"01 01 01 01", { 1, 1, 1, 1 }) 88 | 89 | local sizeof = { 90 | ["u4 b1 x2"] = 7; 91 | ["c4"] = false; 92 | ["z"] = false; 93 | ["z64"] = 64; 94 | ["> coords:{ x:u1 y:u1 } coords.z:u1"] = 3; 95 | ["@2 u1x2"] = false; 96 | ["< size:u2 str:s#size"] = false; 97 | } 98 | 99 | for format,expected_size in pairs(sizeof) do 100 | local actual_size = vstruct.sizeof(format) or false 101 | test.record("sizeof", actual_size == expected_size, 102 | string.format("[%s]: %s != %s", format, expected_size, actual_size)) 103 | end 104 | 105 | local i = 1 106 | for val in vstruct.records("u1", x"01 02 03 04 05 06", true) do 107 | test.record("stream-unpacked #"..i, val == i, val) 108 | i = i+1 109 | end 110 | 111 | local i = 1 112 | for val in vstruct.records("i:u1", x"01 02 03 04 05 06") do 113 | test.record("stream #"..i, val.i == i, val.i) 114 | i = i+1 115 | end 116 | 117 | vstruct.compile("coord", "x:u1 y:u1 z:u1") 118 | T("splice", "> position:{ &coord }", x"01 02 03", { position = { x = 1, y = 2, z = 3 } }) 119 | -------------------------------------------------------------------------------- /test/common.lua: -------------------------------------------------------------------------------- 1 | local vstruct = require "vstruct" 2 | 3 | vstruct.cache = nil 4 | 5 | local char,ord = string.char,string.byte 6 | local unpack = table.unpack or unpack 7 | 8 | local test = { results = {} } 9 | 10 | function test.x(str) 11 | return (str:gsub("%X", ""):gsub("%x%x", function(b) return char(tonumber(b, 16)) end)) 12 | end 13 | 14 | function test.od(str, sep) 15 | local fmt = "%02X"..(sep or " ") 16 | return (str:gsub('.', function(c) return fmt:format(c:byte()) end)) 17 | end 18 | 19 | function test.eq(x, y) 20 | if type(x) ~= type(y) then return false end 21 | 22 | if type(x) == 'table' then 23 | for k,v in pairs(x) do 24 | if not test.eq(v, y[k]) then return false end 25 | end 26 | for k,v in pairs(y) do 27 | if not test.eq(v, x[k]) then return false end 28 | end 29 | return true 30 | end 31 | 32 | return x == y 33 | end 34 | 35 | function test.group(name) 36 | local group = { name=name } 37 | table.insert(test.results, group) 38 | test.current_group = group 39 | end 40 | 41 | -- record the results of the test 42 | -- test is the name 43 | -- result is the boolean pass/fail 44 | -- msg,... are optional and will be string.format()ed and displayed to the user as 45 | -- "note" or "fail" depending on the value of result 46 | function test.record(name, result, data, msg, ...) 47 | table.insert(test.current_group, { 48 | name=name, 49 | result=result, 50 | data=data, 51 | message=msg and tostring(msg):format(...) or nil, 52 | }) 53 | end 54 | 55 | -- automatically test writing and reading. obuf and oval are optional, 56 | -- and default to the same values as ibuf and ival. 57 | -- This tests the following operations: 58 | -- read(ibuf) == oval 59 | -- write(read(ibuf)) == obuf 60 | -- write(ival) == obuf 61 | -- read(write(ival)) == oval 62 | function test.autotest(name, format, ibuf, ival, obuf, oval) 63 | local eq = test.eq 64 | local record = test.record 65 | 66 | obuf = obuf or ibuf 67 | oval = oval or ival 68 | 69 | assert(type(obuf) == type(ibuf)) 70 | assert(type(obuf) == "string") 71 | 72 | if type(ival) ~= "table" then ival = {ival} end 73 | if type(oval) ~= "table" then oval = {oval} end 74 | 75 | local function tester() 76 | local f = vstruct.compile(format) 77 | 78 | local U = f:read(ibuf) 79 | record(name.." (U )", eq(U, oval), unpack(U)) 80 | 81 | local UP = f:write(U) 82 | record(name.." (UP)", eq(UP, obuf), test.od(UP)) 83 | 84 | local P = f:write(ival) 85 | record(name.." (P )", eq(P, obuf), test.od(P)) 86 | 87 | local PU = f:read(P) 88 | record(name.." (PU)", eq(PU, oval), unpack(PU)) 89 | end 90 | 91 | xpcall(tester, function(err) 92 | record(name.." !ERR", false, debug.traceback("Error executing test: "..err)) 93 | end) 94 | end 95 | 96 | -- test whether an error is properly reported 97 | -- this will call fn(...), and verify that it raises an error that matches 98 | -- the pattern 99 | function test.errortest(name, pattern, fn, ...) 100 | local res,err = pcall(fn, ...) 101 | if res then 102 | test.record(name, false, "Expected error("..pattern.."), got success") 103 | else 104 | if err:match(pattern) then 105 | test.record(name, true, err) 106 | else 107 | test.record(name, false, "Expected error("..pattern.."), got "..err) 108 | end 109 | end 110 | end 111 | 112 | function test.report() 113 | local allfailed = 0 114 | for _,group in ipairs(test.results) do 115 | local failed = 0 116 | print("\t=== "..group.name.." ===") 117 | 118 | for _,test in ipairs(group) do 119 | if not test.result then 120 | failed = failed + 1 121 | if type(test.data) == 'string' and test.data:match("%z") then 122 | print("FAIL", test.name, (test.data:gsub("%z", "."))) 123 | else 124 | print("", test.name, test.data) 125 | end 126 | if test.message then 127 | print("", test.message or "") 128 | end 129 | end 130 | end 131 | 132 | print("\tTotal: ", #group) 133 | print("\tFailed:", failed) 134 | print() 135 | allfailed = allfailed + failed 136 | end 137 | 138 | return allfailed 139 | end 140 | 141 | -- determine host endianness 142 | function test.bigendian() 143 | return require "vstruct.io" ("endianness", "host") == "big" 144 | end 145 | 146 | return test 147 | -------------------------------------------------------------------------------- /test/compat1x.lua: -------------------------------------------------------------------------------- 1 | -- Tests for format compability with vstruct 1.x 2 | 3 | local vstruct = require "vstruct" 4 | local test = require "vstruct.test.common" 5 | 6 | local x = test.x 7 | 8 | test.group "compat1x" 9 | 10 | require "vstruct.compat1x" 11 | vstruct.WARN = false 12 | 13 | local fmt = "< i3" 14 | local data = x"FE FF FF" 15 | 16 | test.record(vstruct.unpack(fmt, data)[1] == vstruct.read(fmt, data)[1], "unpack/read") 17 | test.record(vstruct.unpackvals(fmt, data) == vstruct.readvals(fmt, data), "unpack/read") 18 | test.record(vstruct.pack(fmt, data, {-2}) == vstruct.write(fmt, data, {-2}), "pack/write") 19 | 20 | local t = vstruct.compile(fmt) 21 | test.record(t.unpack(data)[1] == t:read(data)[1], ":unpack/read") 22 | test.record(t.pack(data, {-2}) == t:write(data, {-2}), ":pack/write") 23 | 24 | vstruct.WARN = true 25 | -- test that the top-level API works as well 26 | -------------------------------------------------------------------------------- /test/error.lua: -------------------------------------------------------------------------------- 1 | -- tests for error conditions 2 | -- checks that vstruct properly raises an error (and raises the *correct* error) 3 | -- when things go wrong 4 | 5 | local test = require "vstruct.test.common" 6 | local vstruct = require "vstruct" 7 | local E = test.errortest 8 | 9 | test.group "error conditions" 10 | 11 | 12 | -- utility functions gone horribly wrong 13 | E("missing-explode-1", "bad argument #1 to 'vstruct.explode' %(number expected, got nil%)", vstruct.explode) 14 | E("invalid-explode-1", "bad argument #1 to 'vstruct.explode' %(number expected, got boolean%)", vstruct.explode, true, nil) 15 | E("invalid-explode-2", "bad argument #2 to 'vstruct.explode' %(number expected, got boolean%)", vstruct.explode, 0, true) 16 | 17 | E("missing-implode-1", "bad argument #1 to 'vstruct.implode' %(table expected, got nil%)", vstruct.implode) 18 | E("invalid-implode-1", "bad argument #1 to 'vstruct.implode' %(table expected, got boolean%)", vstruct.implode, true, nil) 19 | E("invalid-implode-2", "bad argument #2 to 'vstruct.implode' %(number expected, got boolean%)", vstruct.implode, { 0 }, true) 20 | 21 | -- attempt to read/seek past bounds of file 22 | -- seeking past the end is totally allowed when writing 23 | -- when reading, you will get a different error when you try to do IO 24 | E("invalid-seek-uf", "attempt to read past end of buffer", vstruct.read, "@8 u4", "1234") 25 | E("invalid-seek-ub", "attempt to seek prior to start of file", vstruct.read, "@0 -4", "1234") 26 | E("invalid-seek-pb", "attempt to seek prior to start of file", vstruct.write, "@0 -4", "1234", {}) 27 | 28 | -- invalid argument type 29 | E("invalid-arg-u1", "bad argument #1 to 'vstruct.read' %(string expected, got nil%)", vstruct.read) 30 | E("invalid-arg-p1", "bad argument #1 to 'vstruct.write' %(string expected, got nil%)", vstruct.write) 31 | E("invalid-arg-c1", "bad argument #1 to 'vstruct.compile' %(string expected, got nil%)", vstruct.compile) 32 | E("invalid-arg-u1", "bad argument #1 to 'vstruct.read' %(string expected, got number%)", vstruct.read, 0, "1234") 33 | E("invalid-arg-p1", "bad argument #1 to 'vstruct.write' %(string expected, got number%)", vstruct.write, 0, {}) 34 | E("invalid-arg-c1", "bad argument #1 to 'vstruct.compile' %(string expected, got number%)", vstruct.compile, 0) 35 | 36 | E("invalid-arg-u2", "bad argument #2 to 'vstruct.read' %(file or string expected, got number%)", vstruct.read, "@0", 0) 37 | E("invalid-arg-p2", "bad argument #2 to 'vstruct.write' %(file or string expected, got number%)", vstruct.write, "@0", 0, {}) 38 | 39 | E("invalid-arg-u3", "bad argument #3 to 'vstruct.read' %(table expected, got string%)", vstruct.read, "@0", "", "1234") 40 | E("invalid-arg-p3", "bad argument #3 to 'vstruct.write' %(table expected, got string%)", vstruct.write, "@0", nil, "1234") 41 | E("invalid-arg-p3", "bad argument #3 to 'vstruct.write' %(table expected, got string%)", vstruct.write, "@0", "1234") 42 | 43 | -- format string is ill-formed 44 | -- note that the empty format string is well-formed, does nothing, and returns/accepts the empty table 45 | E("invalid-format-number", "expected.*, got EOF", vstruct.compile, "4") 46 | E("invalid-format-}", "expected.* or io specifier, got }", vstruct.compile, "}") 47 | E("invalid-format-)", "expected.* or io specifier, got %)", vstruct.compile, ")") 48 | E("invalid-format-]", "expected.* or io specifier, got %]", vstruct.compile, "]") 49 | E("invalid-format-{", "expected.*, got EOF", vstruct.compile, "{") 50 | E("invalid-format-(", "expected.*, got EOF", vstruct.compile, "(") 51 | E("invalid-format-[", "expected.*, got EOF", vstruct.compile, "[") 52 | E("invalid-format-*", "expected.*or io specifier, got %*", vstruct.compile, "*4") 53 | E("invalid-format-no-size", "format requires a size", vstruct.compile, "u u4") 54 | 55 | -- format string is well-formed but nonsensical 56 | -- note that empty groups and tables and zero-length repeats make it easier to dynamically construct format strings, and are thus allowed 57 | E("bad-format-no-support", "no support for format 'q'", vstruct.compile, "q1") 58 | E("bad-format-small-bitpack", "bitpack contents do not match bitpack size", vstruct.compile, "[1|u4]") 59 | E("bad-format-large-bitpack", "bitpack contents do not match bitpack size", vstruct.compile, "[1|u16]") 60 | 61 | -- io format size checking occurs on a format-by-format basis 62 | E("bad-format-size-missing-f", "only supports sizes 4", vstruct.compile, 'f') 63 | E("bad-format-size-wrong-f", "only supports sizes 4", vstruct.compile, 'f1') 64 | E("bad-format-fraction-p", "format requires a fractional%-part size", vstruct.compile, 'p4') 65 | E("bad-format-x-bit", "invalid value to `x` format in bitpack: 0 or 1 required, got 2", vstruct.write, "[1|x8,2]", {}) 66 | E("bad-format-x-byte", "bad argument #1 to 'char'", vstruct.write, "x1,300", {}) 67 | -- note that s and z can be used either with or without a size specifier 68 | local sized_formats = "abcimpux@+-" 69 | local plain_formats = "<>=" 70 | for format in sized_formats:gmatch(".") do 71 | E("bad-format-size-missing-"..format, "format requires a size", vstruct.compile, format) 72 | end 73 | for format in plain_formats:gmatch(".") do 74 | E("bad-format-size-present-"..format, "is an endianness control, and does not have size", vstruct.compile, format.."1") 75 | end 76 | 77 | -- format string splicing 78 | vstruct.compile("coord", "x:u1 y:u1 z:u1") 79 | E("splice-wrong-syntax", "parsing format string at character 10.*expected value.*got splice", vstruct.read, "position:&coord", "000") 80 | E("splice-wrong-name", "attempt to splice in format 'coords', which is not registered", vstruct.read, "&coords", "000") 81 | 82 | -- input table doesn't match format string 83 | E("bad-data-missing", "bad input while writing: no value for key 1", vstruct.write, "u4", {}) 84 | E("bad-data-missing-name", "bad input while writing: no value for key t", vstruct.write, "t:{ x:u4 }", {}) 85 | E("bad-data-missing-nested", "bad input while writing: no value for key t.x", vstruct.write, "t.x:u4", {}) 86 | 87 | -- backreference to nonexistent or non-numeric data 88 | E("backreference-missing", "backreferenced field.*not been read yet", vstruct.read, "#count*u4", "000") 89 | E("backreference-wrong-type", "backreferenced field.*not a numeric type", vstruct.read, "count:s4 #count*u4", "abcd000") 90 | 91 | -- these require a bunch of type-specific checks. I don't have a good way to do this yet and it's an open question whether I want to do it at all. 92 | --E("bad-data-u-string", "placeholder", vstruct.write, "u4", { "string" }) 93 | --E("bad-data-u-numeric-string", "placeholder", vstruct.write, "u4", { "0" }) 94 | --E("bad-data-s-number", "placeholder", vstruct.write, "s4", { 0 }) 95 | --E("bad-data-z-number", "placeholder", vstruct.write, "z4", { 0 }) 96 | -------------------------------------------------------------------------------- /test/fp-bigendian.lua: -------------------------------------------------------------------------------- 1 | local vstruct = require "vstruct" 2 | local test = require "vstruct.test.common" 3 | local inf = math.huge 4 | 5 | -- work around a bug in the Lua constant table optimizer 6 | local z = 0.0 7 | local nz = -z 8 | 9 | local function T(n, buf) 10 | return test.autotest(tostring(n), "> f8", buf, n) 11 | end 12 | 13 | test.group("big-endian ieee doubles") 14 | 15 | T(0.0, "\000\000\000\000\000\000\000\000") 16 | T(4.9406564584124654418e-324, "\000\000\000\000\000\000\000\001") 17 | T(7.4169128616906696301e-309, "\000\005\085\085\085\085\085\085") 18 | T(1.483382572338133926e-308, "\000\010\170\170\170\170\170\170") 19 | T(2.225073858507200889e-308, "\000\015\255\255\255\255\255\255") 20 | T(2.2250738585072013831e-308, "\000\016\000\000\000\000\000\000") 21 | T(2.2250738585072018772e-308, "\000\016\000\000\000\000\000\001") 22 | T(2.9667651446762683461e-308, "\000\021\085\085\085\085\085\085") 23 | T(3.7084564308453353091e-308, "\000\026\170\170\170\170\170\170") 24 | T(4.4501477170144022721e-308, "\000\031\255\255\255\255\255\255") 25 | T(8.9884656743115795386e+307, "\127\224\000\000\000\000\000\000") 26 | T(8.9884656743115815345e+307, "\127\224\000\000\000\000\000\001") 27 | T(1.1984620899082105386e+308, "\127\229\085\085\085\085\085\085") 28 | T(1.4980776123852631234e+308, "\127\234\170\170\170\170\170\170") 29 | T(1.7976931348623157081e+308, "\127\239\255\255\255\255\255\255") 30 | T(4.9406564584124654418e-324, "\000\000\000\000\000\000\000\001") 31 | T(2.225073858507200889e-308, "\000\015\255\255\255\255\255\255") 32 | T(2.2250738585072013831e-308, "\000\016\000\000\000\000\000\000") 33 | T(1.7976931348623157081e+308, "\127\239\255\255\255\255\255\255") 34 | T(inf, "\127\240\000\000\000\000\000\000") 35 | T(0.0, "\000\000\000\000\000\000\000\000") 36 | T(nz, "\128\000\000\000\000\000\000\000") 37 | T(-4.9406564584124654418e-324, "\128\000\000\000\000\000\000\001") 38 | T(-7.4169128616906696301e-309, "\128\005\085\085\085\085\085\085") 39 | T(-1.483382572338133926e-308, "\128\010\170\170\170\170\170\170") 40 | T(-2.225073858507200889e-308, "\128\015\255\255\255\255\255\255") 41 | T(-2.2250738585072013831e-308, "\128\016\000\000\000\000\000\000") 42 | T(-2.2250738585072018772e-308, "\128\016\000\000\000\000\000\001") 43 | T(-2.9667651446762683461e-308, "\128\021\085\085\085\085\085\085") 44 | T(-3.7084564308453353091e-308, "\128\026\170\170\170\170\170\170") 45 | T(-4.4501477170144022721e-308, "\128\031\255\255\255\255\255\255") 46 | T(-8.9884656743115795386e+307, "\255\224\000\000\000\000\000\000") 47 | T(-8.9884656743115815345e+307, "\255\224\000\000\000\000\000\001") 48 | T(-1.1984620899082105386e+308, "\255\229\085\085\085\085\085\085") 49 | T(-1.4980776123852631234e+308, "\255\234\170\170\170\170\170\170") 50 | T(-1.7976931348623157081e+308, "\255\239\255\255\255\255\255\255") 51 | T(-4.9406564584124654418e-324, "\128\000\000\000\000\000\000\001") 52 | T(-2.225073858507200889e-308, "\128\015\255\255\255\255\255\255") 53 | T(-2.2250738585072013831e-308, "\128\016\000\000\000\000\000\000") 54 | T(-1.7976931348623157081e+308, "\255\239\255\255\255\255\255\255") 55 | T(-inf, "\255\240\000\000\000\000\000\000") 56 | T(nz, "\128\000\000\000\000\000\000\000") 57 | -------------------------------------------------------------------------------- /test/fp-littleendian.lua: -------------------------------------------------------------------------------- 1 | local vstruct = require "vstruct" 2 | local test = require "vstruct.test.common" 3 | local inf = math.huge 4 | 5 | -- work around a bug in the Lua constant table optimizer 6 | local z = 0.0 7 | local nz = -z 8 | 9 | local function T(n, buf) 10 | return test.autotest(tostring(n), "< f8", buf, n) 11 | end 12 | 13 | test.group("little-endian ieee doubles") 14 | 15 | T(0.0, "\000\000\000\000\000\000\000\000") 16 | T(4.9406564584124654418e-324, "\001\000\000\000\000\000\000\000") 17 | T(7.4169128616906696301e-309, "\085\085\085\085\085\085\005\000") 18 | T(1.483382572338133926e-308, "\170\170\170\170\170\170\010\000") 19 | T(2.225073858507200889e-308, "\255\255\255\255\255\255\015\000") 20 | T(2.2250738585072013831e-308, "\000\000\000\000\000\000\016\000") 21 | T(2.2250738585072018772e-308, "\001\000\000\000\000\000\016\000") 22 | T(2.9667651446762683461e-308, "\085\085\085\085\085\085\021\000") 23 | T(3.7084564308453353091e-308, "\170\170\170\170\170\170\026\000") 24 | T(4.4501477170144022721e-308, "\255\255\255\255\255\255\031\000") 25 | T(8.9884656743115795386e+307, "\000\000\000\000\000\000\224\127") 26 | T(8.9884656743115815345e+307, "\001\000\000\000\000\000\224\127") 27 | T(1.1984620899082105386e+308, "\085\085\085\085\085\085\229\127") 28 | T(1.4980776123852631234e+308, "\170\170\170\170\170\170\234\127") 29 | T(1.7976931348623157081e+308, "\255\255\255\255\255\255\239\127") 30 | T(4.9406564584124654418e-324, "\001\000\000\000\000\000\000\000") 31 | T(2.225073858507200889e-308, "\255\255\255\255\255\255\015\000") 32 | T(2.2250738585072013831e-308, "\000\000\000\000\000\000\016\000") 33 | T(1.7976931348623157081e+308, "\255\255\255\255\255\255\239\127") 34 | T(inf, "\000\000\000\000\000\000\240\127") 35 | T(0.0, "\000\000\000\000\000\000\000\000") 36 | T(nz, "\000\000\000\000\000\000\000\128") 37 | T(-4.9406564584124654418e-324, "\001\000\000\000\000\000\000\128") 38 | T(-7.4169128616906696301e-309, "\085\085\085\085\085\085\005\128") 39 | T(-1.483382572338133926e-308, "\170\170\170\170\170\170\010\128") 40 | T(-2.225073858507200889e-308, "\255\255\255\255\255\255\015\128") 41 | T(-2.2250738585072013831e-308, "\000\000\000\000\000\000\016\128") 42 | T(-2.2250738585072018772e-308, "\001\000\000\000\000\000\016\128") 43 | T(-2.9667651446762683461e-308, "\085\085\085\085\085\085\021\128") 44 | T(-3.7084564308453353091e-308, "\170\170\170\170\170\170\026\128") 45 | T(-4.4501477170144022721e-308, "\255\255\255\255\255\255\031\128") 46 | T(-8.9884656743115795386e+307, "\000\000\000\000\000\000\224\255") 47 | T(-8.9884656743115815345e+307, "\001\000\000\000\000\000\224\255") 48 | T(-1.1984620899082105386e+308, "\085\085\085\085\085\085\229\255") 49 | T(-1.4980776123852631234e+308, "\170\170\170\170\170\170\234\255") 50 | T(-1.7976931348623157081e+308, "\255\255\255\255\255\255\239\255") 51 | T(-4.9406564584124654418e-324, "\001\000\000\000\000\000\000\128") 52 | T(-2.225073858507200889e-308, "\255\255\255\255\255\255\015\128") 53 | T(-2.2250738585072013831e-308, "\000\000\000\000\000\000\016\128") 54 | T(-1.7976931348623157081e+308, "\255\255\255\255\255\255\239\255") 55 | T(-inf, "\000\000\000\000\000\000\240\255") 56 | T(nz, "\000\000\000\000\000\000\000\128") 57 | -------------------------------------------------------------------------------- /test/frexp.lua: -------------------------------------------------------------------------------- 1 | -- regression tests for vstruct 2 | -- tests for fixed bugs go here to make sure they don't recur 3 | -- tests are named after the bug they trigger, not the intended behaviour 4 | 5 | local test = require "vstruct.test.common" 6 | local frexp = require "vstruct.frexp" 7 | 8 | -- No point in running these tests if there isn't a builtin frexp to compare to. 9 | if not math.frexp then return end 10 | 11 | test.group "frexp()" 12 | 13 | local function test_frexp(x) 14 | local m,e = math.frexp(x) 15 | if m >= 1 or m <= -1 then 16 | -- At least on my machine, frexp() sometimes returns 1 as the mantissa, in 17 | -- violation as the spec. Correct this so that we can more properly compare 18 | -- things. 19 | m,e = m/2, e+1 20 | end 21 | local m2,e2 = frexp(x) 22 | local ok = m == m2 and e == e2 23 | test.record("frexp equivalence", ok, x, 24 | "(builtin) %f,%f != %f,%f (lua)", m, e, m2, e2) 25 | end 26 | 27 | local inf = math.huge 28 | local z = 0.0 29 | local nz = -z 30 | local doubles = { 31 | 0.0, 1.0, 2.0, 32 | 1/2, 1/4, 1/8, 33 | 4.9406564584124654418e-324, 34 | 7.4169128616906696301e-309, 35 | 1.483382572338133926e-308, 36 | 2.225073858507200889e-308, 37 | 2.2250738585072013831e-308, 38 | 2.2250738585072018772e-308, 39 | 2.9667651446762683461e-308, 40 | 3.7084564308453353091e-308, 41 | 4.4501477170144022721e-308, 42 | 8.9884656743115795386e+307, 43 | 8.9884656743115815345e+307, 44 | 1.1984620899082105386e+308, 45 | 1.4980776123852631234e+308, 46 | 1.7976931348623157081e+308, 47 | 4.9406564584124654418e-324, 48 | 2.225073858507200889e-308, 49 | 2.2250738585072013831e-308, 50 | 1.7976931348623157081e+308, 51 | 0.0, 52 | nz, 53 | -4.9406564584124654418e-324, 54 | -7.4169128616906696301e-309, 55 | -1.483382572338133926e-308, 56 | -2.225073858507200889e-308, 57 | -2.2250738585072013831e-308, 58 | -2.2250738585072018772e-308, 59 | -2.9667651446762683461e-308, 60 | -3.7084564308453353091e-308, 61 | -4.4501477170144022721e-308, 62 | -8.9884656743115795386e+307, 63 | -8.9884656743115815345e+307, 64 | -1.1984620899082105386e+308, 65 | -1.4980776123852631234e+308, 66 | -1.7976931348623157081e+308, 67 | -4.9406564584124654418e-324, 68 | -2.225073858507200889e-308, 69 | -2.2250738585072013831e-308, 70 | -1.7976931348623157081e+308, 71 | nz, 72 | } 73 | 74 | for _,double in ipairs(doubles) do 75 | test_frexp(double) 76 | end 77 | -------------------------------------------------------------------------------- /test/perf.lua: -------------------------------------------------------------------------------- 1 | -- Test vstruct performance. 2 | 3 | local vstruct = require "vstruct" 4 | local read,write = vstruct.read,vstruct.write 5 | 6 | local SIZE = 2^14 7 | 8 | local data = { 9 | strings = {}; 10 | numbers = {}; 11 | booleans = {}; 12 | masks = {}; 13 | } 14 | 15 | for i=1,SIZE do 16 | data.strings[i] = "xxxx" 17 | data.numbers[i] = i 18 | data.booleans[i] = i % 2 == 0 19 | data.masks[i] = { true, true, false, true, false, false, true, true } 20 | end 21 | 22 | local function perf(format, data, file) 23 | format = '@0 ' .. SIZE .. ' * ' .. format 24 | local wstart,rstart,stop 25 | if not file then 26 | wstart = os.clock() 27 | local buf = write(format, data) 28 | rstart = os.clock() 29 | read(format, buf) 30 | stop = os.clock() 31 | else 32 | local fd = io.open(file, 'w+') 33 | wstart = os.clock() 34 | write(format, fd, data) 35 | rstart = os.clock() 36 | read(format, fd) 37 | stop = os.clock() 38 | fd:close() 39 | end 40 | return stop - rstart,rstart - wstart 41 | end 42 | 43 | local wtotal,rtotal = 0,0 44 | 45 | local function report(format, read, write) 46 | print(("%4s%8.4f%8.4f"):format(format, read, write)) 47 | rtotal = rtotal + read 48 | wtotal = wtotal + write 49 | end 50 | 51 | local formats = { 52 | strings = {'c', 's', 'z'}; 53 | numbers = {'f', 'i', 'p8,', 'u'}; 54 | booleans = {'b'}; 55 | masks = {'m'}; 56 | } 57 | 58 | for _,file in ipairs { false, "/tmp/vstruct_perf" } do 59 | if file then print('---- using file ----') 60 | else print('---- using string ----') end 61 | for type,formats in pairs(formats) do 62 | for _,format in ipairs(formats) do 63 | for _,size in ipairs { 4, 8 } do 64 | report(format..size, perf(format..size, data[type], file)) 65 | end 66 | end 67 | end 68 | end 69 | print('---- -----') 70 | print("total", rtotal, wtotal) 71 | -------------------------------------------------------------------------------- /test/regression.lua: -------------------------------------------------------------------------------- 1 | -- regression tests for vstruct 2 | -- tests for fixed bugs go here to make sure they don't recur 3 | -- tests are named after the bug they trigger, not the intended behaviour 4 | 5 | local test = require "vstruct.test.common" 6 | 7 | local x = test.x 8 | local T = test.autotest 9 | 10 | test.group "regression tests" 11 | 12 | -- B#21 13 | -- this was found as an error in writing fixed point values, but is actually an underlying issue 14 | -- with how u and i handle non-integer inputs 15 | -- in short: don't assume string.char() truncates, because it doesn't. 16 | T("i rounds instead of truncating", 17 | "> i2", 18 | x"FF FF", -1.5, 19 | x"FF FF", -1) 20 | T("u rounds instead of truncating", 21 | "> u2", 22 | x"00 01", 1.5, 23 | x"00 01", 1) 24 | T("p passes invalid value to string.char and crashes", 25 | "> p2,2", 26 | x"FD 00", -192.098910, 27 | x"FD 00", -192.00) 28 | -------------------------------------------------------------------------------- /test/struct-test-gen.lua: -------------------------------------------------------------------------------- 1 | -- Config 2 | local CONF = { 3 | VERBOSE = false, -- Spew verbosely while testing? 4 | 5 | WIDTH = 32, -- Total size of numbers in bits. 6 | PRECISION = 17, -- Fractional precision in bits. 7 | 8 | MAX_DEPTH = 5, -- Limit on nesting depth of test formats. 9 | COMPOUND_LENGTH = 2, -- # formats in a sequence. 10 | SEED = false, -- Seed for generating tests. 11 | 12 | NROF_TESTS = 2^10, -- number of random test cases to run 13 | } 14 | 15 | -- Features to use when making format strings to test. 16 | local USE = { 17 | LEFT_REPETITION = true, --n*fmt 18 | GROUPING = true, --{ fmt } 19 | NESTING = true, --( fmt ) 20 | NAMING = true, --id:fmt 21 | ENDIAN = true, -->< 22 | EMPTY = true, -- 23 | COMPOUND = true --fmt fmt ... fmt 24 | } 25 | 26 | 27 | -- Non-config 28 | local PROTECT_STRUCT = true -- If set, wrap calls to struct with pcall. 29 | local TEST_KINDS = {} --"read"|"write" 30 | 31 | -- Should every format string explicitly set the endianness at the start? 32 | -- Setting this to true mask(s|ed?) a bug where the endianness controls leak 33 | -- between calls to struct.write(). 34 | local PREFIX_ENDIAN = false 35 | local PREFIX_ENDIAN_SEP = " " --"" gives format string parse errors 36 | 37 | local TOKEN_SEP = "" -- Separator between tokens. 38 | local MAX_BITS = 52 -- Maximum bit size we care about. 39 | local MACHINE_ENDIAN = "<" -- Native endianness. 40 | 41 | local COMPOUND_SEP = " " 42 | 43 | local loadstring = loadstring or load 44 | 45 | -- Parse arguments and sort then out into USE vs. CONF options. 46 | for _,given_arg in ipairs(arg) do 47 | local k,v = given_arg:match("^([^=]+)=(.+)") 48 | k = k or given_arg 49 | k = k:upper():gsub("-", "_") 50 | v = v or true 51 | 52 | if nil ~= CONF[k] then 53 | CONF[k] = assert(loadstring("return "..tostring(v)))() 54 | elseif nil ~= USE[k] then 55 | USE[k] = v ~= "FALSE" and true or false 56 | elseif k == "NONE" then 57 | for feature,_ in pairs(USE) do USE[feature] = false end 58 | elseif given_arg == "read" or given_arg == "write" then 59 | table.insert(TEST_KINDS, given_arg) 60 | else 61 | io.stderr:write(("unknown argument %q\n"):format(given_arg)) 62 | os.exit(1) 63 | end 64 | end 65 | 66 | -- Imports 67 | local io, math, os, string, table = 68 | io, math, os, string, table 69 | 70 | local assert, ipairs, pairs, pcall = 71 | assert, ipairs, pairs, pcall 72 | local require, setmetatable, tonumber, tostring, type = 73 | require, setmetatable, tonumber, tostring, type 74 | 75 | local struct = require "vstruct"; struct.cache = nil 76 | local test = require "vstruct.test.common" 77 | local record = test.record 78 | local __EOG__ = __EOG__ 79 | 80 | local abs,ceil,floor,fmod = math.abs, math.ceil, math.floor, math.fmod 81 | local frexp = math.frexp or require "vstruct.frexp" 82 | local char_ = string.char 83 | 84 | local function char(x) return char_(x % 256) end 85 | 86 | -- Structural equality (no cycles please) 87 | local function eq(a,b) 88 | if a == b then return true end 89 | local t = type(a) 90 | if t ~= type(b) then return false end 91 | if t ~= "table" then return false end 92 | 93 | for k,av in pairs(a) do 94 | local bv = b[k] 95 | if not eq(av,bv) then 96 | return false 97 | end 98 | end 99 | for k,bv in pairs(b) do 100 | local av = a[k] 101 | if not eq(av,bv) then 102 | return false 103 | end 104 | end 105 | return true 106 | end 107 | 108 | -- str = dump(value) 109 | local dump = {} 110 | setmetatable(dump, 111 | { 112 | __call = 113 | function (self, value) 114 | local t = type(value) 115 | local f = dump[t] 116 | assert(f, "don't know how to dump ".. 117 | "values of type "..t) 118 | return f(value) 119 | end 120 | }) 121 | 122 | dump["nil"] = function() return "nil" end 123 | 124 | function dump.string(s) return ("%q"):format(s) end 125 | 126 | function dump.boolean(b) return tostring(b) end 127 | 128 | function dump.number(x) 129 | local m, e = frexp(x) --0.5 <= m <= 1, e an int 130 | for i=1,53 do --scale m to an integer 131 | if floor(m) == m then break end 132 | m = m*2 133 | e = e - 1 134 | end 135 | 136 | if e == 0 then return ("%.0f"):format(m) end 137 | 138 | local op = e < 0 and "/" or "*" 139 | e = abs(e) 140 | local exp = e > 1 and "2^"..e or "2" 141 | return ("%.0f%s%s"):format(m, op, exp) 142 | end 143 | 144 | function dump.table(t) 145 | local dumped = {} 146 | 147 | local pieces = {} 148 | local function push(x) table.insert(pieces, x) end 149 | 150 | push("{") 151 | 152 | for i, v in ipairs(t) do 153 | if #pieces > 1 then push(", ") end 154 | push(dump(v)) 155 | dumped[i] = true 156 | end 157 | 158 | for k,v in pairs(t) do 159 | if not dumped[k] then 160 | if #pieces > 1 then push(", ") end 161 | if type(k) == "string" and k:match("^%a[a-zA-Z0-9_]*$") then 162 | push(k) 163 | else 164 | push("[") 165 | push(dump(k)) 166 | push("]") 167 | end 168 | push("=") 169 | push(dump(v)) 170 | end 171 | end 172 | 173 | push("}") 174 | return table.concat(pieces) 175 | end 176 | 177 | -- List module 178 | local list 179 | list = { 180 | reverse = 181 | function (t) 182 | local out = {} 183 | local n = #t 184 | for i=1,n do 185 | local j = n-i+1 186 | out[j] = t[i] 187 | end 188 | return out 189 | end, 190 | 191 | iter = 192 | function (t, f) 193 | for i,v in ipairs(t) do f(v) end 194 | return t 195 | end, 196 | 197 | map = 198 | function (t, f) 199 | local out = {} 200 | for i,v in ipairs(t) do out[i] = f(v) end 201 | return out 202 | end, 203 | 204 | foldl = 205 | function (t, f, accu) 206 | for i,v in ipairs(t) do accu = f(v, accu) end 207 | return accu 208 | end, 209 | 210 | merge = 211 | function (dst, src) 212 | local merged = {} 213 | for k,v in ipairs(src) do --append the array part 214 | table.insert(dst, v) 215 | merged[k] = true 216 | end 217 | for k,v in pairs(src) do --overwrite the hash part 218 | if not merged[k] then 219 | dst[k] = v 220 | end 221 | end 222 | return dst 223 | end, 224 | 225 | zeros = 226 | function (n) 227 | local t={} 228 | for i=1,n do t[i] = 0 end 229 | return t 230 | end, 231 | 232 | range = 233 | function (first, last, step) 234 | local t = {} 235 | if nil == last then 236 | first,last,step = 1,first,step 237 | end 238 | step = step or 1 239 | for i=first,last,step do t[#t+1] = i end 240 | return t 241 | end, 242 | 243 | cat = 244 | function (...) 245 | local accu = {} 246 | local function loop(t,...) 247 | if nil == t then return accu end 248 | list.merge(accu, t) 249 | return loop(...) 250 | end 251 | return loop(...) 252 | end, 253 | 254 | rep = 255 | function (t, count) 256 | count = count or 0 257 | local accu = {} 258 | for i=1,count do list.merge(accu, t) end 259 | return accu 260 | end 261 | } 262 | 263 | -- MRG32k3a random number generator. 264 | -- Returns numbers which are random for the full range of the IEEE 265 | -- double mantissa. 266 | local function make_rand(seed) 267 | 268 | local s10, s11, s12 = 12345,12345,12345 269 | local s20, s21, s22 = 12345,12345,12345 270 | 271 | local function rand() 272 | -- Return the next random number in the range [0,4294967087). 273 | local p1 = (1403580*s11 - 810728*s10) % 4294967087 274 | local p2 = (527612*s22 - 1370589*s20) % 4294944443 275 | s10,s11,s12 = s11,s12,p1 276 | s20,s21,s22 = s21,s22,p2 277 | return (p1 - p2) % 4294967087 278 | end 279 | 280 | if seed then 281 | seed = tonumber(seed) or 12345 282 | local m = 4294967087 283 | local a = seed % 4294967087 284 | a = a==a and a or 12345 285 | s10,s11,s12,s20,s21,s22 = a,a+1,a+2,a+3,a+4,a+5 -- not all zero 286 | rand(); rand(); rand(); -- pump out the seed. 287 | end 288 | 289 | local function rand52() 290 | local hi,lo = rand(), rand() 291 | local m,e = frexp(hi * 4294967087.0 + lo) -- 0.5 <= m < 1 292 | local t = m*2^53 - 2^52 -- map mantissa to 0 <= t < 2^52 range 293 | return t 294 | end 295 | 296 | return rand52 297 | end 298 | 299 | local function byte_rand(rand) 300 | return function () return rand() % 256 end 301 | end 302 | 303 | local function bool_rand(rand) 304 | return function () return rand() % 2 == 0 end 305 | end 306 | 307 | local function char_rand(rand) 308 | return function () return char(rand()) end 309 | end 310 | 311 | local function clamped_rand(rand, lo, hi) 312 | local wid = hi - lo + 1 313 | return function () return lo + (rand() % wid) end 314 | end 315 | 316 | local function string_rand(rand, minlen, maxlen) 317 | minlen = minlen or 0 318 | maxlen = maxlen or 512 319 | local gen_len = clamped_rand(rand, minlen, maxlen) 320 | local gen_char = char_rand(clamped_rand(rand, 66, 90)) 321 | return function (len) 322 | len = len or gen_len() 323 | return table.concat(list.map(list.range(len), gen_char)) 324 | end 325 | end 326 | 327 | -- Return an array of size representing value in two's 328 | -- complement in little-endian format. 329 | local function int_to_bytes_le(value, nbytes) 330 | local t = {} 331 | for i=1,nbytes do 332 | local r = value % 256 333 | t[i] = r 334 | value = (value - r) / 256 335 | end 336 | return t 337 | end 338 | 339 | local function keys(tab) 340 | local ks = {} 341 | for k,_ in pairs(tab) do table.insert(ks, k) end 342 | return ks 343 | end 344 | 345 | local function bless(obj, class) --make obj an instance of class 346 | return setmetatable(obj, {__index = class}) 347 | end 348 | 349 | -- Test case interface. 350 | local Test = { 351 | new = 352 | function (class, self) 353 | return bless(self or {}, class) 354 | end, 355 | 356 | get_naked = nil, --write/read format string w/o endian control 357 | get_data = nil, --data byte string 358 | get_values = nil, --value list 359 | get_length = nil, --# top level entities in the format 360 | get_endian = nil, --before/after endian; nil=don't care 361 | 362 | get_format = 363 | function (self, endian, fmt) 364 | local fmt = fmt or self:get_naked() 365 | local before, after = self:get_endian() 366 | endian = endian or before 367 | before = before or endian 368 | after = after or before 369 | if before ~= endian then 370 | fmt = before..fmt 371 | end 372 | return fmt, after 373 | end 374 | } 375 | 376 | local function as_hex(str) 377 | if not str then return nil end 378 | return str:gsub('.', function(c) 379 | return string.format("%02X ", c:byte()) 380 | end) 381 | end 382 | 383 | local function perform_write_test(test, endian) 384 | endian = endian or MACHINE_ENDIAN 385 | 386 | local format = test:get_format(endian) 387 | local want_data = test:get_data() 388 | local values = test:get_values() 389 | 390 | if PREFIX_ENDIAN then 391 | format = "=" .. PREFIX_ENDIAN_SEP .. format 392 | end 393 | 394 | local pcall = pcall 395 | if not PROTECT_STRUCT then 396 | pcall = function (f,...) return true, f(...) end 397 | end 398 | 399 | local errmsg = nil 400 | 401 | local success, got_data = pcall(struct.write, format, values) 402 | if not success then 403 | errmsg = got_data 404 | got_data = nil 405 | end 406 | 407 | success = success and eq(got_data, want_data) 408 | 409 | record("random write test", success, dump { 410 | got_data = got_data and as_hex(got_data) or nil, 411 | want_data = as_hex(want_data), 412 | format = format, 413 | values = values, 414 | error = errmsg, 415 | ok = success 416 | }) 417 | end 418 | 419 | local function perform_read_test(test, endian) 420 | endian = endian or MACHINE_ENDIAN 421 | 422 | local format = test:get_format(endian) 423 | local from_data = test:get_data() 424 | local want_values = test:get_values() 425 | 426 | if PREFIX_ENDIAN then 427 | format = "=" .. PREFIX_ENDIAN_SEP .. format 428 | end 429 | 430 | local pcall = pcall 431 | if not PROTECT_STRUCT then 432 | pcall = function (f,...) return true, f(...) end 433 | end 434 | 435 | local errmsg = nil 436 | 437 | local success, got_values = pcall(struct.read, format, from_data) 438 | if not success then 439 | errmsg = got_values 440 | got_values = nil 441 | end 442 | 443 | success = success and eq(got_values, want_values) 444 | 445 | record("random read test", success, dump { 446 | got_values = got_values, 447 | want_values = want_values, 448 | from_data = as_hex(from_data), 449 | format = format, 450 | error = errmsg, 451 | ok = success 452 | }) 453 | end 454 | 455 | local ScalarTest = Test:new{ 456 | new = function (class, self) 457 | self = Test.new(class, self) 458 | self.is_scalar = true 459 | return self 460 | end, 461 | get_naked = function (self, endian) return self.format end, 462 | get_values = function (self) return { self.value } end, 463 | get_data = function (self) return self.data end, 464 | get_length = function (self) return self.length or 1 end, 465 | get_endian = function (self) return self.endian, self.endian end, 466 | } 467 | 468 | local EmptyTest = ScalarTest:new{ 469 | format = "", 470 | value = nil, 471 | data = "", 472 | length = 0, 473 | endian = nil 474 | } 475 | 476 | -- A compound test models a sequence of unnested or ungrouped formats. 477 | local CompoundTest = Test:new{ 478 | new = 479 | function (class, self) 480 | self = Test.new(class, self) 481 | self.tests = {} 482 | return self 483 | end, 484 | 485 | add_test = 486 | function (self, t) 487 | table.insert(self.tests, t) 488 | return self 489 | end, 490 | 491 | get_data = 492 | function (self) 493 | return table.concat(list.map(self.tests, 494 | function (t) return t:get_data() end)) 495 | end, 496 | 497 | get_format = 498 | function (self, endian) 499 | return table.concat(list.map(self.tests, 500 | function (t) 501 | local fmt 502 | fmt,endian = t:get_format(endian) 503 | return fmt 504 | end), 505 | COMPOUND_SEP), endian 506 | end, 507 | 508 | get_endian = 509 | function (self) 510 | local n = #(self.tests) 511 | if n == 0 then return nil,nil end 512 | local before, after 513 | for i=1,n do 514 | local b,a = self.tests[i]:get_endian() 515 | before = before or b 516 | after = a or after 517 | end 518 | return before, after 519 | end, 520 | 521 | get_values = 522 | function (self) 523 | local accu = {} 524 | list.iter(self.tests, 525 | function (t) 526 | list.merge(accu, t:get_values()) 527 | end) 528 | return accu 529 | end, 530 | 531 | get_length = 532 | function (self) 533 | return #(self.tests) 534 | end, 535 | } 536 | 537 | -- A ProxyTest just resends its messages to a test named in self.body. 538 | local ProxyTest = Test:new{ 539 | new = 540 | function (class, self, body) 541 | local self = Test.new(class, self) 542 | self.body = self.body or body 543 | return self 544 | end, 545 | 546 | get_naked = 547 | function (self) 548 | return self.body:get_naked() 549 | end, 550 | 551 | get_data = 552 | function (self) 553 | return self.body:get_data() 554 | end, 555 | 556 | get_values = 557 | function (self) 558 | return self.body:get_values() 559 | end, 560 | 561 | get_length = 562 | function (self) 563 | return self.body:get_length() 564 | end, 565 | 566 | get_endian = 567 | function (self) 568 | return self.body:get_endian() 569 | end 570 | } 571 | 572 | -- Gives the self.body test a self.name. 573 | local NamedTest = ProxyTest:new{ 574 | new = 575 | function (class, self) 576 | self = ProxyTest.new(class, self) 577 | assert(self.name, "NamedTest unnamed") 578 | return self 579 | end, 580 | 581 | get_naked = 582 | function (self) 583 | return self.name ..":" .. self.body:get_naked() 584 | end, 585 | 586 | get_values = 587 | function (self) 588 | local values = self.body:get_values() 589 | local key, value 590 | for k,v in ipairs(values) do 591 | assert(nil == value, 592 | "NamedTest bound to multiple values") 593 | value = v 594 | key = k 595 | end 596 | for k,v in pairs(values) do 597 | assert(nil == value or key == k, 598 | "NamedTest bound to multiple values") 599 | value = v 600 | end 601 | assert(nil ~= value, "NamedTest unbound to a value") 602 | return { [self.name] = value } 603 | end 604 | } 605 | 606 | -- Models a test group nested in curlies. 607 | local GroupedTest = ProxyTest:new{ 608 | get_format = 609 | function (self, endian) 610 | local fmt,after = self.body:get_format(endian) 611 | return "{"..fmt.."}", after 612 | end, 613 | 614 | get_values = 615 | function (self) 616 | return { self.body:get_values() } 617 | end, 618 | 619 | get_length = 620 | function (self) 621 | return 1 622 | end 623 | } 624 | 625 | -- Models a test nested within parens. 626 | local NestedTest = ProxyTest:new{ 627 | get_format = 628 | function (self, endian) 629 | local fmt,after = self.body:get_format(endian) 630 | return "("..fmt..")", after 631 | end, 632 | 633 | get_length = 634 | function (self) 635 | return 1 636 | end 637 | } 638 | 639 | local LeftRepetitionTest = ProxyTest:new{ 640 | new = 641 | function (class, self) 642 | self = ProxyTest.new(class, self) 643 | self.count = self.count or 1 644 | return self 645 | end, 646 | 647 | get_format = 648 | function (self, endian) 649 | local before = self.body:get_endian() 650 | local fmt, after = self.body:get_format(endian) 651 | before = before or endian 652 | after = after or before 653 | after = self.count > 0 and after or endian 654 | 655 | assert(after == before or self.count == 0, 656 | ("Non-idempotent left repetition for format '%s'"):format(fmt)) 657 | fmt = "("..fmt..")" 658 | return string.format( 659 | "%d%s*%s%s", self.count, TOKEN_SEP, TOKEN_SEP, fmt), after 660 | end, 661 | 662 | get_data = 663 | function (self) 664 | return self.body:get_data():rep(self.count) 665 | end, 666 | 667 | get_values = 668 | function (self) 669 | return list.rep(self.body:get_values(), self.count) 670 | end, 671 | 672 | get_length = 673 | function (self) 674 | return 1 675 | end, 676 | 677 | get_endian = 678 | function (self) 679 | if self.count > 0 then 680 | return self.body:get_endian() 681 | end 682 | end, 683 | } 684 | 685 | -- Test generation state. 686 | local State = { 687 | new = 688 | function (class, seed) 689 | local self = { 690 | rand = make_rand(seed), --data source 691 | endian = nil, --nil|">"|"<" 692 | size = 56, 693 | precision = 16, 694 | depth = 0, --recursion depth 695 | uniq = 0 --unique name seq 696 | } 697 | return bless(self, class) 698 | end, 699 | 700 | is_big_endian = 701 | function (self) 702 | return self.endian == ">" --XXX: should test vs. native also? 703 | end, 704 | 705 | raw_of_list = --byte list, endian -> byte string 706 | function (self, t, reverse) 707 | if nil == reverse then reverse = self:is_big_endian() end 708 | 709 | local chars = list.map(t, char) 710 | if reverse then 711 | chars = list.reverse(chars) 712 | end 713 | return table.concat(chars) 714 | end, 715 | 716 | raw_of_int = --int, size, endian -> byte string 717 | function (self, val, W, reverse) 718 | return self:raw_of_list(int_to_bytes_le(val, W), reverse) 719 | end 720 | } 721 | 722 | local function zero_pad_to_length(s, len) 723 | local pad_len = len - #s 724 | if pad_len <= 0 then 725 | return s 726 | end 727 | return s .. ("\0"):rep(pad_len) 728 | end 729 | 730 | local function cruft_pad_to_length(s, len) 731 | local cruft = "cruft" 732 | while #s < len do 733 | local ix = 1 + #s % #cruft 734 | s = s .. cruft:sub(ix,ix) 735 | end 736 | return s 737 | end 738 | 739 | -- Scalar test generators 740 | -- TODO: fW 741 | -- manually tested: @A +A -A 742 | local ScalarGen 743 | ScalarGen = { 744 | 745 | bitmask = 746 | function (state, w) 747 | local W = ceil(w/8) 748 | w = w < MAX_BITS and w or MAX_BITS 749 | local value = state.rand() 750 | local value = fmod(value,2^w) 751 | 752 | local write_bits = {} 753 | local read_bits = {} 754 | local n = 0 755 | local v = value 756 | 757 | while n < 8*W do 758 | local r = v % 2 759 | 760 | table.insert(read_bits, r == 1) 761 | if v > 0 then 762 | table.insert(write_bits, r == 1) 763 | end 764 | 765 | v = (v - r) / 2 766 | n = n+1 767 | end 768 | 769 | local write_test = ScalarTest:new{ 770 | endian = state.endian, 771 | format = ("m%d"):format(W), 772 | value = write_bits, 773 | data = state:raw_of_int(value, W) 774 | } 775 | local read_test = ScalarTest:new{ 776 | endian = state.endian, 777 | format = ("m%d"):format(W), 778 | value = read_bits, 779 | data = state:raw_of_int(value, W) 780 | } 781 | return write_test, read_test 782 | end, 783 | 784 | skip_pad = 785 | function (state, w) 786 | local W = ceil(w/8) 787 | return ScalarTest:new{ 788 | format = "x"..W, 789 | value = nil, --? 790 | data = zero_pad_to_length("", W) 791 | }, 792 | ScalarTest:new{ 793 | format = "x"..W, 794 | value = nil, --? 795 | data = cruft_pad_to_length("", W) 796 | } 797 | end, 798 | 799 | cstring_implicit = 800 | function (state, w) 801 | local W = ceil(w/8) 802 | local len = W-1 803 | local s = string_rand(state.rand)(len) 804 | 805 | local write_test = ScalarTest:new{ 806 | endian = state.endian, 807 | format = "z", 808 | value = s, 809 | data = s.."\0" 810 | } 811 | local read_test = ScalarTest:new{ 812 | endian = state.endian, 813 | format = ("z%d"):format(W), 814 | value = s, 815 | data = write_test.data 816 | } 817 | return write_test, read_test 818 | end, 819 | 820 | cstring_short = 821 | function (state, w) 822 | local W = ceil(w/8) -->0 823 | local len = state.rand() % (W) -- must be < W 824 | local s = string_rand(state.rand)(len) 825 | 826 | local data = s.."\0" 827 | local write_data = zero_pad_to_length(data, W) 828 | local read_data = cruft_pad_to_length(data, W) 829 | 830 | local write_test = ScalarTest:new{ 831 | endian = state.endian, 832 | format = ("z%d"):format(W), 833 | value = s, 834 | data = write_data:sub(1,W) 835 | } 836 | local read_test = ScalarTest:new{ 837 | endian = state.endian, 838 | format = ("z%d"):format(W), 839 | value = s, 840 | data = read_data:sub(1,W) 841 | } 842 | return write_test, read_test 843 | end, 844 | 845 | cstring_long = 846 | function (state, w) 847 | local W = ceil(w/8) -->0 848 | local s = string_rand(state.rand)(W-1) 849 | local cruft = string_rand(state.rand,1,W)() 850 | 851 | local write_test = ScalarTest:new{ 852 | endian = state.endian, 853 | format = ("z%d"):format(W), 854 | value = s .. cruft, 855 | data = s.."\0" 856 | } 857 | local read_test = ScalarTest:new{ 858 | endian = state.endian, 859 | format = ("z%d"):format(W), 860 | value = s, 861 | data = write_test.data 862 | } 863 | return write_test, read_test 864 | end, 865 | 866 | boolean = 867 | function (state, w) 868 | local W = ceil(w/8) 869 | local value = fmod((state.rand() % 2) * state.rand(), 2^8) 870 | local flag = value ~= 0 and w > 0 871 | if not flag then value = 0 end 872 | 873 | local write_test = ScalarTest:new{ 874 | endian = state.endian, 875 | format = ("b%d"):format(W), 876 | value = flag, 877 | data = state:raw_of_int(flag and 1 or 0, W) 878 | } 879 | local read_test = ScalarTest:new{ 880 | endian = state.endian, 881 | format = ("b%d"):format(W), 882 | value = flag, 883 | data = state:raw_of_int(value, W) 884 | } 885 | return write_test, read_test 886 | end, 887 | 888 | unsigned_int = 889 | function (state, w) 890 | local W = ceil(w/8) 891 | w = w < MAX_BITS and w or MAX_BITS 892 | local value = state.rand() 893 | local value = fmod(value,2^w) 894 | return ScalarTest:new{ 895 | endian = state.endian, 896 | format = ("u%d"):format(W), 897 | value = value, 898 | data = state:raw_of_int(value, W) 899 | } 900 | end, 901 | 902 | signed_int = 903 | function (state, w) 904 | local W = ceil(w/8) 905 | w = w < MAX_BITS and w or MAX_BITS 906 | local bias = 2^(w-1) 907 | local value = (fmod(state.rand(), 2^w) - bias) 908 | if w == 0 then value = 0 end 909 | return ScalarTest:new{ 910 | endian = state.endian, 911 | format = ("i%d"):format(W), 912 | value = value, 913 | data = state:raw_of_int(value, W) 914 | } 915 | end, 916 | 917 | --[[ 918 | fixed_rational_bit = 919 | function (state, w, f) 920 | local W = ceil(w/8) 921 | local test = ScalarGen.signed_int(state, w) 922 | test.format = ("P%d.%d"):format(8*W - f, f) 923 | test.value = test.value / 2^f 924 | return test 925 | end, 926 | 927 | fixed_rational_byte = 928 | function (state, w, f) 929 | local W = ceil(w/8) 930 | local F = ceil(f/8) 931 | local test = ScalarGen.signed_int(state, w) 932 | test.format = ("p%d.%d"):format(W - F, F) 933 | test.value = test.value / 2^(8*F) 934 | return test 935 | end, 936 | 937 | --]] 938 | string_implicit = 939 | function (state, w) 940 | local W = ceil(w/8) 941 | local len = W 942 | local s = string_rand(state.rand)(len) 943 | 944 | local write_test = ScalarTest:new{ 945 | endian = state.endian, 946 | format = "s", 947 | value = s, 948 | data = s 949 | } 950 | local read_test = ScalarTest:new{ 951 | endian = state.endian, 952 | format = ("s%d"):format(W), 953 | value = s, 954 | data = write_test.data 955 | } 956 | return write_test, read_test 957 | end, 958 | 959 | string_short = 960 | function (state, w) 961 | local W = ceil(w/8) 962 | local len = state.rand() % (W+1) 963 | local s = string_rand(state.rand)(len) 964 | local pad = ("\0"):rep(W - #s) 965 | 966 | local write_test = ScalarTest:new{ 967 | endian = state.endian, 968 | format = ("s%d"):format(W), 969 | value = s, 970 | data = s .. pad 971 | } 972 | local read_test = ScalarTest:new{ 973 | endian = state.endian, 974 | format = ("s%d"):format(W), 975 | value = s .. pad, 976 | data = write_test.data 977 | } 978 | return write_test, read_test 979 | end, 980 | 981 | string_long = 982 | function (state, w) 983 | local W = ceil(w/8) 984 | local s = string_rand(state.rand)(W) 985 | local cruft = string_rand(state.rand,0,W)() 986 | 987 | local write_test = ScalarTest:new{ 988 | endian = state.endian, 989 | format = ("s%d"):format(W), 990 | value = s .. cruft, 991 | data = s 992 | } 993 | local read_test = ScalarTest:new{ 994 | endian = state.endian, 995 | format = ("s%d"):format(W), 996 | value = s, 997 | data = write_test.data 998 | } 999 | return write_test, read_test 1000 | end, 1001 | 1002 | counted_string = 1003 | function (state, w) 1004 | local W = ceil(w/8) 1005 | w = w > 4 and 4 or w 1006 | local count = fmod(state.rand(), 2^w) 1007 | local count_data = state:raw_of_int(count, W) 1008 | local s = string_rand(state.rand)(count) 1009 | return ScalarTest:new{ 1010 | endian = state.endian, 1011 | format = ("c%d"):format(W), 1012 | value = s, 1013 | data = count_data..s 1014 | } 1015 | end, 1016 | } 1017 | 1018 | local EmptyGen = {} 1019 | 1020 | if USE.EMPTY then 1021 | function EmptyGen.empty(state) return EmptyTest:new() end 1022 | function EmptyGen.space(state) return EmptyTest:new{ format = " " } end 1023 | function EmptyGen.tab(state) return EmptyTest:new{ format = "\t" } end 1024 | function EmptyGen.nl(state) return EmptyTest:new{ format = "\n" } end 1025 | --function EmptyGen.cr(state) return EmptyTest:new{ format = "\r" } end 1026 | end 1027 | 1028 | if USE.ENDIAN then 1029 | function EmptyGen.big_endian(state) 1030 | state.endian = ">" 1031 | local self = EmptyTest:new() 1032 | self.endian = state.endian 1033 | return self 1034 | end 1035 | 1036 | function EmptyGen.little_endian(state) 1037 | state.endian = "<" 1038 | local self = EmptyTest:new() 1039 | self.endian = state.endian 1040 | return self 1041 | end 1042 | end 1043 | 1044 | -- Leaf tests don't recurse. 1045 | local LeafGen = {} 1046 | for _,f in pairs(EmptyGen) do table.insert(LeafGen, f) end 1047 | for _,f in pairs(ScalarGen) do table.insert(LeafGen, f) end 1048 | 1049 | local gen_random_leaf 1050 | 1051 | local function gen_random_leaf_for_write(state, w, f) 1052 | local i = 1 + state.rand() % #LeafGen 1053 | local gen = LeafGen[i] 1054 | return gen(state, w or state.size, f or state.precision) 1055 | end 1056 | 1057 | local function gen_random_leaf_for_read(state, w, f) 1058 | local write,read = gen_random_leaf_for_write(state, w, f) 1059 | read = read or write 1060 | return read 1061 | end 1062 | 1063 | -- List of all test generators (populated below) 1064 | local TestGen = {} 1065 | 1066 | local function gen_random_test(state, w, f) 1067 | if state.depth >= CONF.MAX_DEPTH then 1068 | return gen_random_leaf(state, w, f) 1069 | end 1070 | state.depth = state.depth + 1 1071 | local i = 1 + state.rand() % #TestGen 1072 | local gen = TestGen[i] 1073 | local t = gen(state, w, f) 1074 | state.depth = state.depth - 1 1075 | return t 1076 | end 1077 | 1078 | -- Populate the test generators. 1079 | table.insert(TestGen, 1080 | function (...) 1081 | return gen_random_leaf(...) 1082 | end) 1083 | 1084 | if USE.COMPOUND then 1085 | table.insert(TestGen, 1086 | function (state,w,f) 1087 | local n = CONF.COMPOUND_LENGTH 1088 | local t = CompoundTest:new() 1089 | for i=1,n do 1090 | t:add_test(gen_random_test(state, w, f)) 1091 | end 1092 | return t 1093 | end) 1094 | end 1095 | 1096 | if USE.NESTING then 1097 | table.insert(TestGen, 1098 | function (state,w,f) 1099 | local t = gen_random_test(state, w, f) 1100 | return NestedTest:new{body = t} 1101 | end) 1102 | end 1103 | 1104 | if USE.GROUPING then 1105 | table.insert(TestGen, 1106 | function (state,w,f) 1107 | local t = gen_random_test(state, w, f) 1108 | return GroupedTest:new{body = t} 1109 | end) 1110 | end 1111 | 1112 | if USE.NAMING then 1113 | table.insert(TestGen, 1114 | function (state,w,f) 1115 | local t = gen_random_test(state, w, f) 1116 | local vs = t:get_values() 1117 | if #vs == 1 and t.is_scalar then 1118 | state.uniq = state.uniq + 1 1119 | t = NamedTest:new{body = t, name="u"..state.uniq} 1120 | end 1121 | return t 1122 | end) 1123 | end 1124 | 1125 | if USE.LEFT_REPETITION then 1126 | table.insert(TestGen, 1127 | function (state,w,f) 1128 | local t = gen_random_test(state, w, f) 1129 | local before, after = t:get_endian() 1130 | if before == after then 1131 | t = LeftRepetitionTest:new{ 1132 | body = t, 1133 | count = state.rand() % 5 1134 | } 1135 | end 1136 | return t 1137 | end) 1138 | end 1139 | 1140 | local function main_write() 1141 | test.group "randomly generated write tests" 1142 | local state = State:new(CONF.SEED or os.time()) 1143 | state.size = CONF.WIDTH 1144 | state.precision = CONF.PRECISION 1145 | 1146 | gen_random_leaf = gen_random_leaf_for_write 1147 | 1148 | for k,gen in pairs(EmptyGen) do 1149 | for _,e in ipairs{"<",">"} do 1150 | state.endian = e 1151 | local t = gen(state, state.size, state.precision) 1152 | perform_write_test(t, "=") 1153 | end 1154 | end 1155 | 1156 | for k,gen in pairs(ScalarGen) do 1157 | for _,e in ipairs{"<",">"} do 1158 | state.endian = e 1159 | local t = gen(state, state.size, state.precision) 1160 | perform_write_test(t, "=") 1161 | end 1162 | end 1163 | 1164 | for i=1,CONF.NROF_TESTS do 1165 | for _,e in ipairs{"<",">"} do 1166 | state.endian = e 1167 | local t = gen_random_test(state, state.size, state.precision) 1168 | perform_write_test(t) 1169 | end 1170 | end 1171 | end 1172 | 1173 | local function main_read() 1174 | test.group "randomly generated read tests" 1175 | local state = State:new(CONF.SEED or os.time()) 1176 | state.size = CONF.WIDTH 1177 | state.precision = CONF.PRECISION 1178 | 1179 | gen_random_leaf = gen_random_leaf_for_read 1180 | 1181 | for k,gen in pairs(EmptyGen) do 1182 | for _,e in ipairs{"<",">"} do 1183 | state.endian = e 1184 | local w,r = gen(state, state.size, state.precision) 1185 | local t = r or w 1186 | perform_read_test(t, "=") 1187 | end 1188 | end 1189 | 1190 | for k,gen in pairs(ScalarGen) do 1191 | for _,e in ipairs{"<",">"} do 1192 | state.endian = e 1193 | local w,r = gen(state, state.size, state.precision) 1194 | local t = r or w 1195 | perform_read_test(t, "=") 1196 | end 1197 | end 1198 | 1199 | for i=1,CONF.NROF_TESTS do 1200 | for _,e in ipairs{"<",">"} do 1201 | state.endian = e 1202 | local t = gen_random_test(state, state.size, state.precision) 1203 | perform_read_test(t) 1204 | end 1205 | end 1206 | end 1207 | 1208 | for _,kind in ipairs(TEST_KINDS) do 1209 | if kind == "read" then 1210 | main_read() 1211 | elseif kind == "write" then 1212 | main_write() 1213 | else 1214 | local function opt_of_keyword(k) 1215 | local o = k:lower():gsub("_", "-") 1216 | return o 1217 | end 1218 | 1219 | io.write"usage: read|write\n\n" 1220 | 1221 | io.write"Features:\n" 1222 | io.write("\tnone") 1223 | for i,k in ipairs(keys(USE)) do 1224 | io.write", " 1225 | if i % 5 == 0 then io.write"\n\t" end 1226 | io.write(opt_of_keyword(k)) 1227 | end 1228 | io.write"\n" 1229 | 1230 | io.write"Options:\n" 1231 | for k, v in pairs(CONF) do 1232 | io.write("\t",opt_of_keyword(k),"=",tostring(v),"\n") 1233 | end 1234 | break 1235 | end 1236 | end 1237 | -------------------------------------------------------------------------------- /vstruct-2.0.1-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "vstruct" 2 | version = "2.0.1-1" 3 | source = { 4 | url = "git+https://github.com/ToxicFrog/vstruct.git", 5 | tag = "v2.0.1" 6 | } 7 | description = { 8 | summary = "Lua library to manipulate binary data", 9 | homepage = "https://github.com/ToxicFrog/vstruct", 10 | } 11 | dependencies = { 12 | "lua >= 5.1, < 5.3" 13 | } 14 | build = { 15 | type = "builtin", 16 | modules = { 17 | ["vstruct.api"] = "api.lua", 18 | ["vstruct.ast"] = "ast.lua", 19 | ["vstruct.ast.Bitpack"] = "ast/Bitpack.lua", 20 | ["vstruct.ast.IO"] = "ast/IO.lua", 21 | ["vstruct.ast.List"] = "ast/List.lua", 22 | ["vstruct.ast.Name"] = "ast/Name.lua", 23 | ["vstruct.ast.Node"] = "ast/Node.lua", 24 | ["vstruct.ast.Repeat"] = "ast/Repeat.lua", 25 | ["vstruct.ast.Root"] = "ast/Root.lua", 26 | ["vstruct.ast.Table"] = "ast/Table.lua", 27 | ["vstruct.compat1x"] = "compat1x.lua", 28 | ["vstruct.cursor"] = "cursor.lua", 29 | ["vstruct"] = "init.lua", 30 | ["vstruct.io"] = "io.lua", 31 | ["vstruct.io.a"] = "io/a.lua", 32 | ["vstruct.io.b"] = "io/b.lua", 33 | ["vstruct.io.bigendian"] = "io/bigendian.lua", 34 | ["vstruct.io.c"] = "io/c.lua", 35 | ["vstruct.io.defaults"] = "io/defaults.lua", 36 | ["vstruct.io.endianness"] = "io/endianness.lua", 37 | ["vstruct.io.f"] = "io/f.lua", 38 | ["vstruct.io.hostendian"] = "io/hostendian.lua", 39 | ["vstruct.io.i"] = "io/i.lua", 40 | ["vstruct.io.littleendian"] = "io/littleendian.lua", 41 | ["vstruct.io.m"] = "io/m.lua", 42 | ["vstruct.io.p"] = "io/p.lua", 43 | ["vstruct.io.s"] = "io/s.lua", 44 | ["vstruct.io.seekb"] = "io/seekb.lua", 45 | ["vstruct.io.seekf"] = "io/seekf.lua", 46 | ["vstruct.io.seekto"] = "io/seekto.lua", 47 | ["vstruct.io.u"] = "io/u.lua", 48 | ["vstruct.io.x"] = "io/x.lua", 49 | ["vstruct.io.z"] = "io/z.lua", 50 | ["vstruct.lexer"] = "lexer.lua", 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /vstruct-2.0.2-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "vstruct" 2 | version = "2.0.2-1" 3 | source = { 4 | url = "git+https://github.com/ToxicFrog/vstruct.git", 5 | tag = "v2.0.1" 6 | } 7 | description = { 8 | summary = "Lua library to manipulate binary data", 9 | homepage = "https://github.com/ToxicFrog/vstruct", 10 | } 11 | dependencies = { 12 | "lua >= 5.1, < 5.3" 13 | } 14 | build = { 15 | type = "builtin", 16 | modules = { 17 | ["vstruct.api"] = "api.lua", 18 | ["vstruct.ast"] = "ast.lua", 19 | ["vstruct.ast.Bitpack"] = "ast/Bitpack.lua", 20 | ["vstruct.ast.IO"] = "ast/IO.lua", 21 | ["vstruct.ast.List"] = "ast/List.lua", 22 | ["vstruct.ast.Name"] = "ast/Name.lua", 23 | ["vstruct.ast.Node"] = "ast/Node.lua", 24 | ["vstruct.ast.Repeat"] = "ast/Repeat.lua", 25 | ["vstruct.ast.Root"] = "ast/Root.lua", 26 | ["vstruct.ast.Table"] = "ast/Table.lua", 27 | ["vstruct.compat1x"] = "compat1x.lua", 28 | ["vstruct.cursor"] = "cursor.lua", 29 | ["vstruct"] = "init.lua", 30 | ["vstruct.io"] = "io.lua", 31 | ["vstruct.io.a"] = "io/a.lua", 32 | ["vstruct.io.b"] = "io/b.lua", 33 | ["vstruct.io.bigendian"] = "io/bigendian.lua", 34 | ["vstruct.io.c"] = "io/c.lua", 35 | ["vstruct.io.defaults"] = "io/defaults.lua", 36 | ["vstruct.io.endianness"] = "io/endianness.lua", 37 | ["vstruct.io.f"] = "io/f.lua", 38 | ["vstruct.io.hostendian"] = "io/hostendian.lua", 39 | ["vstruct.io.i"] = "io/i.lua", 40 | ["vstruct.io.littleendian"] = "io/littleendian.lua", 41 | ["vstruct.io.m"] = "io/m.lua", 42 | ["vstruct.io.p"] = "io/p.lua", 43 | ["vstruct.io.s"] = "io/s.lua", 44 | ["vstruct.io.seekb"] = "io/seekb.lua", 45 | ["vstruct.io.seekf"] = "io/seekf.lua", 46 | ["vstruct.io.seekto"] = "io/seekto.lua", 47 | ["vstruct.io.u"] = "io/u.lua", 48 | ["vstruct.io.x"] = "io/x.lua", 49 | ["vstruct.io.z"] = "io/z.lua", 50 | ["vstruct.lexer"] = "lexer.lua", 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /vstruct-2.1.0-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "vstruct" 2 | version = "2.1.0-1" 3 | source = { 4 | url = "git+https://github.com/ToxicFrog/vstruct.git", 5 | tag = "v2.1.0" 6 | } 7 | description = { 8 | summary = "Lua library to manipulate binary data", 9 | homepage = "https://github.com/ToxicFrog/vstruct", 10 | } 11 | dependencies = { 12 | "lua >= 5.1, <= 5.4" 13 | } 14 | build = { 15 | type = "builtin", 16 | modules = { 17 | ["vstruct.api"] = "api.lua", 18 | ["vstruct.ast"] = "ast.lua", 19 | ["vstruct.ast.Bitpack"] = "ast/Bitpack.lua", 20 | ["vstruct.ast.IO"] = "ast/IO.lua", 21 | ["vstruct.ast.List"] = "ast/List.lua", 22 | ["vstruct.ast.Name"] = "ast/Name.lua", 23 | ["vstruct.ast.Node"] = "ast/Node.lua", 24 | ["vstruct.ast.Repeat"] = "ast/Repeat.lua", 25 | ["vstruct.ast.Root"] = "ast/Root.lua", 26 | ["vstruct.ast.Table"] = "ast/Table.lua", 27 | ["vstruct.compat1x"] = "compat1x.lua", 28 | ["vstruct.cursor"] = "cursor.lua", 29 | ["vstruct"] = "init.lua", 30 | ["vstruct.io"] = "io.lua", 31 | ["vstruct.io.a"] = "io/a.lua", 32 | ["vstruct.io.b"] = "io/b.lua", 33 | ["vstruct.io.bigendian"] = "io/bigendian.lua", 34 | ["vstruct.io.c"] = "io/c.lua", 35 | ["vstruct.io.defaults"] = "io/defaults.lua", 36 | ["vstruct.io.endianness"] = "io/endianness.lua", 37 | ["vstruct.io.f"] = "io/f.lua", 38 | ["vstruct.io.hostendian"] = "io/hostendian.lua", 39 | ["vstruct.io.i"] = "io/i.lua", 40 | ["vstruct.io.littleendian"] = "io/littleendian.lua", 41 | ["vstruct.io.m"] = "io/m.lua", 42 | ["vstruct.io.p"] = "io/p.lua", 43 | ["vstruct.io.s"] = "io/s.lua", 44 | ["vstruct.io.seekb"] = "io/seekb.lua", 45 | ["vstruct.io.seekf"] = "io/seekf.lua", 46 | ["vstruct.io.seekto"] = "io/seekto.lua", 47 | ["vstruct.io.u"] = "io/u.lua", 48 | ["vstruct.io.x"] = "io/x.lua", 49 | ["vstruct.io.z"] = "io/z.lua", 50 | ["vstruct.lexer"] = "lexer.lua", 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /vstruct-2.1.1-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "vstruct" 2 | version = "2.1.1-1" 3 | source = { 4 | url = "git+https://github.com/ToxicFrog/vstruct.git", 5 | tag = "v2.1.1" 6 | } 7 | description = { 8 | summary = "Lua library to manipulate binary data", 9 | homepage = "https://github.com/ToxicFrog/vstruct", 10 | } 11 | dependencies = { 12 | "lua >= 5.1, <= 5.4" 13 | } 14 | build = { 15 | type = "builtin", 16 | modules = { 17 | ["vstruct.api"] = "api.lua", 18 | ["vstruct.ast"] = "ast.lua", 19 | ["vstruct.ast.Bitpack"] = "ast/Bitpack.lua", 20 | ["vstruct.ast.IO"] = "ast/IO.lua", 21 | ["vstruct.ast.List"] = "ast/List.lua", 22 | ["vstruct.ast.Name"] = "ast/Name.lua", 23 | ["vstruct.ast.Node"] = "ast/Node.lua", 24 | ["vstruct.ast.Repeat"] = "ast/Repeat.lua", 25 | ["vstruct.ast.Root"] = "ast/Root.lua", 26 | ["vstruct.ast.Table"] = "ast/Table.lua", 27 | ["vstruct.compat1x"] = "compat1x.lua", 28 | ["vstruct.cursor"] = "cursor.lua", 29 | ["vstruct.frexp"] = "frexp.lua", 30 | ["vstruct"] = "init.lua", 31 | ["vstruct.io"] = "io.lua", 32 | ["vstruct.io.a"] = "io/a.lua", 33 | ["vstruct.io.b"] = "io/b.lua", 34 | ["vstruct.io.bigendian"] = "io/bigendian.lua", 35 | ["vstruct.io.c"] = "io/c.lua", 36 | ["vstruct.io.defaults"] = "io/defaults.lua", 37 | ["vstruct.io.endianness"] = "io/endianness.lua", 38 | ["vstruct.io.f"] = "io/f.lua", 39 | ["vstruct.io.hostendian"] = "io/hostendian.lua", 40 | ["vstruct.io.i"] = "io/i.lua", 41 | ["vstruct.io.littleendian"] = "io/littleendian.lua", 42 | ["vstruct.io.m"] = "io/m.lua", 43 | ["vstruct.io.p"] = "io/p.lua", 44 | ["vstruct.io.s"] = "io/s.lua", 45 | ["vstruct.io.seekb"] = "io/seekb.lua", 46 | ["vstruct.io.seekf"] = "io/seekf.lua", 47 | ["vstruct.io.seekto"] = "io/seekto.lua", 48 | ["vstruct.io.u"] = "io/u.lua", 49 | ["vstruct.io.x"] = "io/x.lua", 50 | ["vstruct.io.z"] = "io/z.lua", 51 | ["vstruct.lexer"] = "lexer.lua", 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /vstruct-2.2.0-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "vstruct" 2 | version = "2.2.0-1" 3 | source = { 4 | url = "git+https://github.com/ToxicFrog/vstruct.git", 5 | tag = "v2.2.0" 6 | } 7 | description = { 8 | summary = "Lua library to manipulate binary data", 9 | homepage = "https://github.com/ToxicFrog/vstruct", 10 | } 11 | dependencies = { 12 | "lua >= 5.1, <= 5.4" 13 | } 14 | build = { 15 | type = "builtin", 16 | modules = { 17 | ["vstruct.api"] = "api.lua", 18 | ["vstruct.ast"] = "ast.lua", 19 | ["vstruct.ast.Bitpack"] = "ast/Bitpack.lua", 20 | ["vstruct.ast.IO"] = "ast/IO.lua", 21 | ["vstruct.ast.List"] = "ast/List.lua", 22 | ["vstruct.ast.Name"] = "ast/Name.lua", 23 | ["vstruct.ast.Node"] = "ast/Node.lua", 24 | ["vstruct.ast.Number"] = "ast/Number.lua", 25 | ["vstruct.ast.Repeat"] = "ast/Repeat.lua", 26 | ["vstruct.ast.Root"] = "ast/Root.lua", 27 | ["vstruct.ast.Table"] = "ast/Table.lua", 28 | ["vstruct.compat1x"] = "compat1x.lua", 29 | ["vstruct.cursor"] = "cursor.lua", 30 | ["vstruct.frexp"] = "frexp.lua", 31 | ["vstruct"] = "init.lua", 32 | ["vstruct.io"] = "io.lua", 33 | ["vstruct.io.a"] = "io/a.lua", 34 | ["vstruct.io.b"] = "io/b.lua", 35 | ["vstruct.io.bigendian"] = "io/bigendian.lua", 36 | ["vstruct.io.c"] = "io/c.lua", 37 | ["vstruct.io.defaults"] = "io/defaults.lua", 38 | ["vstruct.io.endianness"] = "io/endianness.lua", 39 | ["vstruct.io.f"] = "io/f.lua", 40 | ["vstruct.io.hostendian"] = "io/hostendian.lua", 41 | ["vstruct.io.i"] = "io/i.lua", 42 | ["vstruct.io.littleendian"] = "io/littleendian.lua", 43 | ["vstruct.io.m"] = "io/m.lua", 44 | ["vstruct.io.p"] = "io/p.lua", 45 | ["vstruct.io.s"] = "io/s.lua", 46 | ["vstruct.io.seekb"] = "io/seekb.lua", 47 | ["vstruct.io.seekf"] = "io/seekf.lua", 48 | ["vstruct.io.seekto"] = "io/seekto.lua", 49 | ["vstruct.io.u"] = "io/u.lua", 50 | ["vstruct.io.x"] = "io/x.lua", 51 | ["vstruct.io.z"] = "io/z.lua", 52 | ["vstruct.lexer"] = "lexer.lua", 53 | } 54 | } 55 | --------------------------------------------------------------------------------