├── .gitattributes ├── .github └── FUNDING.yml ├── .gitignore ├── LICENCE ├── README.md ├── build.zig ├── build.zig.zon ├── design ├── logo.png ├── logo.xcf ├── social-media-preview.png └── social-media-preview.xcf └── s2s.zig /.gitattributes: -------------------------------------------------------------------------------- 1 | *.zig text=auto eol=lf 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: MasterQ32 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | zig-cache/ 2 | zig-out/ 3 | .zig-cache/ 4 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022 Felix "xq" Queißner 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 14 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 15 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 16 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 19 | OR OTHER DEALINGS IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # struct to stream | stream to struct 2 | 3 | A Zig binary serialization format and library. 4 | 5 | ![Project logo](design/logo.png) 6 | 7 | ## Features 8 | 9 | - Convert (nearly) any Zig runtime datatype to binary data and back. 10 | - Computes a stream signature that prevents deserialization of invalid data. 11 | - No support for graph like structures. Everything is considered to be tree data. 12 | 13 | **Unsupported types**: 14 | 15 | - All `comptime` only types 16 | - Unbound pointers (c pointers, pointer to many) 17 | - `volatile` pointers 18 | - Untagged or `external` unions 19 | - Opaque types 20 | - Function pointers 21 | - Frames 22 | 23 | ## API 24 | 25 | The library itself provides only some APIs, as most of the serialization process is not configurable. 26 | 27 | ```zig 28 | /// Serializes the given `value: T` into the `stream`. 29 | /// - `stream` is a instance of `std.io.Writer` 30 | /// - `T` is the type to serialize 31 | /// - `value` is the instance to serialize. 32 | fn serialize(stream: anytype, comptime T: type, value: T) StreamError!void; 33 | 34 | /// Deserializes a value of type `T` from the `stream`. 35 | /// - `stream` is a instance of `std.io.Reader` 36 | /// - `T` is the type to deserialize 37 | fn deserialize(stream: anytype, comptime T: type) (StreamError || error{UnexpectedData,EndOfStream})!T; 38 | 39 | /// Deserializes a value of type `T` from the `stream`. 40 | /// - `stream` is a instance of `std.io.Reader` 41 | /// - `T` is the type to deserialize 42 | /// - `allocator` is an allocator require to allocate slices and pointers. 43 | /// Result must be freed by using `free()`. 44 | fn deserializeAlloc(stream: anytype, comptime T: type, allocator: std.mem.Allocator) (StreamError || error{ UnexpectedData, OutOfMemory,EndOfStream })!T; 45 | 46 | /// Releases all memory allocated by `deserializeAlloc`. 47 | /// - `allocator` is the allocator passed to `deserializeAlloc`. 48 | /// - `T` is the type that was passed to `deserializeAlloc`. 49 | /// - `value` is the value that was returned by `deserializeAlloc`. 50 | fn free(allocator: std.mem.Allocator, comptime T: type, value: T) void; 51 | ``` 52 | 53 | ## Usage and Development 54 | 55 | ### Adding the library 56 | 57 | Just add the `s2s.zig` as a package to your Zig project. It has no external dependencies. 58 | 59 | ### Running the test suite 60 | 61 | ```sh-session 62 | [user@host s2s]$ zig test s2s.zig 63 | All 3 tests passed. 64 | [user@host s2s]$ 65 | ``` 66 | 67 | ## Project Status 68 | 69 | Most of the serialization/deserialization is implemented for the _trivial_ case. 70 | 71 | Pointers/slices with non-standard alignment aren't properly supported yet. 72 | -------------------------------------------------------------------------------- /build.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | 3 | pub fn build(b: *std.Build) void { 4 | const optimize = b.standardOptimizeOption(.{}); 5 | const target = b.standardTargetOptions(.{}); 6 | 7 | _ = b.addModule("s2s", .{ 8 | .root_source_file = b.path("s2s.zig"), 9 | }); 10 | 11 | const tests = b.addTest(.{ 12 | .root_source_file = b.path("s2s.zig"), 13 | .target = target, 14 | .optimize = optimize, 15 | }); 16 | 17 | const test_step = b.step("test", "Run unit tests"); 18 | const run_test = b.addRunArtifact(tests); 19 | test_step.dependOn(&run_test.step); 20 | } 21 | -------------------------------------------------------------------------------- /build.zig.zon: -------------------------------------------------------------------------------- 1 | .{ 2 | .name = .s2s, 3 | .version = "0.0.1", 4 | .fingerprint = 0x8d8a7bec8d1adfaa, 5 | .dependencies = .{}, 6 | .paths = .{ 7 | "build.zig", 8 | "s2s.zig", 9 | }, 10 | } 11 | -------------------------------------------------------------------------------- /design/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ziglibs/s2s/76d7aea86c1d05afaddab9a1081b6dad68088756/design/logo.png -------------------------------------------------------------------------------- /design/logo.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ziglibs/s2s/76d7aea86c1d05afaddab9a1081b6dad68088756/design/logo.xcf -------------------------------------------------------------------------------- /design/social-media-preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ziglibs/s2s/76d7aea86c1d05afaddab9a1081b6dad68088756/design/social-media-preview.png -------------------------------------------------------------------------------- /design/social-media-preview.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ziglibs/s2s/76d7aea86c1d05afaddab9a1081b6dad68088756/design/social-media-preview.xcf -------------------------------------------------------------------------------- /s2s.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const testing = std.testing; 3 | 4 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// 5 | // Public API: 6 | 7 | /// Serializes the given `value: T` into the `stream`. 8 | /// - `stream` is a instance of `std.io.Writer` 9 | /// - `T` is the type to serialize 10 | /// - `value` is the instance to serialize. 11 | pub fn serialize(stream: anytype, comptime T: type, value: T) @TypeOf(stream).Error!void { 12 | comptime validateTopLevelType(T); 13 | const type_hash = comptime computeTypeHash(T); 14 | 15 | try stream.writeAll(type_hash[0..]); 16 | try serializeRecursive(stream, T, value); 17 | } 18 | 19 | /// Deserializes a value of type `T` from the `stream`. 20 | /// - `stream` is a instance of `std.io.Reader` 21 | /// - `T` is the type to deserialize 22 | pub fn deserialize( 23 | stream: anytype, 24 | comptime T: type, 25 | ) (@TypeOf(stream).Error || error{ UnexpectedData, EndOfStream })!T { 26 | comptime validateTopLevelType(T); 27 | if (comptime requiresAllocationForDeserialize(T)) 28 | @compileError(@typeName(T) ++ " requires allocation to be deserialized. Use deserializeAlloc instead of deserialize!"); 29 | return deserializeInternal(stream, T, null) catch |err| switch (err) { 30 | error.OutOfMemory => unreachable, 31 | else => |e| return e, 32 | }; 33 | } 34 | 35 | /// Deserializes a value of type `T` from the `stream`. 36 | /// - `stream` is a instance of `std.io.Reader` 37 | /// - `T` is the type to deserialize 38 | /// - `allocator` is an allocator require to allocate slices and pointers. 39 | /// Result must be freed by using `free()`. 40 | pub fn deserializeAlloc( 41 | stream: anytype, 42 | comptime T: type, 43 | allocator: std.mem.Allocator, 44 | ) (@TypeOf(stream).Error || error{ UnexpectedData, OutOfMemory, EndOfStream })!T { 45 | comptime validateTopLevelType(T); 46 | return try deserializeInternal(stream, T, allocator); 47 | } 48 | 49 | /// Releases all memory allocated by `deserializeAlloc`. 50 | /// - `allocator` is the allocator passed to `deserializeAlloc`. 51 | /// - `T` is the type that was passed to `deserializeAlloc`. 52 | /// - `value` is the value that was returned by `deserializeAlloc`. 53 | pub fn free(allocator: std.mem.Allocator, comptime T: type, value: *T) void { 54 | recursiveFree(allocator, T, value); 55 | } 56 | 57 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// 58 | // Implementation: 59 | 60 | fn serializeRecursive(stream: anytype, comptime T: type, value: T) @TypeOf(stream).Error!void { 61 | switch (@typeInfo(T)) { 62 | // Primitive types: 63 | .void => {}, // no data 64 | .bool => try stream.writeByte(@intFromBool(value)), 65 | .float => switch (T) { 66 | f16 => try stream.writeInt(u16, @bitCast(value), .little), 67 | f32 => try stream.writeInt(u32, @bitCast(value), .little), 68 | f64 => try stream.writeInt(u64, @bitCast(value), .little), 69 | f80 => try stream.writeInt(u80, @bitCast(value), .little), 70 | f128 => try stream.writeInt(u128, @bitCast(value), .little), 71 | else => unreachable, 72 | }, 73 | 74 | .int => { 75 | if (T == usize) { 76 | try stream.writeInt(u64, value, .little); 77 | } else { 78 | try stream.writeInt(AlignedInt(T), value, .little); 79 | } 80 | }, 81 | .pointer => |ptr| { 82 | if (ptr.sentinel() != null) @compileError("Sentinels are not supported yet!"); 83 | switch (ptr.size) { 84 | .one => try serializeRecursive(stream, ptr.child, value.*), 85 | .slice => { 86 | try stream.writeInt(u64, value.len, .little); 87 | if (ptr.child == u8) { 88 | try stream.writeAll(value); 89 | } else { 90 | for (value) |item| { 91 | try serializeRecursive(stream, ptr.child, item); 92 | } 93 | } 94 | }, 95 | .c => unreachable, 96 | .many => unreachable, 97 | } 98 | }, 99 | .array => |arr| { 100 | if (arr.child == u8) { 101 | try stream.writeAll(&value); 102 | } else { 103 | for (value) |item| { 104 | try serializeRecursive(stream, arr.child, item); 105 | } 106 | } 107 | if (arr.sentinel() != null) @compileError("Sentinels are not supported yet!"); 108 | }, 109 | .@"struct" => |str| { 110 | // we can safely ignore the struct layout here as we will serialize the data by field order, 111 | // instead of memory representation 112 | 113 | inline for (str.fields) |fld| { 114 | try serializeRecursive(stream, fld.type, @field(value, fld.name)); 115 | } 116 | }, 117 | .optional => |opt| { 118 | if (value) |item| { 119 | try stream.writeInt(u8, 1, .little); 120 | try serializeRecursive(stream, opt.child, item); 121 | } else { 122 | try stream.writeInt(u8, 0, .little); 123 | } 124 | }, 125 | .error_union => |eu| { 126 | if (value) |item| { 127 | try stream.writeInt(u8, 1, .little); 128 | try serializeRecursive(stream, eu.payload, item); 129 | } else |item| { 130 | try stream.writeInt(u8, 0, .little); 131 | try serializeRecursive(stream, eu.error_set, item); 132 | } 133 | }, 134 | .error_set => { 135 | // Error unions are serialized by "index of sorted name", so we 136 | // hash all names in the right order 137 | const names = comptime getSortedErrorNames(T); 138 | 139 | const index = for (names, 0..) |name, i| { 140 | if (std.mem.eql(u8, name, @errorName(value))) 141 | break @as(u16, @intCast(i)); 142 | } else unreachable; 143 | 144 | try stream.writeInt(u16, index, .little); 145 | }, 146 | .@"enum" => |list| { 147 | const Tag = if (list.tag_type == usize) u64 else list.tag_type; 148 | try stream.writeInt(AlignedInt(Tag), @intFromEnum(value), .little); 149 | }, 150 | .@"union" => |un| { 151 | const Tag = un.tag_type orelse @compileError("Untagged unions are not supported!"); 152 | 153 | const active_tag = std.meta.activeTag(value); 154 | 155 | try serializeRecursive(stream, Tag, active_tag); 156 | 157 | inline for (std.meta.fields(T)) |fld| { 158 | if (@field(Tag, fld.name) == active_tag) { 159 | try serializeRecursive(stream, fld.type, @field(value, fld.name)); 160 | } 161 | } 162 | }, 163 | .vector => |vec| { 164 | const array: [vec.len]vec.child = value; 165 | try serializeRecursive(stream, @TypeOf(array), array); 166 | }, 167 | 168 | // Unsupported types: 169 | .noreturn, 170 | .type, 171 | .comptime_float, 172 | .comptime_int, 173 | .undefined, 174 | .null, 175 | .@"fn", 176 | .@"opaque", 177 | .frame, 178 | .@"anyframe", 179 | .enum_literal, 180 | => unreachable, 181 | } 182 | } 183 | 184 | fn deserializeInternal( 185 | stream: anytype, 186 | comptime T: type, 187 | allocator: ?std.mem.Allocator, 188 | ) (@TypeOf(stream).Error || error{ UnexpectedData, OutOfMemory, EndOfStream })!T { 189 | const type_hash = comptime computeTypeHash(T); 190 | 191 | var ref_hash: [type_hash.len]u8 = undefined; 192 | try stream.readNoEof(&ref_hash); 193 | if (!std.mem.eql(u8, type_hash[0..], ref_hash[0..])) 194 | return error.UnexpectedData; 195 | 196 | var result: T = undefined; 197 | try recursiveDeserialize(stream, T, allocator, &result); 198 | return result; 199 | } 200 | 201 | ///Determines the size of the next byte aligned integer type that can accommodate the same range of values as `T` 202 | fn AlignedInt(comptime T: type) type { 203 | return std.math.ByteAlignedInt(T); 204 | } 205 | 206 | fn recursiveDeserialize( 207 | stream: anytype, 208 | comptime T: type, 209 | allocator: ?std.mem.Allocator, 210 | target: *T, 211 | ) (@TypeOf(stream).Error || error{ UnexpectedData, OutOfMemory, EndOfStream })!void { 212 | switch (@typeInfo(T)) { 213 | // Primitive types: 214 | .void => target.* = {}, 215 | .bool => target.* = (try stream.readByte()) != 0, 216 | .float => target.* = @bitCast(switch (T) { 217 | f16 => try stream.readInt(u16, .little), 218 | f32 => try stream.readInt(u32, .little), 219 | f64 => try stream.readInt(u64, .little), 220 | f80 => try stream.readInt(u80, .little), 221 | f128 => try stream.readInt(u128, .little), 222 | else => unreachable, 223 | }), 224 | 225 | .int => target.* = if (T == usize) 226 | std.math.cast(usize, try stream.readInt(u64, .little)) orelse return error.UnexpectedData 227 | else 228 | @truncate(try stream.readInt(AlignedInt(T), .little)), 229 | 230 | .pointer => |ptr| { 231 | if (ptr.sentinel() != null) @compileError("Sentinels are not supported yet!"); 232 | switch (ptr.size) { 233 | .one => { 234 | const pointer = try allocator.?.create(ptr.child); 235 | errdefer allocator.?.destroy(pointer); 236 | 237 | try recursiveDeserialize(stream, ptr.child, allocator, pointer); 238 | 239 | target.* = pointer; 240 | }, 241 | .slice => { 242 | const length = std.math.cast(usize, try stream.readInt(u64, .little)) orelse return error.UnexpectedData; 243 | 244 | const slice = try allocator.?.alloc(ptr.child, length); 245 | errdefer allocator.?.free(slice); 246 | 247 | if (ptr.child == u8) { 248 | try stream.readNoEof(slice); 249 | } else { 250 | for (slice) |*item| { 251 | try recursiveDeserialize(stream, ptr.child, allocator, item); 252 | } 253 | } 254 | 255 | target.* = slice; 256 | }, 257 | .c => unreachable, 258 | .many => unreachable, 259 | } 260 | }, 261 | .array => |arr| { 262 | if (arr.child == u8) { 263 | try stream.readNoEof(target); 264 | } else { 265 | for (&target.*) |*item| { 266 | try recursiveDeserialize(stream, arr.child, allocator, item); 267 | } 268 | } 269 | }, 270 | .@"struct" => |str| { 271 | // we can safely ignore the struct layout here as we will serialize the data by field order, 272 | // instead of memory representation 273 | 274 | inline for (str.fields) |fld| { 275 | try recursiveDeserialize(stream, fld.type, allocator, &@field(target.*, fld.name)); 276 | } 277 | }, 278 | .optional => |opt| { 279 | const is_set = try stream.readInt(u8, .little); 280 | 281 | if (is_set != 0) { 282 | target.* = @as(opt.child, undefined); 283 | try recursiveDeserialize(stream, opt.child, allocator, &target.*.?); 284 | } else { 285 | target.* = null; 286 | } 287 | }, 288 | .error_union => |eu| { 289 | const is_value = try stream.readInt(u8, .little); 290 | if (is_value != 0) { 291 | var value: eu.payload = undefined; 292 | try recursiveDeserialize(stream, eu.payload, allocator, &value); 293 | target.* = value; 294 | } else { 295 | var err: eu.error_set = undefined; 296 | try recursiveDeserialize(stream, eu.error_set, allocator, &err); 297 | target.* = err; 298 | } 299 | }, 300 | .error_set => { 301 | // Error unions are serialized by "index of sorted name", so we 302 | // hash all names in the right order 303 | const names = comptime getSortedErrorNames(T); 304 | const index = try stream.readInt(u16, .little); 305 | 306 | switch (index) { 307 | inline 0...names.len - 1 => |idx| target.* = @field(T, names[idx]), 308 | else => return error.UnexpectedData, 309 | } 310 | }, 311 | .@"enum" => |list| { 312 | const Tag = if (list.tag_type == usize) u64 else list.tag_type; 313 | const tag_value: Tag = @truncate(try stream.readInt(AlignedInt(Tag), .little)); 314 | if (list.is_exhaustive) { 315 | target.* = std.meta.intToEnum(T, tag_value) catch return error.UnexpectedData; 316 | } else { 317 | target.* = @enumFromInt(tag_value); 318 | } 319 | }, 320 | .@"union" => |un| { 321 | const Tag = un.tag_type orelse @compileError("Untagged unions are not supported!"); 322 | 323 | var active_tag: Tag = undefined; 324 | try recursiveDeserialize(stream, Tag, allocator, &active_tag); 325 | 326 | inline for (std.meta.fields(T)) |fld| { 327 | if (@field(Tag, fld.name) == active_tag) { 328 | var union_value: fld.type = undefined; 329 | try recursiveDeserialize(stream, fld.type, allocator, &union_value); 330 | target.* = @unionInit(T, fld.name, union_value); 331 | return; 332 | } 333 | } 334 | 335 | return error.UnexpectedData; 336 | }, 337 | .vector => |vec| { 338 | var array: [vec.len]vec.child = undefined; 339 | try recursiveDeserialize(stream, @TypeOf(array), allocator, &array); 340 | target.* = array; 341 | }, 342 | 343 | // Unsupported types: 344 | .noreturn, 345 | .type, 346 | .comptime_float, 347 | .comptime_int, 348 | .undefined, 349 | .null, 350 | .@"fn", 351 | .@"opaque", 352 | .frame, 353 | .@"anyframe", 354 | .enum_literal, 355 | => unreachable, 356 | } 357 | } 358 | 359 | fn makeMutableSlice(comptime T: type, slice: []const T) []T { 360 | if (slice.len == 0) { 361 | var buf: [0]T = .{}; 362 | return &buf; 363 | } else { 364 | return @as([*]T, @constCast(slice.ptr))[0..slice.len]; 365 | } 366 | } 367 | 368 | fn recursiveFree(allocator: std.mem.Allocator, comptime T: type, value: *T) void { 369 | switch (@typeInfo(T)) { 370 | // Non-allocating primitives: 371 | .void, .bool, .float, .int, .error_set, .@"enum" => {}, 372 | 373 | // Composite types: 374 | .pointer => |ptr| { 375 | switch (ptr.size) { 376 | .one => { 377 | const mut_ptr = @constCast(value.*); 378 | recursiveFree(allocator, ptr.child, mut_ptr); 379 | allocator.destroy(mut_ptr); 380 | }, 381 | .slice => { 382 | const mut_slice = makeMutableSlice(ptr.child, value.*); 383 | for (mut_slice) |*item| { 384 | recursiveFree(allocator, ptr.child, item); 385 | } 386 | allocator.free(mut_slice); 387 | }, 388 | .c => unreachable, 389 | .many => unreachable, 390 | } 391 | }, 392 | .array => |arr| { 393 | for (&value.*) |*item| { 394 | recursiveFree(allocator, arr.child, item); 395 | } 396 | }, 397 | .@"struct" => |str| { 398 | // we can safely ignore the struct layout here as we will serialize the data by field order, 399 | // instead of memory representation 400 | 401 | inline for (str.fields) |fld| { 402 | recursiveFree(allocator, fld.type, &@field(value.*, fld.name)); 403 | } 404 | }, 405 | .optional => |opt| { 406 | if (value.*) |*item| { 407 | recursiveFree(allocator, opt.child, item); 408 | } 409 | }, 410 | .error_union => |eu| { 411 | if (value.*) |*item| { 412 | recursiveFree(allocator, eu.payload, item); 413 | } else |_| { 414 | // errors aren't meant to be freed 415 | } 416 | }, 417 | .@"union" => |un| { 418 | const Tag = un.tag_type orelse @compileError("Untagged unions are not supported!"); 419 | 420 | const active_tag: Tag = value.*; 421 | 422 | inline for (std.meta.fields(T)) |fld| { 423 | if (@field(Tag, fld.name) == active_tag) { 424 | recursiveFree(allocator, fld.type, &@field(value.*, fld.name)); 425 | return; 426 | } 427 | } 428 | }, 429 | .vector => |vec| { 430 | var array: [vec.len]vec.child = value.*; 431 | for (&array) |*item| { 432 | recursiveFree(allocator, vec.child, item); 433 | } 434 | }, 435 | 436 | // Unsupported types: 437 | .noreturn, 438 | .type, 439 | .comptime_float, 440 | .comptime_int, 441 | .undefined, 442 | .null, 443 | .@"fn", 444 | .@"opaque", 445 | .frame, 446 | .@"anyframe", 447 | .enum_literal, 448 | => unreachable, 449 | } 450 | } 451 | 452 | /// Returns `true` if `T` requires allocation to be deserialized. 453 | fn requiresAllocationForDeserialize(comptime T: type) bool { 454 | switch (@typeInfo(T)) { 455 | .pointer => return true, 456 | .@"struct", .@"union" => { 457 | inline for (comptime std.meta.fields(T)) |fld| { 458 | if (requiresAllocationForDeserialize(fld.type)) { 459 | return true; 460 | } 461 | } 462 | return false; 463 | }, 464 | .error_union => |eu| return requiresAllocationForDeserialize(eu.payload), 465 | else => return false, 466 | } 467 | } 468 | 469 | const TypeHashFn = std.hash.Fnv1a_64; 470 | 471 | fn intToLittleEndianBytes(val: anytype) [@sizeOf(@TypeOf(val))]u8 { 472 | const T = @TypeOf(val); 473 | var res: [@sizeOf(T)]u8 = undefined; 474 | std.mem.writeInt(AlignedInt(T), &res, val, .little); 475 | return res; 476 | } 477 | 478 | /// Computes a unique type hash from `T` to identify deserializing invalid data. 479 | /// Incorporates field order and field type, but not field names, so only checks 480 | /// for structural equivalence. Compile errors on unsupported or comptime types. 481 | fn computeTypeHash(comptime T: type) [8]u8 { 482 | var hasher = TypeHashFn.init(); 483 | 484 | computeTypeHashInternal(&hasher, T); 485 | 486 | return intToLittleEndianBytes(hasher.final()); 487 | } 488 | 489 | fn getSortedErrorNames(comptime T: type) []const []const u8 { 490 | comptime { 491 | const error_set = @typeInfo(T).error_set orelse @compileError("Cannot serialize anyerror"); 492 | 493 | var sorted_names: [error_set.len][]const u8 = undefined; 494 | for (error_set, 0..) |err, i| { 495 | sorted_names[i] = err.name; 496 | } 497 | 498 | std.mem.sortUnstable([]const u8, &sorted_names, {}, struct { 499 | fn order(ctx: void, lhs: []const u8, rhs: []const u8) bool { 500 | _ = ctx; 501 | return (std.mem.order(u8, lhs, rhs) == .lt); 502 | } 503 | }.order); 504 | return &sorted_names; 505 | } 506 | } 507 | 508 | fn getSortedEnumNames(comptime T: type) []const []const u8 { 509 | comptime { 510 | const type_info = @typeInfo(T).@"enum"; 511 | 512 | var sorted_names: [type_info.fields.len][]const u8 = undefined; 513 | for (type_info.fields, 0..) |err, i| { 514 | sorted_names[i] = err.name; 515 | } 516 | 517 | std.mem.sortUnstable([]const u8, &sorted_names, {}, struct { 518 | fn order(ctx: void, lhs: []const u8, rhs: []const u8) bool { 519 | _ = ctx; 520 | return (std.mem.order(u8, lhs, rhs) == .lt); 521 | } 522 | }.order); 523 | return &sorted_names; 524 | } 525 | } 526 | 527 | fn computeTypeHashInternal(hasher: *TypeHashFn, comptime T: type) void { 528 | @setEvalBranchQuota(10_000); 529 | switch (@typeInfo(T)) { 530 | // Primitive types: 531 | .void, 532 | .bool, 533 | .float, 534 | => hasher.update(@typeName(T)), 535 | 536 | .int => { 537 | if (T == usize) { 538 | // special case: usize can differ between platforms, this 539 | // format uses u64 internally. 540 | hasher.update(@typeName(u64)); 541 | } else { 542 | hasher.update(@typeName(T)); 543 | } 544 | }, 545 | .pointer => |ptr| { 546 | if (ptr.is_volatile) @compileError("Serializing volatile pointers is most likely a mistake."); 547 | if (ptr.sentinel() != null) @compileError("Sentinels are not supported yet!"); 548 | switch (ptr.size) { 549 | .one => { 550 | hasher.update("pointer"); 551 | computeTypeHashInternal(hasher, ptr.child); 552 | }, 553 | .slice => { 554 | hasher.update("slice"); 555 | computeTypeHashInternal(hasher, ptr.child); 556 | }, 557 | .c => @compileError("C-pointers are not supported"), 558 | .many => @compileError("Many-pointers are not supported"), 559 | } 560 | }, 561 | .array => |arr| { 562 | if (arr.sentinel() != null) @compileError("Sentinels are not supported yet!"); 563 | hasher.update(&intToLittleEndianBytes(@as(u64, arr.len))); 564 | computeTypeHashInternal(hasher, arr.child); 565 | }, 566 | .@"struct" => |str| { 567 | // we can safely ignore the struct layout here as we will serialize the data by field order, 568 | // instead of memory representation 569 | 570 | // add some generic marker to the hash so emtpy structs get 571 | // added as information 572 | hasher.update("struct"); 573 | 574 | for (str.fields) |fld| { 575 | if (fld.is_comptime) @compileError("comptime fields are not supported."); 576 | computeTypeHashInternal(hasher, fld.type); 577 | } 578 | }, 579 | .optional => |opt| { 580 | hasher.update("optional"); 581 | computeTypeHashInternal(hasher, opt.child); 582 | }, 583 | .error_union => |eu| { 584 | hasher.update("error union"); 585 | computeTypeHashInternal(hasher, eu.error_set); 586 | computeTypeHashInternal(hasher, eu.payload); 587 | }, 588 | .error_set => { 589 | // Error unions are serialized by "index of sorted name", so we 590 | // hash all names in the right order 591 | 592 | hasher.update("error set"); 593 | const names = comptime getSortedErrorNames(T); 594 | for (names) |name| { 595 | hasher.update(name); 596 | } 597 | }, 598 | .@"enum" => |list| { 599 | const Tag = if (list.tag_type == usize) 600 | u64 601 | else if (list.tag_type == isize) 602 | i64 603 | else 604 | list.tag_type; 605 | if (list.is_exhaustive) { 606 | // Exhaustive enums only allow certain values, so we 607 | // tag them via the value type 608 | hasher.update("enum.exhaustive"); 609 | computeTypeHashInternal(hasher, Tag); 610 | const names = getSortedEnumNames(T); 611 | inline for (names) |name| { 612 | hasher.update(name); 613 | hasher.update(&intToLittleEndianBytes(@as(Tag, @intFromEnum(@field(T, name))))); 614 | } 615 | } else { 616 | // Non-exhaustive enums are basically integers. Treat them as such. 617 | hasher.update("enum.non-exhaustive"); 618 | computeTypeHashInternal(hasher, Tag); 619 | } 620 | }, 621 | .@"union" => |un| { 622 | const tag = un.tag_type orelse @compileError("Untagged unions are not supported!"); 623 | hasher.update("union"); 624 | computeTypeHashInternal(hasher, tag); 625 | for (un.fields) |fld| { 626 | computeTypeHashInternal(hasher, fld.type); 627 | } 628 | }, 629 | .vector => |vec| { 630 | hasher.update("vector"); 631 | hasher.update(&intToLittleEndianBytes(@as(u64, vec.len))); 632 | computeTypeHashInternal(hasher, vec.child); 633 | }, 634 | 635 | // Unsupported types: 636 | .noreturn, 637 | .type, 638 | .comptime_float, 639 | .comptime_int, 640 | .undefined, 641 | .null, 642 | .@"fn", 643 | .@"opaque", 644 | .frame, 645 | .@"anyframe", 646 | .enum_literal, 647 | => @compileError("Unsupported type " ++ @typeName(T)), 648 | } 649 | } 650 | 651 | fn validateTopLevelType(comptime T: type) void { 652 | switch (@typeInfo(T)) { 653 | 654 | // Unsupported top level types: 655 | .error_set, 656 | .error_union, 657 | => @compileError("Unsupported top level type " ++ @typeName(T) ++ ". Wrap into struct to serialize these."), 658 | 659 | else => {}, 660 | } 661 | } 662 | 663 | fn testSameHash(comptime T1: type, comptime T2: type) void { 664 | const hash_1 = comptime computeTypeHash(T1); 665 | const hash_2 = comptime computeTypeHash(T2); 666 | if (comptime !std.mem.eql(u8, hash_1[0..], hash_2[0..])) 667 | @compileError("The computed hash for " ++ @typeName(T1) ++ " and " ++ @typeName(T2) ++ " does not match."); 668 | } 669 | 670 | test "type hasher basics" { 671 | testSameHash(void, void); 672 | testSameHash(bool, bool); 673 | testSameHash(u1, u1); 674 | testSameHash(u32, u32); 675 | testSameHash(f32, f32); 676 | testSameHash(f64, f64); 677 | testSameHash(@Vector(4, u32), @Vector(4, u32)); 678 | testSameHash(usize, u64); 679 | testSameHash([]const u8, []const u8); 680 | testSameHash([]const u8, []u8); 681 | testSameHash([]const u8, []u8); 682 | testSameHash(?*struct { a: f32, b: u16 }, ?*const struct { hello: f32, lol: u16 }); 683 | testSameHash(enum { a, b, c }, enum { a, b, c }); 684 | testSameHash(enum(u8) { a, b, c, _ }, enum(u8) { c, b, a, _ }); 685 | 686 | testSameHash(enum(u8) { a, b, c }, enum(u8) { a, b, c }); 687 | testSameHash(enum(u8) { a = 1, b = 6, c = 9 }, enum(u8) { a = 1, b = 6, c = 9 }); 688 | 689 | testSameHash(enum(usize) { a, b, c }, enum(u64) { a, b, c }); 690 | testSameHash(enum(isize) { a, b, c }, enum(i64) { a, b, c }); 691 | testSameHash([5]@Vector(4, u32), [5]@Vector(4, u32)); 692 | 693 | testSameHash(union(enum) { a: u32, b: f32 }, union(enum) { a: u32, b: f32 }); 694 | 695 | testSameHash(error{ Foo, Bar }, error{ Foo, Bar }); 696 | testSameHash(error{ Foo, Bar }, error{ Bar, Foo }); 697 | testSameHash(error{ Foo, Bar }!void, error{ Bar, Foo }!void); 698 | } 699 | 700 | fn testSerialize(comptime T: type, value: T) !void { 701 | var data = std.ArrayList(u8).init(std.testing.allocator); 702 | defer data.deinit(); 703 | 704 | try serialize(data.writer(), T, value); 705 | } 706 | 707 | const enable_failing_test = false; 708 | 709 | test "serialize basics" { 710 | try testSerialize(void, {}); 711 | try testSerialize(bool, false); 712 | try testSerialize(bool, true); 713 | try testSerialize(u1, 0); 714 | try testSerialize(u1, 1); 715 | try testSerialize(u8, 0xFF); 716 | try testSerialize(u32, 0xDEADBEEF); 717 | try testSerialize(usize, 0xDEADBEEF); 718 | 719 | try testSerialize(f16, std.math.pi); 720 | try testSerialize(f32, std.math.pi); 721 | try testSerialize(f64, std.math.pi); 722 | try testSerialize(f80, std.math.pi); 723 | try testSerialize(f128, std.math.pi); 724 | 725 | try testSerialize([3]u8, "hi!".*); 726 | try testSerialize([]const u8, "Hello, World!"); 727 | try testSerialize(*const [3]u8, "foo"); 728 | 729 | try testSerialize(enum { a, b, c }, .a); 730 | try testSerialize(enum { a, b, c }, .b); 731 | try testSerialize(enum { a, b, c }, .c); 732 | 733 | try testSerialize(enum(u8) { a, b, c }, .a); 734 | try testSerialize(enum(u8) { a, b, c }, .b); 735 | try testSerialize(enum(u8) { a, b, c }, .c); 736 | 737 | try testSerialize(enum(isize) { a, b, c }, .a); 738 | try testSerialize(enum(isize) { a, b, c }, .b); 739 | try testSerialize(enum(isize) { a, b, c }, .c); 740 | 741 | try testSerialize(enum(usize) { a, b, c }, .a); 742 | try testSerialize(enum(usize) { a, b, c }, .b); 743 | try testSerialize(enum(usize) { a, b, c }, .c); 744 | 745 | const TestEnum = enum(u8) { a, b, c, _ }; 746 | try testSerialize(TestEnum, .a); 747 | try testSerialize(TestEnum, .b); 748 | try testSerialize(TestEnum, .c); 749 | try testSerialize(TestEnum, @as(TestEnum, @enumFromInt(0xB1))); 750 | 751 | if (enable_failing_test) { 752 | try testSerialize(struct { val: error{ Foo, Bar } }, .{ .val = error.Foo }); 753 | try testSerialize(struct { val: error{ Bar, Foo } }, .{ .val = error.Bar }); 754 | try testSerialize(struct { val: error{ Bar, Foo }!u32 }, .{ .val = error.Bar }); 755 | try testSerialize(struct { val: error{ Bar, Foo }!u32 }, .{ .val = 0xFF }); 756 | } 757 | try testSerialize(union(enum) { a: f32, b: u32 }, .{ .a = 1.5 }); 758 | try testSerialize(union(enum) { a: f32, b: u32 }, .{ .b = 2.0 }); 759 | 760 | try testSerialize(?u32, null); 761 | try testSerialize(?u32, 143); 762 | } 763 | 764 | fn testSerDesAlloc(comptime T: type, value: T) !void { 765 | var data: std.ArrayList(u8) = .init(std.testing.allocator); 766 | defer data.deinit(); 767 | 768 | try serialize(data.writer(), T, value); 769 | 770 | var stream = std.io.fixedBufferStream(data.items); 771 | 772 | var deserialized = try deserializeAlloc(stream.reader(), T, std.testing.allocator); 773 | defer free(std.testing.allocator, T, &deserialized); 774 | 775 | try std.testing.expectEqual(value, deserialized); 776 | } 777 | 778 | fn testSerDesPtrContentEquality(comptime T: type, value: T) !void { 779 | var data = std.ArrayList(u8).init(std.testing.allocator); 780 | defer data.deinit(); 781 | 782 | try serialize(data.writer(), T, value); 783 | 784 | var stream = std.io.fixedBufferStream(data.items); 785 | 786 | var deserialized = try deserializeAlloc(stream.reader(), T, std.testing.allocator); 787 | defer free(std.testing.allocator, T, &deserialized); 788 | 789 | try std.testing.expectEqual(value.*, deserialized.*); 790 | } 791 | 792 | fn testSerDesSliceContentEquality(comptime T: type, value: T) !void { 793 | var data = std.ArrayList(u8).init(std.testing.allocator); 794 | defer data.deinit(); 795 | 796 | try serialize(data.writer(), T, value); 797 | 798 | var stream = std.io.fixedBufferStream(data.items); 799 | 800 | var deserialized = try deserializeAlloc(stream.reader(), T, std.testing.allocator); 801 | defer free(std.testing.allocator, T, &deserialized); 802 | 803 | try std.testing.expectEqualSlices(std.meta.Child(T), value, deserialized); 804 | } 805 | 806 | test "ser/des" { 807 | try testSerDesAlloc(void, {}); 808 | try testSerDesAlloc(bool, false); 809 | try testSerDesAlloc(bool, true); 810 | try testSerDesAlloc(u1, 0); 811 | try testSerDesAlloc(u1, 1); 812 | try testSerDesAlloc(u8, 0xFF); 813 | try testSerDesAlloc(u32, 0xDEADBEEF); 814 | try testSerDesAlloc(usize, 0xDEADBEEF); 815 | 816 | try testSerDesAlloc(f16, std.math.pi); 817 | try testSerDesAlloc(f32, std.math.pi); 818 | try testSerDesAlloc(f64, std.math.pi); 819 | try testSerDesAlloc(f80, std.math.pi); 820 | try testSerDesAlloc(f128, std.math.pi); 821 | 822 | try testSerDesAlloc([3]u8, "hi!".*); 823 | try testSerDesSliceContentEquality([]const u8, "Hello, World!"); 824 | try testSerDesPtrContentEquality(*const [3]u8, "foo"); 825 | 826 | try testSerDesAlloc(enum { a, b, c }, .a); 827 | try testSerDesAlloc(enum { a, b, c }, .b); 828 | try testSerDesAlloc(enum { a, b, c }, .c); 829 | 830 | try testSerDesAlloc(enum(u8) { a, b, c }, .a); 831 | try testSerDesAlloc(enum(u8) { a, b, c }, .b); 832 | try testSerDesAlloc(enum(u8) { a, b, c }, .c); 833 | 834 | try testSerDesAlloc(enum(usize) { a, b, c }, .a); 835 | try testSerDesAlloc(enum(usize) { a, b, c }, .b); 836 | try testSerDesAlloc(enum(usize) { a, b, c }, .c); 837 | 838 | try testSerDesAlloc(enum(isize) { a, b, c }, .a); 839 | try testSerDesAlloc(enum(isize) { a, b, c }, .b); 840 | try testSerDesAlloc(enum(isize) { a, b, c }, .c); 841 | 842 | const TestEnum = enum(u8) { a, b, c, _ }; 843 | try testSerDesAlloc(TestEnum, .a); 844 | try testSerDesAlloc(TestEnum, .b); 845 | try testSerDesAlloc(TestEnum, .c); 846 | try testSerDesAlloc(TestEnum, @as(TestEnum, @enumFromInt(0xB1))); 847 | 848 | if (enable_failing_test) { 849 | try testSerDesAlloc(struct { val: error{ Foo, Bar } }, .{ .val = error.Foo }); 850 | try testSerDesAlloc(struct { val: error{ Bar, Foo } }, .{ .val = error.Bar }); 851 | try testSerDesAlloc(struct { val: error{ Bar, Foo }!u32 }, .{ .val = error.Bar }); 852 | try testSerDesAlloc(struct { val: error{ Bar, Foo }!u32 }, .{ .val = 0xFF }); 853 | } 854 | 855 | try testSerDesAlloc(union(enum) { a: f32, b: u32 }, .{ .a = 1.5 }); 856 | try testSerDesAlloc(union(enum) { a: f32, b: u32 }, .{ .b = 2.0 }); 857 | 858 | try testSerDesAlloc(?u32, null); 859 | try testSerDesAlloc(?u32, 143); 860 | } 861 | --------------------------------------------------------------------------------