├── LICENSE ├── README.md ├── build.zig ├── build.zig.zon └── src ├── lib.zig └── test.zig /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (Expat) 2 | 3 | Copyright (c) contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | xitdb is an immutable database written in Zig. 2 | 3 | * Each transaction efficiently creates a new "copy" of the database, and past copies can still be read from. 4 | * It supports writing to a file as well as purely in-memory use. 5 | * No query engine of any kind. You just write data structures (primarily an `ArrayList` and `HashMap`) that can be nested arbitrarily. 6 | * No dependencies besides the Zig standard library (currently requires Zig 0.14.0). 7 | * There is also a [Java port](https://github.com/radarroark/xitdb-java) of this library. 8 | 9 | This database was originally made for the [xit version control system](https://github.com/radarroark/xit), but I bet it has a lot of potential for other projects. The combination of being immutable and having an API similar to in-memory data structures is pretty powerful. Consider using it instead of SQLite for your Zig projects: it's simpler, it's pure Zig, and it creates no impedence mismatch with your program the way SQL databases do. 10 | 11 | Usually, you want to use a top-level `ArrayList` like in the example below, because that allows you to store a reference to each copy of the database (which I call a "moment"). This is how it supports transactions, despite not having any rollback journal or write-ahead log. It's an append-only database, so the data you are writing is invisible to any reader until the very last step, when the top-level list's header is updated. 12 | 13 | You can also use a top-level `HashMap`, which is useful for ephemeral databases where immutability or transaction safety isn't necessary. Since xitdb supports in-memory databases, you could use it as an over-the-wire serialization format. Much like "Cap'n Proto", xitdb has no encoding/decoding step: you just give the buffer to xitdb and it can immediately read from it. 14 | 15 | The `HashMap` and `ArrayList` are based on the hash array mapped trie from Phil Bagwell. There is also a `LinkedArrayList`, which is based on the RRB tree, also from Phil Bagwell. It is similar to an `ArrayList`, except it can be efficiently sliced and concatenated. If you need a `HashMap` that maintains a count of its contents, there is a `CountedHashMap`. Lastly, there is a `HashSet` and `CountedHashSet` which work like a `HashMap` that only sets its keys. Check out the example below and the tests. 16 | 17 | ```zig 18 | // create db file 19 | const file = try std.fs.cwd().createFile("main.db", .{ .read = true }); 20 | defer file.close(); 21 | 22 | // init the db 23 | const DB = xitdb.Database(.file, HashInt); 24 | var db = try DB.init(allocator, .{ .file = file }); 25 | 26 | // to get the benefits of immutability, the top-level data structure 27 | // must be an ArrayList, so each transaction is stored as an item in it 28 | const history = try DB.ArrayList(.read_write).init(db.rootCursor()); 29 | 30 | // this is how a transaction is executed. we call history.appendContext, 31 | // providing it with the most recent copy of the db and a context 32 | // object. the context object has a method that will run before the 33 | // transaction has completed. this method is where we can write 34 | // changes to the db. if any error happens in it, the transaction 35 | // will not complete, the data added to the file will be truncated, 36 | // and the db will be unaffected. 37 | // 38 | // after this transaction, the db will look like this if represented 39 | // as JSON (in reality the format is binary): 40 | // 41 | // {"foo": "foo", 42 | // "bar": "bar", 43 | // "fruits": ["apple", "pear", "grape"], 44 | // "people": [ 45 | // {"name": "Alice", "age": 25}, 46 | // {"name": "Bob", "age": 42} 47 | // ]} 48 | const Ctx = struct { 49 | pub fn run(_: @This(), cursor: *DB.Cursor(.read_write)) !void { 50 | const moment = try DB.HashMap(.read_write).init(cursor.*); 51 | 52 | try moment.put(hashInt("foo"), .{ .bytes = "foo" }); 53 | try moment.put(hashInt("bar"), .{ .bytes = "bar" }); 54 | 55 | const fruits_cursor = try moment.putCursor(hashInt("fruits")); 56 | const fruits = try DB.ArrayList(.read_write).init(fruits_cursor); 57 | try fruits.append(.{ .bytes = "apple" }); 58 | try fruits.append(.{ .bytes = "pear" }); 59 | try fruits.append(.{ .bytes = "grape" }); 60 | 61 | const people_cursor = try moment.putCursor(hashInt("people")); 62 | const people = try DB.ArrayList(.read_write).init(people_cursor); 63 | 64 | const alice_cursor = try people.appendCursor(); 65 | const alice = try DB.HashMap(.read_write).init(alice_cursor); 66 | try alice.put(hashInt("name"), .{ .bytes = "Alice" }); 67 | try alice.put(hashInt("age"), .{ .uint = 25 }); 68 | 69 | const bob_cursor = try people.appendCursor(); 70 | const bob = try DB.HashMap(.read_write).init(bob_cursor); 71 | try bob.put(hashInt("name"), .{ .bytes = "Bob" }); 72 | try bob.put(hashInt("age"), .{ .uint = 42 }); 73 | } 74 | }; 75 | try history.appendContext(.{ .slot = try history.getSlot(-1) }, Ctx{}); 76 | 77 | // get the most recent copy of the database, like a moment 78 | // in time. the -1 index will return the last index in the list. 79 | const moment_cursor = (try history.getCursor(-1)).?; 80 | const moment = try DB.HashMap(.read_only).init(moment_cursor); 81 | 82 | // we can read the value of "foo" from the map by getting 83 | // the cursor to "foo" and then calling readBytesAlloc on it 84 | const foo_cursor = (try moment.getCursor(hashInt("foo"))).?; 85 | const foo_value = try foo_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 86 | defer allocator.free(foo_value); 87 | try std.testing.expectEqualStrings("foo", foo_value); 88 | 89 | // to get the "fruits" list, we get the cursor to it and 90 | // then pass it to the ArrayList init method 91 | const fruits_cursor = (try moment.getCursor(hashInt("fruits"))).?; 92 | const fruits = try DB.ArrayList(.read_only).init(fruits_cursor); 93 | 94 | // now we can get the first item from the fruits list and read it 95 | const apple_cursor = (try fruits.getCursor(0)).?; 96 | const apple_value = try apple_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 97 | defer allocator.free(apple_value); 98 | try std.testing.expectEqualStrings("apple", apple_value); 99 | ``` 100 | 101 | It is possible to read the database from multiple threads without locks, even while writes are happening. This is a big benefit of immutable databases. However, each thread needs to use its own file handle and Database object. Keep in mind that writes still need to come from a single thread. 102 | 103 | The `.file` database currently does not do any in-memory buffering, so the write performance won't be optimal. I plan on implementing that eventually. 104 | -------------------------------------------------------------------------------- /build.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | 3 | pub fn build(b: *std.Build) void { 4 | const target = b.standardTargetOptions(.{}); 5 | const optimize = b.standardOptimizeOption(.{}); 6 | 7 | const xitdb = b.addModule("xitdb", .{ 8 | .root_source_file = b.path("src/lib.zig"), 9 | }); 10 | 11 | const unit_tests = b.addTest(.{ 12 | .root_source_file = b.path("src/test.zig"), 13 | .target = target, 14 | .optimize = optimize, 15 | }); 16 | unit_tests.root_module.addImport("xitdb", xitdb); 17 | 18 | const run_unit_tests = b.addRunArtifact(unit_tests); 19 | const test_step = b.step("test", "Run library tests"); 20 | test_step.dependOn(&run_unit_tests.step); 21 | } 22 | -------------------------------------------------------------------------------- /build.zig.zon: -------------------------------------------------------------------------------- 1 | .{ 2 | .name = .xitdb, 3 | .version = "0.0.0", 4 | .dependencies = .{}, 5 | .paths = .{ 6 | "build.zig", 7 | "build.zig.zon", 8 | "src", 9 | "LICENSE", 10 | }, 11 | .fingerprint = 0xb87b8ac1c6644110, 12 | } 13 | -------------------------------------------------------------------------------- /src/test.zig: -------------------------------------------------------------------------------- 1 | const std = @import("std"); 2 | const xitdb = @import("xitdb"); 3 | 4 | const HashInt = u160; 5 | const MAX_READ_BYTES = 1024; 6 | 7 | test "high level api" { 8 | const allocator = std.testing.allocator; 9 | 10 | var buffer = std.ArrayList(u8).init(allocator); 11 | defer buffer.deinit(); 12 | try testHighLevelApi(allocator, .memory, .{ .buffer = &buffer, .max_size = 50_000 }); 13 | 14 | if (std.fs.cwd().openFile("main.db", .{})) |file| { 15 | file.close(); 16 | try std.fs.cwd().deleteFile("main.db"); 17 | } else |_| {} 18 | 19 | const file = try std.fs.cwd().createFile("main.db", .{ .read = true }); 20 | defer { 21 | file.close(); 22 | std.fs.cwd().deleteFile("main.db") catch {}; 23 | } 24 | try testHighLevelApi(allocator, .file, .{ .file = file }); 25 | } 26 | 27 | test "low level api" { 28 | const allocator = std.testing.allocator; 29 | 30 | var buffer = std.ArrayList(u8).init(allocator); 31 | defer buffer.deinit(); 32 | try testLowLevelApi(allocator, .memory, .{ .buffer = &buffer, .max_size = 50_000_000 }); 33 | 34 | if (std.fs.cwd().openFile("main.db", .{})) |file| { 35 | file.close(); 36 | try std.fs.cwd().deleteFile("main.db"); 37 | } else |_| {} 38 | 39 | const file = try std.fs.cwd().createFile("main.db", .{ .read = true }); 40 | defer { 41 | file.close(); 42 | std.fs.cwd().deleteFile("main.db") catch {}; 43 | } 44 | try testLowLevelApi(allocator, .file, .{ .file = file }); 45 | } 46 | 47 | test "not using arraylist at the top level" { 48 | // normally an arraylist makes the most sense at the top level, 49 | // but this test just ensures we can use other data structures 50 | // at the top level. in theory a top-level hash map might make 51 | // sense if we're using xitdb as a format to send data over a 52 | // network. in that case, immutability isn't important because 53 | // the data is just created and immediately sent over the wire. 54 | 55 | const allocator = std.testing.allocator; 56 | 57 | // hash map 58 | { 59 | var buffer = std.ArrayList(u8).init(allocator); 60 | defer buffer.deinit(); 61 | 62 | const DB = xitdb.Database(.memory, HashInt); 63 | var db = try DB.init(.{ .buffer = &buffer, .max_size = 50000 }); 64 | 65 | const map = try DB.HashMap(.read_write).init(db.rootCursor()); 66 | try map.put(hashInt("foo"), .{ .bytes = "foo" }); 67 | try map.put(hashInt("bar"), .{ .bytes = "bar" }); 68 | 69 | // init inner map 70 | { 71 | const inner_map_cursor = try map.putCursor(hashInt("inner-map")); 72 | _ = try DB.HashMap(.read_write).init(inner_map_cursor); 73 | } 74 | 75 | // re-init inner map 76 | // since the top-level type isn't an arraylist, there is no concept of 77 | // a transaction, so this does not perform a copy of the root node 78 | { 79 | const inner_map_cursor = try map.putCursor(hashInt("inner-map")); 80 | _ = try DB.HashMap(.read_write).init(inner_map_cursor); 81 | } 82 | } 83 | 84 | // linked array list is not currently allowed at the top level 85 | { 86 | var buffer = std.ArrayList(u8).init(allocator); 87 | defer buffer.deinit(); 88 | 89 | const DB = xitdb.Database(.memory, HashInt); 90 | var db = try DB.init(.{ .buffer = &buffer, .max_size = 50000 }); 91 | 92 | try std.testing.expectError(error.InvalidTopLevelType, DB.LinkedArrayList(.read_write).init(db.rootCursor())); 93 | } 94 | } 95 | 96 | test "low level memory operations" { 97 | const allocator = std.testing.allocator; 98 | 99 | var buffer = std.ArrayList(u8).init(allocator); 100 | defer buffer.deinit(); 101 | var db = try xitdb.Database(.memory, HashInt).init(.{ .buffer = &buffer, .max_size = 10000 }); 102 | 103 | var writer = db.core.writer(); 104 | try db.core.seekTo(0); 105 | try writer.writeAll("Hello"); 106 | try std.testing.expectEqualStrings("Hello", db.core.buffer.items[0..5]); 107 | try writer.writeInt(u64, 42, .big); 108 | var bytes = [_]u8{0} ** (@bitSizeOf(u64) / 8); 109 | std.mem.writeInt(u64, &bytes, 42, .big); 110 | const hello = try std.fmt.allocPrint(allocator, "Hello{s}", .{bytes}); 111 | defer allocator.free(hello); 112 | try std.testing.expectEqualStrings(hello, db.core.buffer.items[0..13]); 113 | 114 | var reader = db.core.reader(); 115 | try db.core.seekTo(0); 116 | var block = [_]u8{0} ** 5; 117 | try reader.readNoEof(&block); 118 | try std.testing.expectEqualStrings("Hello", &block); 119 | try std.testing.expectEqual(42, reader.readInt(u64, .big)); 120 | } 121 | 122 | test "validate tag" { 123 | const Slot = packed struct { 124 | value: u64, 125 | tag: u7, 126 | flag: u1, 127 | }; 128 | const invalid: xitdb.Slot = @bitCast(Slot{ .value = 0, .tag = 127, .flag = 0 }); 129 | try std.testing.expectEqual(error.InvalidEnumTag, invalid.tag.validate()); 130 | } 131 | 132 | fn hashInt(buffer: []const u8) HashInt { 133 | var hash = [_]u8{0} ** (@bitSizeOf(HashInt) / 8); 134 | var h = std.crypto.hash.Sha1.init(.{}); 135 | h.update(buffer); 136 | h.final(&hash); 137 | return std.mem.readInt(HashInt, &hash, .big); 138 | } 139 | 140 | fn testHighLevelApi(allocator: std.mem.Allocator, comptime db_kind: xitdb.DatabaseKind, init_opts: xitdb.Database(db_kind, HashInt).InitOpts) !void { 141 | // init the db 142 | const DB = xitdb.Database(db_kind, HashInt); 143 | var db = try DB.init(init_opts); 144 | 145 | { 146 | // to get the benefits of immutability, the top-level data structure 147 | // must be an ArrayList, so each transaction is stored as an item in it 148 | const history = try DB.ArrayList(.read_write).init(db.rootCursor()); 149 | 150 | // this is how a transaction is executed. we call history.appendContext, 151 | // providing it with the most recent copy of the db and a context 152 | // object. the context object has a method that will run before the 153 | // transaction has completed. this method is where we can write 154 | // changes to the db. if any error happens in it, the transaction 155 | // will not complete and the db will be unaffected. 156 | const Ctx = struct { 157 | pub fn run(_: @This(), cursor: *DB.Cursor(.read_write)) !void { 158 | const moment = try DB.HashMap(.read_write).init(cursor.*); 159 | 160 | try moment.put(hashInt("foo"), .{ .bytes = "foo" }); 161 | try moment.put(hashInt("bar"), .{ .bytes = "bar" }); 162 | 163 | const fruits_cursor = try moment.putCursor(hashInt("fruits")); 164 | const fruits = try DB.ArrayList(.read_write).init(fruits_cursor); 165 | try fruits.append(.{ .bytes = "apple" }); 166 | try fruits.append(.{ .bytes = "pear" }); 167 | try fruits.append(.{ .bytes = "grape" }); 168 | 169 | const people_cursor = try moment.putCursor(hashInt("people")); 170 | const people = try DB.ArrayList(.read_write).init(people_cursor); 171 | 172 | const alice_cursor = try people.appendCursor(); 173 | const alice = try DB.HashMap(.read_write).init(alice_cursor); 174 | try alice.put(hashInt("name"), .{ .bytes = "Alice" }); 175 | try alice.put(hashInt("age"), .{ .uint = 25 }); 176 | 177 | const bob_cursor = try people.appendCursor(); 178 | const bob = try DB.HashMap(.read_write).init(bob_cursor); 179 | try bob.put(hashInt("name"), .{ .bytes = "Bob" }); 180 | try bob.put(hashInt("age"), .{ .uint = 42 }); 181 | 182 | const todos_cursor = try moment.putCursor(hashInt("todos")); 183 | const todos = try DB.LinkedArrayList(.read_write).init(todos_cursor); 184 | try todos.append(.{ .bytes = "Pay the bills" }); 185 | try todos.append(.{ .bytes = "Get an oil change" }); 186 | try todos.insert(1, .{ .bytes = "Wash the car" }); 187 | 188 | // make sure `insertCursor` works as well 189 | const todo_cursor = try todos.insertCursor(1); 190 | _ = try DB.HashMap(.read_write).init(todo_cursor); 191 | try todos.remove(1); 192 | 193 | const letters_counted_map_cursor = try moment.putCursor(hashInt("letters-counted-map")); 194 | const letters_counted_map = try DB.CountedHashMap(.read_write).init(letters_counted_map_cursor); 195 | try letters_counted_map.put(hashInt("a"), .{ .uint = 1 }); 196 | try letters_counted_map.putKey(hashInt("a"), .{ .bytes = "a" }); 197 | try letters_counted_map.put(hashInt("a"), .{ .uint = 2 }); 198 | try letters_counted_map.putKey(hashInt("a"), .{ .bytes = "b" }); 199 | try letters_counted_map.put(hashInt("c"), .{ .uint = 3 }); 200 | try letters_counted_map.putKey(hashInt("c"), .{ .bytes = "c" }); 201 | 202 | const letters_set_cursor = try moment.putCursor(hashInt("letters-set")); 203 | const letters_set = try DB.HashSet(.read_write).init(letters_set_cursor); 204 | try letters_set.put(hashInt("a"), .{ .bytes = "a" }); 205 | try letters_set.put(hashInt("a"), .{ .bytes = "a" }); 206 | try letters_set.put(hashInt("c"), .{ .bytes = "c" }); 207 | 208 | const letters_counted_set_cursor = try moment.putCursor(hashInt("letters-counted-set")); 209 | const letters_counted_set = try DB.CountedHashSet(.read_write).init(letters_counted_set_cursor); 210 | try letters_counted_set.put(hashInt("a"), .{ .bytes = "a" }); 211 | try letters_counted_set.put(hashInt("a"), .{ .bytes = "a" }); 212 | try letters_counted_set.put(hashInt("c"), .{ .bytes = "c" }); 213 | } 214 | }; 215 | try history.appendContext(.{ .slot = try history.getSlot(-1) }, Ctx{}); 216 | 217 | // get the most recent copy of the database, like a moment 218 | // in time. the -1 index will return the last index in the list. 219 | const moment_cursor = (try history.getCursor(-1)).?; 220 | const moment = try DB.HashMap(.read_only).init(moment_cursor); 221 | 222 | // we can read the value of "foo" from the map by getting 223 | // the cursor to "foo" and then calling readBytesAlloc on it 224 | const foo_cursor = (try moment.getCursor(hashInt("foo"))).?; 225 | const foo_value = try foo_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 226 | defer allocator.free(foo_value); 227 | try std.testing.expectEqualStrings("foo", foo_value); 228 | 229 | try std.testing.expectEqual(.short_bytes, (try moment.getSlot(hashInt("foo"))).?.tag); 230 | try std.testing.expectEqual(.short_bytes, (try moment.getSlot(hashInt("bar"))).?.tag); 231 | 232 | // to get the "fruits" list, we get the cursor to it and 233 | // then pass it to the ArrayList.init method 234 | const fruits_cursor = (try moment.getCursor(hashInt("fruits"))).?; 235 | const fruits = try DB.ArrayList(.read_only).init(fruits_cursor); 236 | try std.testing.expectEqual(3, try fruits.count()); 237 | 238 | // now we can get the first item from the fruits list and read it 239 | const apple_cursor = (try fruits.getCursor(0)).?; 240 | const apple_value = try apple_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 241 | defer allocator.free(apple_value); 242 | try std.testing.expectEqualStrings("apple", apple_value); 243 | 244 | const people_cursor = (try moment.getCursor(hashInt("people"))).?; 245 | const people = try DB.ArrayList(.read_only).init(people_cursor); 246 | try std.testing.expectEqual(2, try people.count()); 247 | 248 | const alice_cursor = (try people.getCursor(0)).?; 249 | const alice = try DB.HashMap(.read_only).init(alice_cursor); 250 | const alice_age_cursor = (try alice.getCursor(hashInt("age"))).?; 251 | try std.testing.expectEqual(25, try alice_age_cursor.readUint()); 252 | 253 | const todos_cursor = (try moment.getCursor(hashInt("todos"))).?; 254 | const todos = try DB.LinkedArrayList(.read_only).init(todos_cursor); 255 | try std.testing.expectEqual(3, try todos.count()); 256 | 257 | const todo_cursor = (try todos.getCursor(0)).?; 258 | const todo_value = try todo_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 259 | defer allocator.free(todo_value); 260 | try std.testing.expectEqualStrings("Pay the bills", todo_value); 261 | 262 | var people_iter = try people.iterator(); 263 | while (try people_iter.next()) |person_cursor| { 264 | const person = try DB.HashMap(.read_only).init(person_cursor); 265 | var person_iter = try person.iterator(); 266 | while (try person_iter.next()) |kv_pair_cursor| { 267 | _ = try kv_pair_cursor.readKeyValuePair(); 268 | } 269 | } 270 | 271 | { 272 | const letters_counted_map_cursor = (try moment.getCursor(hashInt("letters-counted-map"))).?; 273 | const letters_counted_map = try DB.CountedHashMap(.read_only).init(letters_counted_map_cursor); 274 | try std.testing.expectEqual(2, try letters_counted_map.count()); 275 | 276 | var iter = try letters_counted_map.iterator(); 277 | var count: usize = 0; 278 | while (try iter.next()) |kv_pair_cursor| { 279 | const kv_pair = try kv_pair_cursor.readKeyValuePair(); 280 | var letter_buf = [_]u8{0} ** 8; 281 | _ = try kv_pair.key_cursor.readBytes(&letter_buf); 282 | count += 1; 283 | } 284 | try std.testing.expectEqual(2, count); 285 | } 286 | 287 | { 288 | const letters_set_cursor = (try moment.getCursor(hashInt("letters-set"))).?; 289 | const letters_set = try DB.HashSet(.read_only).init(letters_set_cursor); 290 | try std.testing.expect(null != try letters_set.getCursor(hashInt("a"))); 291 | try std.testing.expect(null != try letters_set.getCursor(hashInt("c"))); 292 | 293 | var iter = try letters_set.iterator(); 294 | var count: usize = 0; 295 | while (try iter.next()) |kv_pair_cursor| { 296 | const kv_pair = try kv_pair_cursor.readKeyValuePair(); 297 | var letter_buf = [_]u8{0} ** 8; 298 | _ = try kv_pair.key_cursor.readBytes(&letter_buf); 299 | count += 1; 300 | } 301 | try std.testing.expectEqual(2, count); 302 | } 303 | 304 | { 305 | const letters_counted_set_cursor = (try moment.getCursor(hashInt("letters-counted-set"))).?; 306 | const letters_counted_set = try DB.CountedHashSet(.read_only).init(letters_counted_set_cursor); 307 | try std.testing.expectEqual(2, try letters_counted_set.count()); 308 | 309 | var iter = try letters_counted_set.iterator(); 310 | var count: usize = 0; 311 | while (try iter.next()) |kv_pair_cursor| { 312 | const kv_pair = try kv_pair_cursor.readKeyValuePair(); 313 | var letter_buf = [_]u8{0} ** 8; 314 | _ = try kv_pair.key_cursor.readBytes(&letter_buf); 315 | count += 1; 316 | } 317 | try std.testing.expectEqual(2, count); 318 | } 319 | } 320 | 321 | // make a new transaction and change the data 322 | { 323 | const history = try DB.ArrayList(.read_write).init(db.rootCursor()); 324 | 325 | const Ctx = struct { 326 | pub fn run(_: @This(), cursor: *DB.Cursor(.read_write)) !void { 327 | const moment = try DB.HashMap(.read_write).init(cursor.*); 328 | 329 | try std.testing.expect(try moment.remove(hashInt("bar"))); 330 | try std.testing.expect(!try moment.remove(hashInt("doesn't exist"))); 331 | 332 | // this associates the hash of "fruits" with the actual string. 333 | // hash maps use hashes directly as keys so they are not able 334 | // to get the original bytes of the key unless we store it 335 | // explicitly this way. 336 | try moment.putKey(hashInt("fruits"), .{ .bytes = "fruits" }); 337 | 338 | const fruits_cursor = try moment.putCursor(hashInt("fruits")); 339 | const fruits = try DB.ArrayList(.read_write).init(fruits_cursor); 340 | try fruits.put(0, .{ .bytes = "lemon" }); 341 | try fruits.slice(2); 342 | 343 | const people_cursor = try moment.putCursor(hashInt("people")); 344 | const people = try DB.ArrayList(.read_write).init(people_cursor); 345 | 346 | const alice_cursor = try people.putCursor(0); 347 | const alice = try DB.HashMap(.read_write).init(alice_cursor); 348 | try alice.put(hashInt("age"), .{ .uint = 26 }); 349 | 350 | const todos_cursor = try moment.putCursor(hashInt("todos")); 351 | const todos = try DB.LinkedArrayList(.read_write).init(todos_cursor); 352 | try todos.concat(todos_cursor.slot()); 353 | try todos.slice(1, 2); 354 | try todos.remove(1); 355 | 356 | const letters_counted_map_cursor = try moment.putCursor(hashInt("letters-counted-map")); 357 | const letters_counted_map = try DB.CountedHashMap(.read_write).init(letters_counted_map_cursor); 358 | _ = try letters_counted_map.remove(hashInt("b")); 359 | _ = try letters_counted_map.remove(hashInt("c")); 360 | 361 | const letters_set_cursor = try moment.putCursor(hashInt("letters-set")); 362 | const letters_set = try DB.HashSet(.read_write).init(letters_set_cursor); 363 | _ = try letters_set.remove(hashInt("b")); 364 | _ = try letters_set.remove(hashInt("c")); 365 | 366 | const letters_counted_set_cursor = try moment.putCursor(hashInt("letters-counted-set")); 367 | const letters_counted_set = try DB.CountedHashSet(.read_write).init(letters_counted_set_cursor); 368 | _ = try letters_counted_set.remove(hashInt("b")); 369 | _ = try letters_counted_set.remove(hashInt("c")); 370 | } 371 | }; 372 | try history.appendContext(.{ .slot = try history.getSlot(-1) }, Ctx{}); 373 | 374 | const moment_cursor = (try history.getCursor(-1)).?; 375 | const moment = try DB.HashMap(.read_only).init(moment_cursor); 376 | 377 | try std.testing.expectEqual(null, try moment.getCursor(hashInt("bar"))); 378 | 379 | const fruits_key_cursor = (try moment.getKeyCursor(hashInt("fruits"))).?; 380 | const fruits_key_value = try fruits_key_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 381 | defer allocator.free(fruits_key_value); 382 | try std.testing.expectEqualStrings("fruits", fruits_key_value); 383 | 384 | const fruits_cursor = (try moment.getCursor(hashInt("fruits"))).?; 385 | const fruits = try DB.ArrayList(.read_only).init(fruits_cursor); 386 | try std.testing.expectEqual(2, try fruits.count()); 387 | 388 | // you can get both the key and value cursor this way 389 | const fruits_kv_cursor = (try moment.getKeyValuePair(hashInt("fruits"))).?; 390 | try std.testing.expectEqual(.short_bytes, fruits_kv_cursor.key_cursor.slot_ptr.slot.tag); 391 | try std.testing.expectEqual(.array_list, fruits_kv_cursor.value_cursor.slot_ptr.slot.tag); 392 | 393 | const lemon_cursor = (try fruits.getCursor(0)).?; 394 | const lemon_value = try lemon_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 395 | defer allocator.free(lemon_value); 396 | try std.testing.expectEqualStrings("lemon", lemon_value); 397 | 398 | const people_cursor = (try moment.getCursor(hashInt("people"))).?; 399 | const people = try DB.ArrayList(.read_only).init(people_cursor); 400 | try std.testing.expectEqual(2, try people.count()); 401 | 402 | const alice_cursor = (try people.getCursor(0)).?; 403 | const alice = try DB.HashMap(.read_only).init(alice_cursor); 404 | const alice_age_cursor = (try alice.getCursor(hashInt("age"))).?; 405 | try std.testing.expectEqual(26, try alice_age_cursor.readUint()); 406 | 407 | const todos_cursor = (try moment.getCursor(hashInt("todos"))).?; 408 | const todos = try DB.LinkedArrayList(.read_only).init(todos_cursor); 409 | try std.testing.expectEqual(1, try todos.count()); 410 | 411 | const todo_cursor = (try todos.getCursor(0)).?; 412 | const todo_value = try todo_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 413 | defer allocator.free(todo_value); 414 | try std.testing.expectEqualStrings("Wash the car", todo_value); 415 | 416 | const letters_counted_map_cursor = (try moment.getCursor(hashInt("letters-counted-map"))).?; 417 | const letters_counted_map = try DB.CountedHashMap(.read_only).init(letters_counted_map_cursor); 418 | try std.testing.expectEqual(1, try letters_counted_map.count()); 419 | 420 | const letters_set_cursor = (try moment.getCursor(hashInt("letters-set"))).?; 421 | const letters_set = try DB.HashSet(.read_only).init(letters_set_cursor); 422 | try std.testing.expect(null != try letters_set.getCursor(hashInt("a"))); 423 | try std.testing.expect(null == try letters_set.getCursor(hashInt("b"))); 424 | 425 | const letters_counted_set_cursor = (try moment.getCursor(hashInt("letters-counted-set"))).?; 426 | const letters_counted_set = try DB.CountedHashSet(.read_only).init(letters_counted_set_cursor); 427 | try std.testing.expectEqual(1, try letters_counted_set.count()); 428 | } 429 | 430 | // the old data hasn't changed 431 | { 432 | const history = try DB.ArrayList(.read_write).init(db.rootCursor()); 433 | 434 | const moment_cursor = (try history.getCursor(0)).?; 435 | const moment = try DB.HashMap(.read_only).init(moment_cursor); 436 | 437 | const foo_cursor = (try moment.getCursor(hashInt("foo"))).?; 438 | const foo_value = try foo_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 439 | defer allocator.free(foo_value); 440 | try std.testing.expectEqualStrings("foo", foo_value); 441 | 442 | try std.testing.expectEqual(.short_bytes, (try moment.getSlot(hashInt("foo"))).?.tag); 443 | try std.testing.expectEqual(.short_bytes, (try moment.getSlot(hashInt("bar"))).?.tag); 444 | 445 | const fruits_cursor = (try moment.getCursor(hashInt("fruits"))).?; 446 | const fruits = try DB.ArrayList(.read_only).init(fruits_cursor); 447 | try std.testing.expectEqual(3, try fruits.count()); 448 | 449 | const apple_cursor = (try fruits.getCursor(0)).?; 450 | const apple_value = try apple_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 451 | defer allocator.free(apple_value); 452 | try std.testing.expectEqualStrings("apple", apple_value); 453 | 454 | const people_cursor = (try moment.getCursor(hashInt("people"))).?; 455 | const people = try DB.ArrayList(.read_only).init(people_cursor); 456 | try std.testing.expectEqual(2, try people.count()); 457 | 458 | const alice_cursor = (try people.getCursor(0)).?; 459 | const alice = try DB.HashMap(.read_only).init(alice_cursor); 460 | const alice_age_cursor = (try alice.getCursor(hashInt("age"))).?; 461 | try std.testing.expectEqual(25, try alice_age_cursor.readUint()); 462 | 463 | const todos_cursor = (try moment.getCursor(hashInt("todos"))).?; 464 | const todos = try DB.LinkedArrayList(.read_only).init(todos_cursor); 465 | try std.testing.expectEqual(3, try todos.count()); 466 | 467 | const todo_cursor = (try todos.getCursor(0)).?; 468 | const todo_value = try todo_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 469 | defer allocator.free(todo_value); 470 | try std.testing.expectEqualStrings("Pay the bills", todo_value); 471 | } 472 | 473 | // remove the last transaction with `slice` 474 | { 475 | const history = try DB.ArrayList(.read_write).init(db.rootCursor()); 476 | 477 | try history.slice(1); 478 | 479 | const moment_cursor = (try history.getCursor(-1)).?; 480 | const moment = try DB.HashMap(.read_only).init(moment_cursor); 481 | 482 | const foo_cursor = (try moment.getCursor(hashInt("foo"))).?; 483 | const foo_value = try foo_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 484 | defer allocator.free(foo_value); 485 | try std.testing.expectEqualStrings("foo", foo_value); 486 | 487 | try std.testing.expectEqual(.short_bytes, (try moment.getSlot(hashInt("foo"))).?.tag); 488 | try std.testing.expectEqual(.short_bytes, (try moment.getSlot(hashInt("bar"))).?.tag); 489 | 490 | const fruits_cursor = (try moment.getCursor(hashInt("fruits"))).?; 491 | const fruits = try DB.ArrayList(.read_only).init(fruits_cursor); 492 | try std.testing.expectEqual(3, try fruits.count()); 493 | 494 | const apple_cursor = (try fruits.getCursor(0)).?; 495 | const apple_value = try apple_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 496 | defer allocator.free(apple_value); 497 | try std.testing.expectEqualStrings("apple", apple_value); 498 | 499 | const people_cursor = (try moment.getCursor(hashInt("people"))).?; 500 | const people = try DB.ArrayList(.read_only).init(people_cursor); 501 | try std.testing.expectEqual(2, try people.count()); 502 | 503 | const alice_cursor = (try people.getCursor(0)).?; 504 | const alice = try DB.HashMap(.read_only).init(alice_cursor); 505 | const alice_age_cursor = (try alice.getCursor(hashInt("age"))).?; 506 | try std.testing.expectEqual(25, try alice_age_cursor.readUint()); 507 | 508 | const todos_cursor = (try moment.getCursor(hashInt("todos"))).?; 509 | const todos = try DB.LinkedArrayList(.read_only).init(todos_cursor); 510 | try std.testing.expectEqual(3, try todos.count()); 511 | 512 | const todo_cursor = (try todos.getCursor(0)).?; 513 | const todo_value = try todo_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 514 | defer allocator.free(todo_value); 515 | try std.testing.expectEqualStrings("Pay the bills", todo_value); 516 | } 517 | 518 | // the db size remains the same after writing junk data 519 | // and then reinitializing the db. this is useful because 520 | // there could be data from a transaction that never 521 | // completed due to an unclean shutdown. 522 | { 523 | try db.core.seekFromEnd(0); 524 | const size_before = try db.core.getPos(); 525 | 526 | const writer = db.core.writer(); 527 | try writer.writeAll("this is junk data that will be deleted during init"); 528 | 529 | // no error is thrown if db file is opened in read-only mode 530 | if (db_kind == .file) { 531 | const file = try std.fs.cwd().openFile("main.db", .{ .mode = .read_only }); 532 | defer file.close(); 533 | _ = try DB.init(.{ .file = file }); 534 | } 535 | 536 | db = try DB.init(init_opts); 537 | 538 | try db.core.seekFromEnd(0); 539 | const size_after = try db.core.getPos(); 540 | 541 | try std.testing.expectEqual(size_before, size_after); 542 | } 543 | } 544 | 545 | fn clearStorage(comptime db_kind: xitdb.DatabaseKind, init_opts: xitdb.Database(db_kind, HashInt).InitOpts) !void { 546 | switch (db_kind) { 547 | .file => { 548 | try init_opts.file.setEndPos(0); 549 | }, 550 | .memory => { 551 | init_opts.buffer.clearAndFree(); 552 | }, 553 | } 554 | } 555 | 556 | fn testSlice(allocator: std.mem.Allocator, comptime db_kind: xitdb.DatabaseKind, init_opts: xitdb.Database(db_kind, HashInt).InitOpts, comptime original_size: usize, comptime slice_offset: u64, comptime slice_size: u64) !void { 557 | try clearStorage(db_kind, init_opts); 558 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 559 | var root_cursor = db.rootCursor(); 560 | 561 | const Ctx = struct { 562 | allocator: std.mem.Allocator, 563 | 564 | pub fn run(self: @This(), cursor: *xitdb.Database(db_kind, HashInt).Cursor(.read_write)) !void { 565 | var values = std.ArrayList(u64).init(self.allocator); 566 | defer values.deinit(); 567 | 568 | // create list 569 | for (0..original_size) |i| { 570 | const n = i * 2; 571 | try values.append(n); 572 | _ = try cursor.writePath(void, &.{ 573 | .{ .hash_map_get = .{ .value = hashInt("even") } }, 574 | .linked_array_list_init, 575 | .linked_array_list_append, 576 | .{ .write = .{ .uint = n } }, 577 | }); 578 | } 579 | 580 | // slice list 581 | const even_list_cursor = (try cursor.readPath(void, &.{ 582 | .{ .hash_map_get = .{ .value = hashInt("even") } }, 583 | })).?; 584 | var even_list_slice_cursor = try cursor.writePath(void, &.{ 585 | .{ .hash_map_get = .{ .value = hashInt("even-slice") } }, 586 | .{ .write = .{ .slot = even_list_cursor.slot_ptr.slot } }, 587 | .linked_array_list_init, 588 | .{ .linked_array_list_slice = .{ .offset = slice_offset, .size = slice_size } }, 589 | }); 590 | 591 | // check all values in the new slice 592 | for (values.items[slice_offset .. slice_offset + slice_size], 0..) |val, i| { 593 | const n = (try cursor.readPath(void, &.{ 594 | .{ .hash_map_get = .{ .value = hashInt("even-slice") } }, 595 | .{ .linked_array_list_get = i }, 596 | })).?.slot_ptr.slot.value; 597 | try std.testing.expectEqual(val, n); 598 | } 599 | 600 | // check all values in the new slice with an iterator 601 | { 602 | var iter = try even_list_slice_cursor.iterator(); 603 | var i: u64 = 0; 604 | while (try iter.next()) |num_cursor| { 605 | try std.testing.expectEqual(values.items[slice_offset + i], try num_cursor.readUint()); 606 | i += 1; 607 | } 608 | try std.testing.expectEqual(slice_size, i); 609 | } 610 | 611 | // there are no extra items 612 | try std.testing.expectEqual(null, try cursor.readPath(void, &.{ 613 | .{ .hash_map_get = .{ .value = hashInt("even-slice") } }, 614 | .{ .linked_array_list_get = slice_size }, 615 | })); 616 | 617 | // concat the slice with itself 618 | _ = try cursor.writePath(void, &.{ 619 | .{ .hash_map_get = .{ .value = hashInt("combo") } }, 620 | .{ .write = .{ .slot = even_list_slice_cursor.slot_ptr.slot } }, 621 | .linked_array_list_init, 622 | .{ .linked_array_list_concat = .{ .list = even_list_slice_cursor.slot_ptr.slot } }, 623 | }); 624 | 625 | // check all values in the combo list 626 | var combo_values = std.ArrayList(u64).init(self.allocator); 627 | defer combo_values.deinit(); 628 | try combo_values.appendSlice(values.items[slice_offset .. slice_offset + slice_size]); 629 | try combo_values.appendSlice(values.items[slice_offset .. slice_offset + slice_size]); 630 | for (combo_values.items, 0..) |val, i| { 631 | const n = (try cursor.readPath(void, &.{ 632 | .{ .hash_map_get = .{ .value = hashInt("combo") } }, 633 | .{ .linked_array_list_get = i }, 634 | })).?.slot_ptr.slot.value; 635 | try std.testing.expectEqual(val, n); 636 | } 637 | 638 | // append to the slice 639 | _ = try cursor.writePath(void, &.{ 640 | .{ .hash_map_get = .{ .value = hashInt("even-slice") } }, 641 | .linked_array_list_init, 642 | .linked_array_list_append, 643 | .{ .write = .{ .uint = 3 } }, 644 | }); 645 | 646 | // read the new value from the slice 647 | try std.testing.expectEqual(3, (try cursor.readPath(void, &.{ 648 | .{ .hash_map_get = .{ .value = hashInt("even-slice") } }, 649 | .{ .linked_array_list_get = -1 }, 650 | })).?.slot_ptr.slot.value); 651 | } 652 | }; 653 | _ = try root_cursor.writePath(Ctx, &.{ 654 | .array_list_init, 655 | .array_list_append, 656 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 657 | .{ .hash_map_init = .{} }, 658 | .{ .ctx = .{ .allocator = allocator } }, 659 | }); 660 | } 661 | 662 | fn testConcat(allocator: std.mem.Allocator, comptime db_kind: xitdb.DatabaseKind, init_opts: xitdb.Database(db_kind, HashInt).InitOpts, comptime list_a_size: usize, comptime list_b_size: usize) !void { 663 | try clearStorage(db_kind, init_opts); 664 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 665 | var root_cursor = db.rootCursor(); 666 | 667 | var values = std.ArrayList(u64).init(allocator); 668 | defer values.deinit(); 669 | 670 | { 671 | const Ctx = struct { 672 | allocator: std.mem.Allocator, 673 | values: *std.ArrayList(u64), 674 | 675 | pub fn run(self: @This(), cursor: *xitdb.Database(db_kind, HashInt).Cursor(.read_write)) !void { 676 | // create even list 677 | _ = try cursor.writePath(void, &.{ 678 | .{ .hash_map_get = .{ .value = hashInt("even") } }, 679 | .linked_array_list_init, 680 | }); 681 | for (0..list_a_size) |i| { 682 | const n = i * 2; 683 | try self.values.append(n); 684 | _ = try cursor.writePath(void, &.{ 685 | .{ .hash_map_get = .{ .value = hashInt("even") } }, 686 | .linked_array_list_init, 687 | .linked_array_list_append, 688 | .{ .write = .{ .uint = n } }, 689 | }); 690 | } 691 | 692 | // create odd list 693 | _ = try cursor.writePath(void, &.{ 694 | .{ .hash_map_get = .{ .value = hashInt("odd") } }, 695 | .linked_array_list_init, 696 | }); 697 | for (0..list_b_size) |i| { 698 | const n = (i * 2) + 1; 699 | try self.values.append(n); 700 | _ = try cursor.writePath(void, &.{ 701 | .{ .hash_map_get = .{ .value = hashInt("odd") } }, 702 | .linked_array_list_init, 703 | .linked_array_list_append, 704 | .{ .write = .{ .uint = n } }, 705 | }); 706 | } 707 | } 708 | }; 709 | _ = try root_cursor.writePath(Ctx, &.{ 710 | .array_list_init, 711 | .array_list_append, 712 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 713 | .{ .hash_map_init = .{} }, 714 | .{ .ctx = .{ .allocator = allocator, .values = &values } }, 715 | }); 716 | } 717 | 718 | { 719 | const Ctx = struct { 720 | allocator: std.mem.Allocator, 721 | values: *std.ArrayList(u64), 722 | 723 | pub fn run(self: @This(), cursor: *xitdb.Database(db_kind, HashInt).Cursor(.read_write)) !void { 724 | // get even list 725 | const even_list_cursor = (try cursor.readPath(void, &.{ 726 | .{ .hash_map_get = .{ .value = hashInt("even") } }, 727 | })).?; 728 | 729 | // get odd list 730 | const odd_list_cursor = (try cursor.readPath(void, &.{ 731 | .{ .hash_map_get = .{ .value = hashInt("odd") } }, 732 | })).?; 733 | 734 | // concat the lists 735 | const combo_list_cursor = try cursor.writePath(void, &.{ 736 | .{ .hash_map_get = .{ .value = hashInt("combo") } }, 737 | .{ .write = .{ .slot = even_list_cursor.slot_ptr.slot } }, 738 | .linked_array_list_init, 739 | .{ .linked_array_list_concat = .{ .list = odd_list_cursor.slot_ptr.slot } }, 740 | }); 741 | 742 | // check all values in the new list 743 | for (self.values.items, 0..) |val, i| { 744 | const n = (try cursor.readPath(void, &.{ 745 | .{ .hash_map_get = .{ .value = hashInt("combo") } }, 746 | .{ .linked_array_list_get = i }, 747 | })).?.slot_ptr.slot.value; 748 | try std.testing.expectEqual(val, n); 749 | } 750 | 751 | // check all values in the new list with an iterator 752 | { 753 | var iter = try combo_list_cursor.iterator(); 754 | var i: u64 = 0; 755 | while (try iter.next()) |num_cursor| { 756 | try std.testing.expectEqual(self.values.items[i], try num_cursor.readUint()); 757 | i += 1; 758 | } 759 | try std.testing.expectEqual(try even_list_cursor.count() + try odd_list_cursor.count(), i); 760 | } 761 | 762 | // there are no extra items 763 | try std.testing.expectEqual(null, try cursor.readPath(void, &.{ 764 | .{ .hash_map_get = .{ .value = hashInt("combo") } }, 765 | .{ .linked_array_list_get = self.values.items.len }, 766 | })); 767 | } 768 | }; 769 | _ = try root_cursor.writePath(Ctx, &.{ 770 | .array_list_init, 771 | .array_list_append, 772 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 773 | .{ .hash_map_init = .{} }, 774 | .{ .ctx = .{ .allocator = allocator, .values = &values } }, 775 | }); 776 | } 777 | } 778 | 779 | fn testInsertAndRemove(allocator: std.mem.Allocator, comptime db_kind: xitdb.DatabaseKind, init_opts: xitdb.Database(db_kind, HashInt).InitOpts, comptime original_size: usize, comptime insert_index: u64) !void { 780 | try clearStorage(db_kind, init_opts); 781 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 782 | var root_cursor = db.rootCursor(); 783 | 784 | const insert_value: u64 = 12345; 785 | 786 | const InsertCtx = struct { 787 | allocator: std.mem.Allocator, 788 | 789 | pub fn run(self: @This(), cursor: *xitdb.Database(db_kind, HashInt).Cursor(.read_write)) !void { 790 | var values = std.ArrayList(u64).init(self.allocator); 791 | defer values.deinit(); 792 | 793 | // create list 794 | for (0..original_size) |i| { 795 | if (i == insert_index) { 796 | try values.append(insert_value); 797 | } 798 | const n = i * 2; 799 | try values.append(n); 800 | _ = try cursor.writePath(void, &.{ 801 | .{ .hash_map_get = .{ .value = hashInt("even") } }, 802 | .linked_array_list_init, 803 | .linked_array_list_append, 804 | .{ .write = .{ .uint = n } }, 805 | }); 806 | } 807 | 808 | // insert into list 809 | const even_list_cursor = (try cursor.readPath(void, &.{ 810 | .{ .hash_map_get = .{ .value = hashInt("even") } }, 811 | })).?; 812 | var even_list_insert_cursor = try cursor.writePath(void, &.{ 813 | .{ .hash_map_get = .{ .value = hashInt("even-insert") } }, 814 | .{ .write = .{ .slot = even_list_cursor.slot_ptr.slot } }, 815 | .linked_array_list_init, 816 | }); 817 | _ = try even_list_insert_cursor.writePath(void, &.{ 818 | .{ .linked_array_list_insert = insert_index }, 819 | .{ .write = .{ .uint = insert_value } }, 820 | }); 821 | 822 | // check all values in the new list 823 | for (values.items, 0..) |val, i| { 824 | const n = (try cursor.readPath(void, &.{ 825 | .{ .hash_map_get = .{ .value = hashInt("even-insert") } }, 826 | .{ .linked_array_list_get = i }, 827 | })).?.slot_ptr.slot.value; 828 | try std.testing.expectEqual(val, n); 829 | } 830 | 831 | // check all values in the new list with an iterator 832 | { 833 | var iter = try even_list_insert_cursor.iterator(); 834 | var i: u64 = 0; 835 | while (try iter.next()) |num_cursor| { 836 | try std.testing.expectEqual(values.items[i], try num_cursor.readUint()); 837 | i += 1; 838 | } 839 | try std.testing.expectEqual(values.items.len, i); 840 | } 841 | 842 | // there are no extra items 843 | try std.testing.expectEqual(null, try cursor.readPath(void, &.{ 844 | .{ .hash_map_get = .{ .value = hashInt("even-insert") } }, 845 | .{ .linked_array_list_get = values.items.len }, 846 | })); 847 | } 848 | }; 849 | _ = try root_cursor.writePath(InsertCtx, &.{ 850 | .array_list_init, 851 | .array_list_append, 852 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 853 | .{ .hash_map_init = .{} }, 854 | .{ .ctx = .{ .allocator = allocator } }, 855 | }); 856 | 857 | const RemoveCtx = struct { 858 | allocator: std.mem.Allocator, 859 | 860 | pub fn run(self: @This(), cursor: *xitdb.Database(db_kind, HashInt).Cursor(.read_write)) !void { 861 | var values = std.ArrayList(u64).init(self.allocator); 862 | defer values.deinit(); 863 | 864 | for (0..original_size) |i| { 865 | const n = i * 2; 866 | try values.append(n); 867 | } 868 | 869 | // remove inserted value from the list 870 | const even_list_insert_cursor = try cursor.writePath(void, &.{ 871 | .{ .hash_map_get = .{ .value = hashInt("even-insert") } }, 872 | .{ .linked_array_list_remove = insert_index }, 873 | }); 874 | 875 | // check all values in the new list 876 | for (values.items, 0..) |val, i| { 877 | const n = (try cursor.readPath(void, &.{ 878 | .{ .hash_map_get = .{ .value = hashInt("even-insert") } }, 879 | .{ .linked_array_list_get = i }, 880 | })).?.slot_ptr.slot.value; 881 | try std.testing.expectEqual(val, n); 882 | } 883 | 884 | // check all values in the new list with an iterator 885 | { 886 | var iter = try even_list_insert_cursor.iterator(); 887 | var i: u64 = 0; 888 | while (try iter.next()) |num_cursor| { 889 | try std.testing.expectEqual(values.items[i], try num_cursor.readUint()); 890 | i += 1; 891 | } 892 | try std.testing.expectEqual(values.items.len, i); 893 | } 894 | 895 | // there are no extra items 896 | try std.testing.expectEqual(null, try cursor.readPath(void, &.{ 897 | .{ .hash_map_get = .{ .value = hashInt("even-insert") } }, 898 | .{ .linked_array_list_get = values.items.len }, 899 | })); 900 | } 901 | }; 902 | _ = try root_cursor.writePath(RemoveCtx, &.{ 903 | .array_list_init, 904 | .array_list_append, 905 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 906 | .{ .hash_map_init = .{} }, 907 | .{ .ctx = .{ .allocator = allocator } }, 908 | }); 909 | } 910 | 911 | fn testLowLevelApi(allocator: std.mem.Allocator, comptime db_kind: xitdb.DatabaseKind, init_opts: xitdb.Database(db_kind, HashInt).InitOpts) !void { 912 | // open and re-open empty database 913 | { 914 | // make empty database 915 | try clearStorage(db_kind, init_opts); 916 | _ = try xitdb.Database(db_kind, HashInt).init(init_opts); 917 | 918 | // re-open without error 919 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 920 | const writer = db.core.writer(); 921 | 922 | // modify the magic number 923 | try db.core.seekTo(0); 924 | try writer.writeInt(u8, 'g', .big); 925 | 926 | // re-open with error 927 | { 928 | const db_or_error = xitdb.Database(db_kind, HashInt).init(init_opts); 929 | if (db_or_error) |_| { 930 | return error.ExpectedInvalidDatabaseError; 931 | } else |err| { 932 | try std.testing.expectEqual(error.InvalidDatabase, err); 933 | } 934 | } 935 | 936 | // modify the version 937 | try db.core.seekTo(0); 938 | try writer.writeInt(u8, 'x', .big); 939 | try db.core.seekTo(4); 940 | try writer.writeInt(u16, xitdb.VERSION + 1, .big); 941 | 942 | // re-open with error 943 | { 944 | const db_or_error = xitdb.Database(db_kind, HashInt).init(init_opts); 945 | if (db_or_error) |_| { 946 | return error.ExpectedInvalidVersionError; 947 | } else |err| { 948 | try std.testing.expectEqual(error.InvalidVersion, err); 949 | } 950 | } 951 | } 952 | 953 | // save hash id in header 954 | { 955 | var init_opts_with_hash_id = init_opts; 956 | init_opts_with_hash_id.hash_id = xitdb.HashId.fromBytes("sha1"); 957 | 958 | // make empty database 959 | try clearStorage(db_kind, init_opts); 960 | const db = try xitdb.Database(db_kind, HashInt).init(init_opts_with_hash_id); 961 | 962 | try std.testing.expectEqual(xitdb.HashId.fromBytes("sha1").id, db.header.hash_id.id); 963 | try std.testing.expectEqualStrings("sha1", &db.header.hash_id.toBytes()); 964 | } 965 | 966 | // array_list of hash_maps 967 | { 968 | try clearStorage(db_kind, init_opts); 969 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 970 | var root_cursor = db.rootCursor(); 971 | 972 | // write foo -> bar with a writer 973 | const foo_key = hashInt("foo"); 974 | { 975 | const Ctx = struct { 976 | pub fn run(_: @This(), cursor: *xitdb.Database(db_kind, HashInt).Cursor(.read_write)) !void { 977 | try std.testing.expect(cursor.slot().tag == .none); 978 | var writer = try cursor.writer(); 979 | try writer.writeAll("bar"); 980 | try writer.finish(); 981 | } 982 | }; 983 | _ = try root_cursor.writePath(Ctx, &.{ 984 | .array_list_init, 985 | .array_list_append, 986 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 987 | .{ .hash_map_init = .{} }, 988 | .{ .hash_map_get = .{ .value = foo_key } }, 989 | .{ .ctx = Ctx{} }, 990 | }); 991 | } 992 | 993 | // read foo 994 | { 995 | var bar_cursor = (try root_cursor.readPath(void, &.{ 996 | .{ .array_list_get = -1 }, 997 | .{ .hash_map_get = .{ .value = foo_key } }, 998 | })).?; 999 | try std.testing.expectEqual(3, bar_cursor.count()); 1000 | const bar_value = try bar_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1001 | defer allocator.free(bar_value); 1002 | try std.testing.expectEqualStrings("bar", bar_value); 1003 | 1004 | // make sure we can make a buffered reader 1005 | var buf_reader = std.io.bufferedReader(try bar_cursor.reader()); 1006 | _ = try buf_reader.read(&[_]u8{}); 1007 | } 1008 | 1009 | // read foo from ctx 1010 | { 1011 | const Ctx = struct { 1012 | allocator: std.mem.Allocator, 1013 | 1014 | pub fn run(self: @This(), cursor: *xitdb.Database(db_kind, HashInt).Cursor(.read_write)) !void { 1015 | try std.testing.expect(cursor.slot().tag != .none); 1016 | 1017 | const value = try cursor.readBytesAlloc(self.allocator, MAX_READ_BYTES); 1018 | defer self.allocator.free(value); 1019 | try std.testing.expectEqualStrings("bar", value); 1020 | 1021 | var bar_reader = try cursor.reader(); 1022 | 1023 | // read into buffer 1024 | var bar_bytes = [_]u8{0} ** 10; 1025 | try bar_reader.readNoEof(bar_bytes[0..3]); 1026 | try std.testing.expectEqualStrings("bar", bar_bytes[0..3]); 1027 | try bar_reader.seekTo(0); 1028 | try std.testing.expectEqual(3, try bar_reader.read(&bar_bytes)); 1029 | try std.testing.expectEqualStrings("bar", bar_bytes[0..3]); 1030 | 1031 | // read one char at a time 1032 | { 1033 | var char = [_]u8{0} ** 1; 1034 | try bar_reader.seekTo(0); 1035 | 1036 | try bar_reader.readNoEof(&char); 1037 | try std.testing.expectEqualStrings("b", &char); 1038 | 1039 | try bar_reader.readNoEof(&char); 1040 | try std.testing.expectEqualStrings("a", &char); 1041 | 1042 | try bar_reader.readNoEof(&char); 1043 | try std.testing.expectEqualStrings("r", &char); 1044 | 1045 | try std.testing.expectEqual(error.EndOfStream, bar_reader.readNoEof(&char)); 1046 | 1047 | try bar_reader.seekTo(2); 1048 | try bar_reader.seekBy(-1); 1049 | try std.testing.expectEqual('a', try bar_reader.readInt(u8, .big)); 1050 | 1051 | try bar_reader.seekFromEnd(-3); 1052 | try std.testing.expectEqual('b', try bar_reader.readInt(u8, .big)); 1053 | } 1054 | } 1055 | }; 1056 | _ = try root_cursor.writePath(Ctx, &.{ 1057 | .array_list_init, 1058 | .array_list_append, 1059 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1060 | .{ .hash_map_init = .{} }, 1061 | .{ .hash_map_get = .{ .value = foo_key } }, 1062 | .{ .ctx = Ctx{ .allocator = allocator } }, 1063 | }); 1064 | } 1065 | 1066 | // overwrite foo -> baz 1067 | { 1068 | const Ctx = struct { 1069 | allocator: std.mem.Allocator, 1070 | 1071 | pub fn run(self: @This(), cursor: *xitdb.Database(db_kind, HashInt).Cursor(.read_write)) !void { 1072 | try std.testing.expect(cursor.slot().tag != .none); 1073 | 1074 | var writer = try cursor.writer(); 1075 | try writer.writeAll("x"); 1076 | try writer.writeAll("x"); 1077 | try writer.writeAll("x"); 1078 | try writer.seekBy(-3); 1079 | try writer.writeAll("b"); 1080 | try writer.seekTo(2); 1081 | try writer.writeAll("z"); 1082 | try writer.seekFromEnd(-2); 1083 | try writer.writeAll("a"); 1084 | try writer.finish(); 1085 | 1086 | const value = try cursor.readBytesAlloc(self.allocator, MAX_READ_BYTES); 1087 | defer self.allocator.free(value); 1088 | try std.testing.expectEqualStrings("baz", value); 1089 | } 1090 | }; 1091 | _ = try root_cursor.writePath(Ctx, &.{ 1092 | .array_list_init, 1093 | .array_list_append, 1094 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1095 | .{ .hash_map_init = .{} }, 1096 | .{ .hash_map_get = .{ .value = foo_key } }, 1097 | .{ .ctx = Ctx{ .allocator = allocator } }, 1098 | }); 1099 | } 1100 | 1101 | // if error in ctx, db doesn't change 1102 | { 1103 | try db.core.seekFromEnd(0); 1104 | const size_before = try db.core.getPos(); 1105 | 1106 | const Ctx = struct { 1107 | allocator: std.mem.Allocator, 1108 | 1109 | pub fn run(_: @This(), cursor: *xitdb.Database(db_kind, HashInt).Cursor(.read_write)) !void { 1110 | var writer = try cursor.writer(); 1111 | try writer.writeAll("this value won't be visible"); 1112 | try writer.finish(); 1113 | return error.CancelTransaction; 1114 | } 1115 | }; 1116 | _ = root_cursor.writePath(Ctx, &.{ 1117 | .array_list_init, 1118 | .array_list_append, 1119 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1120 | .{ .hash_map_init = .{} }, 1121 | .{ .hash_map_get = .{ .value = foo_key } }, 1122 | .{ .ctx = Ctx{ .allocator = allocator } }, 1123 | }) catch |err| switch (err) { 1124 | error.CancelTransaction => {}, 1125 | else => |e| return e, 1126 | }; 1127 | 1128 | // read foo 1129 | const value_cursor = (try root_cursor.readPath(void, &.{ 1130 | .{ .array_list_get = -1 }, 1131 | .{ .hash_map_get = .{ .value = foo_key } }, 1132 | })).?; 1133 | const value = try value_cursor.readBytesAlloc(allocator, null); // make sure null max size works 1134 | defer allocator.free(value); 1135 | try std.testing.expectEqualStrings("baz", value); 1136 | 1137 | // verify that the db is properly truncated back to its original size after error 1138 | try db.core.seekFromEnd(0); 1139 | const size_after = try db.core.getPos(); 1140 | try std.testing.expectEqual(size_before, size_after); 1141 | } 1142 | 1143 | // write bar -> longstring 1144 | const bar_key = hashInt("bar"); 1145 | { 1146 | var bar_cursor = try root_cursor.writePath(void, &.{ 1147 | .array_list_init, 1148 | .array_list_append, 1149 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1150 | .{ .hash_map_init = .{} }, 1151 | .{ .hash_map_get = .{ .value = bar_key } }, 1152 | }); 1153 | try bar_cursor.write(.{ .bytes = "longstring" }); 1154 | 1155 | // the slot tag is .bytes because the byte array is > 8 bytes long 1156 | try std.testing.expectEqual(.bytes, bar_cursor.slot().tag); 1157 | 1158 | // writing again returns the same slot 1159 | { 1160 | var next_bar_cursor = try root_cursor.writePath(void, &.{ 1161 | .array_list_init, 1162 | .array_list_append, 1163 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1164 | .{ .hash_map_init = .{} }, 1165 | .{ .hash_map_get = .{ .value = bar_key } }, 1166 | }); 1167 | try next_bar_cursor.writeIfEmpty(.{ .bytes = "longstring" }); 1168 | try std.testing.expectEqual(bar_cursor.slot_ptr.slot, next_bar_cursor.slot_ptr.slot); 1169 | } 1170 | 1171 | // writing with write returns a new slot 1172 | { 1173 | var next_bar_cursor = try root_cursor.writePath(void, &.{ 1174 | .array_list_init, 1175 | .array_list_append, 1176 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1177 | .{ .hash_map_init = .{} }, 1178 | .{ .hash_map_get = .{ .value = bar_key } }, 1179 | }); 1180 | try next_bar_cursor.write(.{ .bytes = "longstring" }); 1181 | try std.testing.expect(!bar_cursor.slot_ptr.slot.eql(next_bar_cursor.slot_ptr.slot)); 1182 | } 1183 | 1184 | // read bar 1185 | { 1186 | const read_bar_cursor = (try root_cursor.readPath(void, &.{ 1187 | .{ .array_list_get = -1 }, 1188 | .{ .hash_map_get = .{ .value = bar_key } }, 1189 | })).?; 1190 | const bar_value = try read_bar_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1191 | defer allocator.free(bar_value); 1192 | try std.testing.expectEqualStrings("longstring", bar_value); 1193 | } 1194 | } 1195 | 1196 | // write bar -> shortstr 1197 | { 1198 | var bar_cursor = try root_cursor.writePath(void, &.{ 1199 | .array_list_init, 1200 | .array_list_append, 1201 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1202 | .{ .hash_map_init = .{} }, 1203 | .{ .hash_map_get = .{ .value = bar_key } }, 1204 | }); 1205 | try bar_cursor.write(.{ .bytes = "shortstr" }); 1206 | 1207 | // the slot tag is .short_bytes because the byte array is <= 8 bytes long 1208 | try std.testing.expectEqual(.short_bytes, bar_cursor.slot().tag); 1209 | try std.testing.expectEqual(8, try bar_cursor.count()); 1210 | 1211 | // make sure .short_bytes can be read with a reader 1212 | var bar_reader = try bar_cursor.reader(); 1213 | const bar_value = try bar_reader.readAllAlloc(allocator, MAX_READ_BYTES); 1214 | defer allocator.free(bar_value); 1215 | try std.testing.expectEqualStrings("shortstr", bar_value); 1216 | } 1217 | 1218 | // write bytes with a format tag 1219 | { 1220 | // shortstr 1221 | { 1222 | var bar_cursor = try root_cursor.writePath(void, &.{ 1223 | .array_list_init, 1224 | .array_list_append, 1225 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1226 | .{ .hash_map_init = .{} }, 1227 | .{ .hash_map_get = .{ .value = bar_key } }, 1228 | }); 1229 | try bar_cursor.write(.{ .bytes_object = .{ .value = "shortstr", .format_tag = "st".* } }); 1230 | 1231 | // the slot tag is .bytes because the byte array is > 8 bytes long including the format tag 1232 | try std.testing.expectEqual(.bytes, bar_cursor.slot().tag); 1233 | try std.testing.expectEqual(8, try bar_cursor.count()); 1234 | 1235 | // read bar 1236 | const read_bar_cursor = (try root_cursor.readPath(void, &.{ 1237 | .{ .array_list_get = -1 }, 1238 | .{ .hash_map_get = .{ .value = bar_key } }, 1239 | })).?; 1240 | const bytes = try read_bar_cursor.readBytesObjectAlloc(allocator, MAX_READ_BYTES); 1241 | defer allocator.free(bytes.value); 1242 | try std.testing.expectEqualStrings("shortstr", bytes.value); 1243 | try std.testing.expectEqualStrings("st", &bytes.format_tag.?); 1244 | 1245 | // make sure .bytes can be read with a reader 1246 | var bar_reader = try bar_cursor.reader(); 1247 | const bar_value = try bar_reader.readAllAlloc(allocator, MAX_READ_BYTES); 1248 | defer allocator.free(bar_value); 1249 | try std.testing.expectEqualStrings("shortstr", bar_value); 1250 | } 1251 | 1252 | // shorts 1253 | { 1254 | var bar_cursor = try root_cursor.writePath(void, &.{ 1255 | .array_list_init, 1256 | .array_list_append, 1257 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1258 | .{ .hash_map_init = .{} }, 1259 | .{ .hash_map_get = .{ .value = bar_key } }, 1260 | }); 1261 | try bar_cursor.write(.{ .bytes_object = .{ .value = "shorts", .format_tag = "st".* } }); 1262 | 1263 | // the slot tag is .short_bytes because the byte array is <= 8 bytes long including the format tag 1264 | try std.testing.expectEqual(.short_bytes, bar_cursor.slot().tag); 1265 | try std.testing.expectEqual(6, try bar_cursor.count()); 1266 | 1267 | // read bar 1268 | const read_bar_cursor = (try root_cursor.readPath(void, &.{ 1269 | .{ .array_list_get = -1 }, 1270 | .{ .hash_map_get = .{ .value = bar_key } }, 1271 | })).?; 1272 | var buffer = [_]u8{0} ** MAX_READ_BYTES; 1273 | const bytes = try read_bar_cursor.readBytesObject(&buffer); 1274 | try std.testing.expectEqualStrings("shorts", bytes.value); 1275 | try std.testing.expectEqualStrings("st", &bytes.format_tag.?); 1276 | 1277 | // make sure .short_bytes can be read with a reader 1278 | var bar_reader = try bar_cursor.reader(); 1279 | const bar_value = try bar_reader.readAllAlloc(allocator, MAX_READ_BYTES); 1280 | defer allocator.free(bar_value); 1281 | try std.testing.expectEqualStrings("shorts", bar_value); 1282 | } 1283 | 1284 | // short 1285 | { 1286 | var bar_cursor = try root_cursor.writePath(void, &.{ 1287 | .array_list_init, 1288 | .array_list_append, 1289 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1290 | .{ .hash_map_init = .{} }, 1291 | .{ .hash_map_get = .{ .value = bar_key } }, 1292 | }); 1293 | try bar_cursor.write(.{ .bytes_object = .{ .value = "short", .format_tag = "st".* } }); 1294 | 1295 | // the slot tag is .short_bytes because the byte array is <= 8 bytes long including the format tag 1296 | try std.testing.expectEqual(.short_bytes, bar_cursor.slot().tag); 1297 | try std.testing.expectEqual(5, try bar_cursor.count()); 1298 | 1299 | // read bar 1300 | const read_bar_cursor = (try root_cursor.readPath(void, &.{ 1301 | .{ .array_list_get = -1 }, 1302 | .{ .hash_map_get = .{ .value = bar_key } }, 1303 | })).?; 1304 | var buffer = [_]u8{0} ** MAX_READ_BYTES; 1305 | const bytes = try read_bar_cursor.readBytesObject(&buffer); 1306 | try std.testing.expectEqualStrings("short", bytes.value); 1307 | try std.testing.expectEqualStrings("st", &bytes.format_tag.?); 1308 | 1309 | // make sure .short_bytes can be read with a reader 1310 | var bar_reader = try bar_cursor.reader(); 1311 | const bar_value = try bar_reader.readAllAlloc(allocator, MAX_READ_BYTES); 1312 | defer allocator.free(bar_value); 1313 | try std.testing.expectEqualStrings("short", bar_value); 1314 | } 1315 | } 1316 | 1317 | // read foo into stack-allocated buffer 1318 | { 1319 | const bar_cursor = (try root_cursor.readPath(void, &.{ 1320 | .{ .array_list_get = -1 }, 1321 | .{ .hash_map_get = .{ .value = foo_key } }, 1322 | })).?; 1323 | var bar_buffer = [_]u8{0} ** 3; 1324 | const bar_buffer_value = try bar_cursor.readBytes(&bar_buffer); 1325 | try std.testing.expectEqualStrings("baz", bar_buffer_value); 1326 | } 1327 | 1328 | // write bar and get slot to it 1329 | const bar_slot = (try root_cursor.writePath(void, &.{ 1330 | .array_list_init, 1331 | .array_list_append, 1332 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1333 | .{ .hash_map_init = .{} }, 1334 | .{ .hash_map_get = .{ .value = bar_key } }, 1335 | .{ .write = .{ .bytes = "bar" } }, 1336 | })).slot_ptr.slot; 1337 | 1338 | // overwrite foo -> bar using the bar slot 1339 | _ = try root_cursor.writePath(void, &.{ 1340 | .array_list_init, 1341 | .array_list_append, 1342 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1343 | .{ .hash_map_init = .{} }, 1344 | .{ .hash_map_get = .{ .value = foo_key } }, 1345 | .{ .write = .{ .slot = bar_slot } }, 1346 | }); 1347 | const bar_cursor = (try root_cursor.readPath(void, &.{ 1348 | .{ .array_list_get = -1 }, 1349 | .{ .hash_map_get = .{ .value = foo_key } }, 1350 | })).?; 1351 | const bar_value = try bar_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1352 | defer allocator.free(bar_value); 1353 | try std.testing.expectEqualStrings("bar", bar_value); 1354 | 1355 | // can still read the old value 1356 | const baz_cursor = (try root_cursor.readPath(void, &.{ 1357 | .{ .array_list_get = -2 }, 1358 | .{ .hash_map_get = .{ .value = foo_key } }, 1359 | })).?; 1360 | const baz_value = try baz_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1361 | defer allocator.free(baz_value); 1362 | try std.testing.expectEqualStrings("baz", baz_value); 1363 | 1364 | // key not found 1365 | const not_found_key = hashInt("this doesn't exist"); 1366 | try std.testing.expectEqual(null, try root_cursor.readPath(void, &.{ 1367 | .{ .array_list_get = -1 }, 1368 | .{ .hash_map_get = .{ .value = not_found_key } }, 1369 | })); 1370 | 1371 | // write key that conflicts with foo the first two bytes 1372 | const small_conflict_mask: u64 = 0b1111_1111; 1373 | const small_conflict_key = (hashInt("small conflict") & ~small_conflict_mask) | (foo_key & small_conflict_mask); 1374 | _ = try root_cursor.writePath(void, &.{ 1375 | .array_list_init, 1376 | .array_list_append, 1377 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1378 | .{ .hash_map_init = .{} }, 1379 | .{ .hash_map_get = .{ .value = small_conflict_key } }, 1380 | .{ .write = .{ .bytes = "small" } }, 1381 | }); 1382 | 1383 | // write key that conflicts with foo the first four bytes 1384 | const conflict_mask: u64 = 0b1111_1111_1111_1111; 1385 | const conflict_key = (hashInt("conflict") & ~conflict_mask) | (foo_key & conflict_mask); 1386 | _ = try root_cursor.writePath(void, &.{ 1387 | .array_list_init, 1388 | .array_list_append, 1389 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1390 | .{ .hash_map_init = .{} }, 1391 | .{ .hash_map_get = .{ .value = conflict_key } }, 1392 | .{ .write = .{ .bytes = "hello" } }, 1393 | }); 1394 | 1395 | // read conflicting key 1396 | const hello_cursor = (try root_cursor.readPath(void, &.{ 1397 | .{ .array_list_get = -1 }, 1398 | .{ .hash_map_get = .{ .value = conflict_key } }, 1399 | })).?; 1400 | const hello_value = try hello_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1401 | defer allocator.free(hello_value); 1402 | try std.testing.expectEqualStrings("hello", hello_value); 1403 | 1404 | // we can still read foo 1405 | const bar_cursor2 = (try root_cursor.readPath(void, &.{ 1406 | .{ .array_list_get = -1 }, 1407 | .{ .hash_map_get = .{ .value = foo_key } }, 1408 | })).?; 1409 | const bar_value2 = try bar_cursor2.readBytesAlloc(allocator, MAX_READ_BYTES); 1410 | defer allocator.free(bar_value2); 1411 | try std.testing.expectEqualStrings("bar", bar_value2); 1412 | 1413 | // overwrite conflicting key 1414 | _ = try root_cursor.writePath(void, &.{ 1415 | .array_list_init, 1416 | .array_list_append, 1417 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1418 | .{ .hash_map_init = .{} }, 1419 | .{ .hash_map_get = .{ .value = conflict_key } }, 1420 | .{ .write = .{ .bytes = "goodbye" } }, 1421 | }); 1422 | const goodbye_cursor = (try root_cursor.readPath(void, &.{ 1423 | .{ .array_list_get = -1 }, 1424 | .{ .hash_map_get = .{ .value = conflict_key } }, 1425 | })).?; 1426 | const goodbye_value = try goodbye_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1427 | defer allocator.free(goodbye_value); 1428 | try std.testing.expectEqualStrings("goodbye", goodbye_value); 1429 | 1430 | // we can still read the old conflicting key 1431 | const hello_cursor2 = (try root_cursor.readPath(void, &.{ 1432 | .{ .array_list_get = -2 }, 1433 | .{ .hash_map_get = .{ .value = conflict_key } }, 1434 | })).?; 1435 | const hello_value2 = try hello_cursor2.readBytesAlloc(allocator, MAX_READ_BYTES); 1436 | defer allocator.free(hello_value2); 1437 | try std.testing.expectEqualStrings("hello", hello_value2); 1438 | 1439 | // remove the conflicting keys 1440 | { 1441 | // foo's slot is an .index slot due to the conflict 1442 | { 1443 | const map_cursor = (try root_cursor.readPath(void, &.{ 1444 | .{ .array_list_get = -1 }, 1445 | })).?; 1446 | const index_pos = map_cursor.slot().value; 1447 | try std.testing.expectEqual(.hash_map, map_cursor.slot().tag); 1448 | 1449 | const reader = db.core.reader(); 1450 | const slot_size: u64 = @bitSizeOf(xitdb.Slot) / 8; 1451 | 1452 | const i: u4 = @intCast(foo_key & xitdb.MASK); 1453 | const slot_pos = index_pos + (slot_size * i); 1454 | try db.core.seekTo(slot_pos); 1455 | const slot: xitdb.Slot = @bitCast(try reader.readInt(u72, .big)); 1456 | 1457 | try std.testing.expectEqual(.index, slot.tag); 1458 | } 1459 | 1460 | // remove the small conflict key 1461 | _ = try root_cursor.writePath(void, &.{ 1462 | .array_list_init, 1463 | .array_list_append, 1464 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1465 | .{ .hash_map_init = .{} }, 1466 | .{ .hash_map_remove = small_conflict_key }, 1467 | }); 1468 | 1469 | // the conflict key still exists in history 1470 | try std.testing.expect(null != try root_cursor.readPath(void, &.{ 1471 | .{ .array_list_get = -2 }, 1472 | .{ .hash_map_get = .{ .value = small_conflict_key } }, 1473 | })); 1474 | 1475 | // the conflict key doesn't exist in the latest moment 1476 | try std.testing.expectEqual(null, try root_cursor.readPath(void, &.{ 1477 | .{ .array_list_get = -1 }, 1478 | .{ .hash_map_get = .{ .value = small_conflict_key } }, 1479 | })); 1480 | 1481 | // the other conflict key still exists 1482 | try std.testing.expect(null != try root_cursor.readPath(void, &.{ 1483 | .{ .array_list_get = -1 }, 1484 | .{ .hash_map_get = .{ .value = conflict_key } }, 1485 | })); 1486 | 1487 | // foo's slot is still an .index slot due to the other conflicting key 1488 | { 1489 | const map_cursor = (try root_cursor.readPath(void, &.{ 1490 | .{ .array_list_get = -1 }, 1491 | })).?; 1492 | const index_pos = map_cursor.slot().value; 1493 | try std.testing.expectEqual(.hash_map, map_cursor.slot().tag); 1494 | 1495 | const reader = db.core.reader(); 1496 | const slot_size: u64 = @bitSizeOf(xitdb.Slot) / 8; 1497 | 1498 | const i: u4 = @intCast(foo_key & xitdb.MASK); 1499 | const slot_pos = index_pos + (slot_size * i); 1500 | try db.core.seekTo(slot_pos); 1501 | const slot: xitdb.Slot = @bitCast(try reader.readInt(u72, .big)); 1502 | 1503 | try std.testing.expectEqual(.index, slot.tag); 1504 | } 1505 | 1506 | // remove the conflict key 1507 | _ = try root_cursor.writePath(void, &.{ 1508 | .array_list_init, 1509 | .array_list_append, 1510 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1511 | .{ .hash_map_init = .{} }, 1512 | .{ .hash_map_remove = conflict_key }, 1513 | }); 1514 | 1515 | // the conflict keys don't exist in the latest moment 1516 | try std.testing.expectEqual(null, try root_cursor.readPath(void, &.{ 1517 | .{ .array_list_get = -1 }, 1518 | .{ .hash_map_get = .{ .value = small_conflict_key } }, 1519 | })); 1520 | try std.testing.expectEqual(null, try root_cursor.readPath(void, &.{ 1521 | .{ .array_list_get = -1 }, 1522 | .{ .hash_map_get = .{ .value = conflict_key } }, 1523 | })); 1524 | 1525 | // foo's slot is now a .kv_pair slot, because the branch was shortened 1526 | { 1527 | const map_cursor = (try root_cursor.readPath(void, &.{ 1528 | .{ .array_list_get = -1 }, 1529 | })).?; 1530 | const index_pos = map_cursor.slot().value; 1531 | try std.testing.expectEqual(.hash_map, map_cursor.slot().tag); 1532 | 1533 | const reader = db.core.reader(); 1534 | const slot_size: u64 = @bitSizeOf(xitdb.Slot) / 8; 1535 | 1536 | const i: u4 = @intCast(foo_key & xitdb.MASK); 1537 | const slot_pos = index_pos + (slot_size * i); 1538 | try db.core.seekTo(slot_pos); 1539 | const slot: xitdb.Slot = @bitCast(try reader.readInt(u72, .big)); 1540 | 1541 | try std.testing.expectEqual(.kv_pair, slot.tag); 1542 | } 1543 | } 1544 | 1545 | { 1546 | // overwrite foo with a uint 1547 | _ = try root_cursor.writePath(void, &.{ 1548 | .array_list_init, 1549 | .array_list_append, 1550 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1551 | .{ .hash_map_init = .{} }, 1552 | .{ .hash_map_get = .{ .value = foo_key } }, 1553 | .{ .write = .{ .uint = 42 } }, 1554 | }); 1555 | 1556 | // read foo 1557 | const uint_value = (try root_cursor.readPath(void, &.{ 1558 | .{ .array_list_get = -1 }, 1559 | .{ .hash_map_get = .{ .value = foo_key } }, 1560 | })).?.readUint(); 1561 | try std.testing.expectEqual(42, uint_value); 1562 | } 1563 | 1564 | { 1565 | // overwrite foo with an int 1566 | _ = try root_cursor.writePath(void, &.{ 1567 | .array_list_init, 1568 | .array_list_append, 1569 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1570 | .{ .hash_map_init = .{} }, 1571 | .{ .hash_map_get = .{ .value = foo_key } }, 1572 | .{ .write = .{ .int = -42 } }, 1573 | }); 1574 | 1575 | // read foo 1576 | const int_value = (try root_cursor.readPath(void, &.{ 1577 | .{ .array_list_get = -1 }, 1578 | .{ .hash_map_get = .{ .value = foo_key } }, 1579 | })).?.readInt(); 1580 | try std.testing.expectEqual(-42, int_value); 1581 | } 1582 | 1583 | { 1584 | // overwrite foo with a float 1585 | _ = try root_cursor.writePath(void, &.{ 1586 | .array_list_init, 1587 | .array_list_append, 1588 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1589 | .{ .hash_map_init = .{} }, 1590 | .{ .hash_map_get = .{ .value = foo_key } }, 1591 | .{ .write = .{ .float = 42.5 } }, 1592 | }); 1593 | 1594 | // read foo 1595 | const float_value = (try root_cursor.readPath(void, &.{ 1596 | .{ .array_list_get = -1 }, 1597 | .{ .hash_map_get = .{ .value = foo_key } }, 1598 | })).?.readFloat(); 1599 | try std.testing.expectEqual(42.5, float_value); 1600 | } 1601 | 1602 | // remove foo 1603 | _ = try root_cursor.writePath(void, &.{ 1604 | .array_list_init, 1605 | .array_list_append, 1606 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1607 | .{ .hash_map_init = .{} }, 1608 | .{ .hash_map_remove = foo_key }, 1609 | }); 1610 | 1611 | // remove key that does not exist 1612 | try std.testing.expectError(error.KeyNotFound, root_cursor.writePath(void, &.{ 1613 | .array_list_init, 1614 | .array_list_append, 1615 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1616 | .{ .hash_map_init = .{} }, 1617 | .{ .hash_map_remove = hashInt("doesn't exist") }, 1618 | })); 1619 | 1620 | // make sure foo doesn't exist anymore 1621 | try std.testing.expectEqual(null, try root_cursor.readPath(void, &.{ 1622 | .{ .array_list_get = -1 }, 1623 | .{ .hash_map_get = .{ .value = foo_key } }, 1624 | })); 1625 | 1626 | // non-top-level list 1627 | { 1628 | // write apple 1629 | _ = try root_cursor.writePath(void, &.{ 1630 | .array_list_init, 1631 | .array_list_append, 1632 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1633 | .{ .hash_map_init = .{} }, 1634 | .{ .hash_map_get = .{ .value = hashInt("fruits") } }, 1635 | .array_list_init, 1636 | .array_list_append, 1637 | .{ .write = .{ .bytes = "apple" } }, 1638 | }); 1639 | 1640 | // read apple 1641 | const apple_cursor = (try root_cursor.readPath(void, &.{ 1642 | .{ .array_list_get = -1 }, 1643 | .{ .hash_map_get = .{ .value = hashInt("fruits") } }, 1644 | .{ .array_list_get = -1 }, 1645 | })).?; 1646 | const apple_value = try apple_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1647 | defer allocator.free(apple_value); 1648 | try std.testing.expectEqualStrings("apple", apple_value); 1649 | 1650 | // write banana 1651 | _ = try root_cursor.writePath(void, &.{ 1652 | .array_list_init, 1653 | .array_list_append, 1654 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1655 | .{ .hash_map_init = .{} }, 1656 | .{ .hash_map_get = .{ .value = hashInt("fruits") } }, 1657 | .array_list_init, 1658 | .array_list_append, 1659 | .{ .write = .{ .bytes = "banana" } }, 1660 | }); 1661 | 1662 | // read banana 1663 | const banana_cursor = (try root_cursor.readPath(void, &.{ 1664 | .{ .array_list_get = -1 }, 1665 | .{ .hash_map_get = .{ .value = hashInt("fruits") } }, 1666 | .{ .array_list_get = -1 }, 1667 | })).?; 1668 | const banana_value = try banana_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1669 | defer allocator.free(banana_value); 1670 | try std.testing.expectEqualStrings("banana", banana_value); 1671 | 1672 | // can't read banana in older array_list 1673 | try std.testing.expectEqual(null, try root_cursor.readPath(void, &.{ 1674 | .{ .array_list_get = -2 }, 1675 | .{ .hash_map_get = .{ .value = hashInt("fruits") } }, 1676 | .{ .array_list_get = 1 }, 1677 | })); 1678 | 1679 | // write pear 1680 | _ = try root_cursor.writePath(void, &.{ 1681 | .array_list_init, 1682 | .array_list_append, 1683 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1684 | .{ .hash_map_init = .{} }, 1685 | .{ .hash_map_get = .{ .value = hashInt("fruits") } }, 1686 | .array_list_init, 1687 | .array_list_append, 1688 | .{ .write = .{ .bytes = "pear" } }, 1689 | }); 1690 | 1691 | // write grape 1692 | _ = try root_cursor.writePath(void, &.{ 1693 | .array_list_init, 1694 | .array_list_append, 1695 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1696 | .{ .hash_map_init = .{} }, 1697 | .{ .hash_map_get = .{ .value = hashInt("fruits") } }, 1698 | .array_list_init, 1699 | .array_list_append, 1700 | .{ .write = .{ .bytes = "grape" } }, 1701 | }); 1702 | 1703 | // read pear 1704 | const pear_cursor = (try root_cursor.readPath(void, &.{ 1705 | .{ .array_list_get = -1 }, 1706 | .{ .hash_map_get = .{ .value = hashInt("fruits") } }, 1707 | .{ .array_list_get = -2 }, 1708 | })).?; 1709 | const pear_value = try pear_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1710 | defer allocator.free(pear_value); 1711 | try std.testing.expectEqualStrings("pear", pear_value); 1712 | 1713 | // read grape 1714 | const grape_cursor = (try root_cursor.readPath(void, &.{ 1715 | .{ .array_list_get = -1 }, 1716 | .{ .hash_map_get = .{ .value = hashInt("fruits") } }, 1717 | .{ .array_list_get = -1 }, 1718 | })).?; 1719 | const grape_value = try grape_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1720 | defer allocator.free(grape_value); 1721 | try std.testing.expectEqualStrings("grape", grape_value); 1722 | } 1723 | } 1724 | 1725 | // append to top-level array_list many times, filling up the array_list until a root overflow occurs 1726 | { 1727 | try clearStorage(db_kind, init_opts); 1728 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 1729 | var root_cursor = db.rootCursor(); 1730 | 1731 | const wat_key = hashInt("wat"); 1732 | for (0..xitdb.SLOT_COUNT + 1) |i| { 1733 | const value = try std.fmt.allocPrint(allocator, "wat{}", .{i}); 1734 | defer allocator.free(value); 1735 | _ = try root_cursor.writePath(void, &.{ 1736 | .array_list_init, 1737 | .array_list_append, 1738 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1739 | .{ .hash_map_init = .{} }, 1740 | .{ .hash_map_get = .{ .value = wat_key } }, 1741 | .{ .write = .{ .bytes = value } }, 1742 | }); 1743 | } 1744 | 1745 | for (0..xitdb.SLOT_COUNT + 1) |i| { 1746 | const value = try std.fmt.allocPrint(allocator, "wat{}", .{i}); 1747 | defer allocator.free(value); 1748 | const cursor = (try root_cursor.readPath(void, &.{ 1749 | .{ .array_list_get = i }, 1750 | .{ .hash_map_get = .{ .value = wat_key } }, 1751 | })).?; 1752 | const value2 = try cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1753 | defer allocator.free(value2); 1754 | try std.testing.expectEqualStrings(value, value2); 1755 | } 1756 | 1757 | // add more slots to cause a new index block to be created. 1758 | // a new index block will be created when i == 32 (the 33rd append). 1759 | // during that transaction, return an error so the transaction is 1760 | // cancelled, causing truncation to happen. this test ensures that 1761 | // the new index block is NOT truncated. this is prevented by updating 1762 | // the file size in the header immediately after making a new index block. 1763 | // see `readArrayListSlot` for more. 1764 | { 1765 | for (xitdb.SLOT_COUNT + 1..xitdb.SLOT_COUNT * 2 + 1) |i| { 1766 | const value = try std.fmt.allocPrint(allocator, "wat{}", .{i}); 1767 | defer allocator.free(value); 1768 | 1769 | const Ctx = struct { 1770 | i: usize, 1771 | pub fn run(self: @This(), _: *xitdb.Database(db_kind, HashInt).Cursor(.read_write)) !void { 1772 | if (self.i == 32) { 1773 | return error.Fail; 1774 | } 1775 | } 1776 | }; 1777 | _ = root_cursor.writePath(Ctx, &.{ 1778 | .array_list_init, 1779 | .array_list_append, 1780 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1781 | .{ .hash_map_init = .{} }, 1782 | .{ .hash_map_get = .{ .value = wat_key } }, 1783 | .{ .write = .{ .bytes = value } }, 1784 | .{ .ctx = .{ .i = i } }, 1785 | }) catch |err| switch (err) { 1786 | error.Fail => break, 1787 | else => |e| return e, 1788 | }; 1789 | } else { 1790 | return error.ExpectedFail; 1791 | } 1792 | 1793 | // try another append to make sure we still can. 1794 | // if truncation destroyed the index block, this would fail. 1795 | _ = try root_cursor.writePath(void, &.{ 1796 | .array_list_init, 1797 | .array_list_append, 1798 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1799 | .{ .hash_map_init = .{} }, 1800 | .{ .hash_map_get = .{ .value = wat_key } }, 1801 | .{ .write = .{ .bytes = "wat32" } }, 1802 | }); 1803 | } 1804 | 1805 | // slice so it contains exactly SLOT_COUNT, 1806 | // so we have the old root again 1807 | _ = try root_cursor.writePath(void, &.{ 1808 | .array_list_init, 1809 | .{ .array_list_slice = .{ .size = xitdb.SLOT_COUNT } }, 1810 | }); 1811 | 1812 | // we can iterate over the remaining slots 1813 | for (0..xitdb.SLOT_COUNT) |i| { 1814 | const value = try std.fmt.allocPrint(allocator, "wat{}", .{i}); 1815 | defer allocator.free(value); 1816 | const cursor = (try root_cursor.readPath(void, &.{ 1817 | .{ .array_list_get = i }, 1818 | .{ .hash_map_get = .{ .value = wat_key } }, 1819 | })).?; 1820 | const value2 = try cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1821 | defer allocator.free(value2); 1822 | try std.testing.expectEqualStrings(value, value2); 1823 | } 1824 | 1825 | // but we can't get the value that we sliced out of the array list 1826 | try std.testing.expectEqual(null, root_cursor.readPath(void, &.{ 1827 | .{ .array_list_get = xitdb.SLOT_COUNT + 1 }, 1828 | })); 1829 | } 1830 | 1831 | // append to inner array_list many times, filling up the array_list until a root overflow occurs 1832 | { 1833 | try clearStorage(db_kind, init_opts); 1834 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 1835 | var root_cursor = db.rootCursor(); 1836 | 1837 | for (0..xitdb.SLOT_COUNT + 1) |i| { 1838 | const value = try std.fmt.allocPrint(allocator, "wat{}", .{i}); 1839 | defer allocator.free(value); 1840 | _ = try root_cursor.writePath(void, &.{ 1841 | .array_list_init, 1842 | .array_list_append, 1843 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1844 | .array_list_init, 1845 | .array_list_append, 1846 | .{ .write = .{ .bytes = value } }, 1847 | }); 1848 | } 1849 | 1850 | for (0..xitdb.SLOT_COUNT + 1) |i| { 1851 | const value = try std.fmt.allocPrint(allocator, "wat{}", .{i}); 1852 | defer allocator.free(value); 1853 | const cursor = (try root_cursor.readPath(void, &.{ 1854 | .{ .array_list_get = -1 }, 1855 | .{ .array_list_get = i }, 1856 | })).?; 1857 | const value2 = try cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1858 | defer allocator.free(value2); 1859 | try std.testing.expectEqualStrings(value, value2); 1860 | } 1861 | 1862 | // slice inner array list so it contains exactly SLOT_COUNT, 1863 | // so we have the old root again 1864 | _ = try root_cursor.writePath(void, &.{ 1865 | .array_list_init, 1866 | .{ .array_list_get = -1 }, 1867 | .array_list_init, 1868 | .{ .array_list_slice = .{ .size = xitdb.SLOT_COUNT } }, 1869 | }); 1870 | 1871 | // we can iterate over the remaining slots 1872 | for (0..xitdb.SLOT_COUNT) |i| { 1873 | const value = try std.fmt.allocPrint(allocator, "wat{}", .{i}); 1874 | defer allocator.free(value); 1875 | const cursor = (try root_cursor.readPath(void, &.{ 1876 | .{ .array_list_get = -1 }, 1877 | .{ .array_list_get = i }, 1878 | })).?; 1879 | const value2 = try cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1880 | defer allocator.free(value2); 1881 | try std.testing.expectEqualStrings(value, value2); 1882 | } 1883 | 1884 | // but we can't get the value that we sliced out of the array list 1885 | try std.testing.expectEqual(null, root_cursor.readPath(void, &.{ 1886 | .{ .array_list_get = -1 }, 1887 | .{ .array_list_get = xitdb.SLOT_COUNT + 1 }, 1888 | })); 1889 | 1890 | // overwrite last value with hello 1891 | _ = try root_cursor.writePath(void, &.{ 1892 | .array_list_init, 1893 | .array_list_append, 1894 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1895 | .array_list_init, 1896 | .{ .array_list_get = -1 }, 1897 | .{ .write = .{ .bytes = "hello" } }, 1898 | }); 1899 | 1900 | // read last value 1901 | { 1902 | const cursor = (try root_cursor.readPath(void, &.{ 1903 | .{ .array_list_get = -1 }, 1904 | .{ .array_list_get = -1 }, 1905 | })).?; 1906 | const value = try cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1907 | defer allocator.free(value); 1908 | try std.testing.expectEqualStrings("hello", value); 1909 | } 1910 | 1911 | // overwrite last value with goodbye 1912 | _ = try root_cursor.writePath(void, &.{ 1913 | .array_list_init, 1914 | .array_list_append, 1915 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1916 | .array_list_init, 1917 | .{ .array_list_get = -1 }, 1918 | .{ .write = .{ .bytes = "goodbye" } }, 1919 | }); 1920 | 1921 | // read last value 1922 | { 1923 | const cursor = (try root_cursor.readPath(void, &.{ 1924 | .{ .array_list_get = -1 }, 1925 | .{ .array_list_get = -1 }, 1926 | })).?; 1927 | const value = try cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1928 | defer allocator.free(value); 1929 | try std.testing.expectEqualStrings("goodbye", value); 1930 | } 1931 | 1932 | // previous last value is still hello 1933 | { 1934 | const cursor = (try root_cursor.readPath(void, &.{ 1935 | .{ .array_list_get = -2 }, 1936 | .{ .array_list_get = -1 }, 1937 | })).?; 1938 | const value = try cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1939 | defer allocator.free(value); 1940 | try std.testing.expectEqualStrings("hello", value); 1941 | } 1942 | } 1943 | 1944 | // iterate over inner array_list 1945 | { 1946 | try clearStorage(db_kind, init_opts); 1947 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 1948 | var root_cursor = db.rootCursor(); 1949 | 1950 | // add wats 1951 | for (0..10) |i| { 1952 | const value = try std.fmt.allocPrint(allocator, "wat{}", .{i}); 1953 | defer allocator.free(value); 1954 | _ = try root_cursor.writePath(void, &.{ 1955 | .array_list_init, 1956 | .array_list_append, 1957 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 1958 | .array_list_init, 1959 | .array_list_append, 1960 | .{ .write = .{ .bytes = value } }, 1961 | }); 1962 | 1963 | const cursor = (try root_cursor.readPath(void, &.{ 1964 | .{ .array_list_get = -1 }, 1965 | .{ .array_list_get = -1 }, 1966 | })).?; 1967 | const value2 = try cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1968 | defer allocator.free(value2); 1969 | try std.testing.expectEqualStrings(value, value2); 1970 | } 1971 | 1972 | // iterate over array_list 1973 | { 1974 | var inner_cursor = (try root_cursor.readPath(void, &.{ 1975 | .{ .array_list_get = -1 }, 1976 | })).?; 1977 | var iter = try inner_cursor.iterator(); 1978 | var i: u64 = 0; 1979 | while (try iter.next()) |*next_cursor| { 1980 | const value = try std.fmt.allocPrint(allocator, "wat{}", .{i}); 1981 | defer allocator.free(value); 1982 | const value2 = try next_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 1983 | defer allocator.free(value2); 1984 | try std.testing.expectEqualStrings(value, value2); 1985 | i += 1; 1986 | } 1987 | try std.testing.expectEqual(10, i); 1988 | } 1989 | 1990 | // set first slot to .none and make sure iteration still works. 1991 | // this validates that it correctly returns .none slots if 1992 | // their flag is set, rather than skipping over them. 1993 | { 1994 | _ = try root_cursor.writePath(void, &.{ 1995 | .array_list_init, 1996 | .{ .array_list_get = -1 }, 1997 | .array_list_init, 1998 | .{ .array_list_get = 0 }, 1999 | .{ .write = .{ .slot = null } }, 2000 | }); 2001 | var inner_cursor = (try root_cursor.readPath(void, &.{ 2002 | .{ .array_list_get = -1 }, 2003 | })).?; 2004 | var iter = try inner_cursor.iterator(); 2005 | var i: u64 = 0; 2006 | while (try iter.next()) |_| { 2007 | i += 1; 2008 | } 2009 | try std.testing.expectEqual(10, i); 2010 | } 2011 | 2012 | // get list slot 2013 | const list_cursor = (try root_cursor.readPath(void, &.{ 2014 | .{ .array_list_get = -1 }, 2015 | })).?; 2016 | try std.testing.expectEqual(10, list_cursor.count()); 2017 | } 2018 | 2019 | // iterate over inner hash_map 2020 | { 2021 | try clearStorage(db_kind, init_opts); 2022 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 2023 | var root_cursor = db.rootCursor(); 2024 | 2025 | // add wats 2026 | for (0..10) |i| { 2027 | const value = try std.fmt.allocPrint(allocator, "wat{}", .{i}); 2028 | defer allocator.free(value); 2029 | const wat_key = hashInt(value); 2030 | _ = try root_cursor.writePath(void, &.{ 2031 | .array_list_init, 2032 | .array_list_append, 2033 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2034 | .{ .hash_map_init = .{} }, 2035 | .{ .hash_map_get = .{ .value = wat_key } }, 2036 | .{ .write = .{ .bytes = value } }, 2037 | }); 2038 | 2039 | const cursor = (try root_cursor.readPath(void, &.{ 2040 | .{ .array_list_get = -1 }, 2041 | .{ .hash_map_get = .{ .value = wat_key } }, 2042 | })).?; 2043 | const value2 = try cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 2044 | defer allocator.free(value2); 2045 | try std.testing.expectEqualStrings(value, value2); 2046 | } 2047 | 2048 | // add foo 2049 | const foo_key = hashInt("foo"); 2050 | _ = try root_cursor.writePath(void, &.{ 2051 | .array_list_init, 2052 | .array_list_append, 2053 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2054 | .{ .hash_map_init = .{} }, 2055 | .{ .hash_map_get = .{ .key = foo_key } }, 2056 | .{ .write = .{ .bytes = "foo" } }, 2057 | }); 2058 | _ = try root_cursor.writePath(void, &.{ 2059 | .array_list_init, 2060 | .array_list_append, 2061 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2062 | .{ .hash_map_init = .{} }, 2063 | .{ .hash_map_get = .{ .value = foo_key } }, 2064 | .{ .write = .{ .uint = 42 } }, 2065 | }); 2066 | 2067 | // remove a wat 2068 | _ = try root_cursor.writePath(void, &.{ 2069 | .array_list_init, 2070 | .array_list_append, 2071 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2072 | .{ .hash_map_init = .{} }, 2073 | .{ .hash_map_remove = hashInt("wat0") }, 2074 | }); 2075 | 2076 | // iterate over hash_map 2077 | { 2078 | var inner_cursor = (try root_cursor.readPath(void, &.{ 2079 | .{ .array_list_get = -1 }, 2080 | })).?; 2081 | var iter = try inner_cursor.iterator(); 2082 | var i: u64 = 0; 2083 | while (try iter.next()) |kv_pair_cursor| { 2084 | const kv_pair = try kv_pair_cursor.readKeyValuePair(); 2085 | if (kv_pair.hash == foo_key) { 2086 | const key = try kv_pair.key_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 2087 | defer allocator.free(key); 2088 | try std.testing.expectEqualStrings("foo", key); 2089 | try std.testing.expectEqual(42, kv_pair.value_cursor.slot_ptr.slot.value); 2090 | } else { 2091 | const value = try kv_pair.value_cursor.readBytesAlloc(allocator, MAX_READ_BYTES); 2092 | defer allocator.free(value); 2093 | try std.testing.expectEqual(kv_pair.hash, hashInt(value)); 2094 | } 2095 | i += 1; 2096 | } 2097 | try std.testing.expectEqual(10, i); 2098 | } 2099 | 2100 | // iterate over hash_map with writeable cursor 2101 | { 2102 | var inner_cursor = try root_cursor.writePath(void, &.{ 2103 | .array_list_init, 2104 | .array_list_append, 2105 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2106 | }); 2107 | var iter = try inner_cursor.iterator(); 2108 | var i: u64 = 0; 2109 | while (try iter.next()) |kv_pair_cursor| { 2110 | var kv_pair = try kv_pair_cursor.readKeyValuePair(); 2111 | if (kv_pair.hash == foo_key) { 2112 | try kv_pair.key_cursor.write(.{ .bytes = "bar" }); 2113 | } 2114 | i += 1; 2115 | } 2116 | try std.testing.expectEqual(10, i); 2117 | } 2118 | } 2119 | 2120 | { 2121 | // slice linked_array_list 2122 | try testSlice(allocator, db_kind, init_opts, xitdb.SLOT_COUNT * 5 + 1, 10, 5); 2123 | try testSlice(allocator, db_kind, init_opts, xitdb.SLOT_COUNT * 5 + 1, 0, xitdb.SLOT_COUNT * 2); 2124 | try testSlice(allocator, db_kind, init_opts, xitdb.SLOT_COUNT * 5, xitdb.SLOT_COUNT * 3, xitdb.SLOT_COUNT); 2125 | try testSlice(allocator, db_kind, init_opts, xitdb.SLOT_COUNT * 5, xitdb.SLOT_COUNT * 3, xitdb.SLOT_COUNT * 2); 2126 | try testSlice(allocator, db_kind, init_opts, xitdb.SLOT_COUNT * 2, 10, xitdb.SLOT_COUNT); 2127 | try testSlice(allocator, db_kind, init_opts, 2, 0, 2); 2128 | try testSlice(allocator, db_kind, init_opts, 2, 1, 1); 2129 | try testSlice(allocator, db_kind, init_opts, 1, 0, 0); 2130 | 2131 | // concat linked_array_list 2132 | try testConcat(allocator, db_kind, init_opts, xitdb.SLOT_COUNT * 5 + 1, xitdb.SLOT_COUNT + 1); 2133 | try testConcat(allocator, db_kind, init_opts, xitdb.SLOT_COUNT, xitdb.SLOT_COUNT); 2134 | try testConcat(allocator, db_kind, init_opts, 1, 1); 2135 | try testConcat(allocator, db_kind, init_opts, 0, 0); 2136 | 2137 | // insert linked_array_list 2138 | try testInsertAndRemove(allocator, db_kind, init_opts, 1, 0); 2139 | try testInsertAndRemove(allocator, db_kind, init_opts, 10, 0); 2140 | try testInsertAndRemove(allocator, db_kind, init_opts, 10, 5); 2141 | try testInsertAndRemove(allocator, db_kind, init_opts, 10, 9); 2142 | try testInsertAndRemove(allocator, db_kind, init_opts, xitdb.SLOT_COUNT * 5, xitdb.SLOT_COUNT * 2); 2143 | } 2144 | 2145 | // concat linked_array_list multiple times 2146 | { 2147 | try clearStorage(db_kind, init_opts); 2148 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 2149 | var root_cursor = db.rootCursor(); 2150 | 2151 | const Ctx = struct { 2152 | allocator: std.mem.Allocator, 2153 | 2154 | pub fn run(self: @This(), cursor: *xitdb.Database(db_kind, HashInt).Cursor(.read_write)) !void { 2155 | var values = std.ArrayList(u64).init(self.allocator); 2156 | defer values.deinit(); 2157 | 2158 | // create list 2159 | for (0..xitdb.SLOT_COUNT + 1) |i| { 2160 | const n = i * 2; 2161 | try values.append(n); 2162 | _ = try cursor.writePath(void, &.{ 2163 | .{ .hash_map_get = .{ .value = hashInt("even") } }, 2164 | .linked_array_list_init, 2165 | .linked_array_list_append, 2166 | .{ .write = .{ .uint = n } }, 2167 | }); 2168 | } 2169 | 2170 | // get list slot 2171 | const even_list_cursor = (try cursor.readPath(void, &.{ 2172 | .{ .hash_map_get = .{ .value = hashInt("even") } }, 2173 | })).?; 2174 | try std.testing.expectEqual(xitdb.SLOT_COUNT + 1, even_list_cursor.count()); 2175 | 2176 | // iterate over list 2177 | var inner_cursor = (try cursor.readPath(void, &.{ 2178 | .{ .hash_map_get = .{ .value = hashInt("even") } }, 2179 | })).?; 2180 | var iter = try inner_cursor.iterator(); 2181 | var i: u64 = 0; 2182 | while (try iter.next()) |_| { 2183 | i += 1; 2184 | } 2185 | try std.testing.expectEqual(xitdb.SLOT_COUNT + 1, i); 2186 | 2187 | // concat the list with itself multiple times. 2188 | // since each list has 17 items, each concat 2189 | // will create a gap, causing a root overflow 2190 | // before a normal array list would've. 2191 | var combo_list_cursor = try cursor.writePath(void, &.{ 2192 | .{ .hash_map_get = .{ .value = hashInt("combo") } }, 2193 | .{ .write = .{ .slot = even_list_cursor.slot_ptr.slot } }, 2194 | .linked_array_list_init, 2195 | }); 2196 | for (0..16) |_| { 2197 | combo_list_cursor = try combo_list_cursor.writePath(void, &.{ 2198 | .{ .linked_array_list_concat = .{ .list = even_list_cursor.slot_ptr.slot } }, 2199 | }); 2200 | } 2201 | 2202 | // append to the new list 2203 | _ = try cursor.writePath(void, &.{ 2204 | .{ .hash_map_get = .{ .value = hashInt("combo") } }, 2205 | .linked_array_list_append, 2206 | .{ .write = .{ .uint = 3 } }, 2207 | }); 2208 | 2209 | // read the new value from the list 2210 | try std.testing.expectEqual(3, (try cursor.readPath(void, &.{ 2211 | .{ .hash_map_get = .{ .value = hashInt("combo") } }, 2212 | .{ .linked_array_list_get = -1 }, 2213 | })).?.slot_ptr.slot.value); 2214 | 2215 | // append more to the new list 2216 | for (0..500) |_| { 2217 | _ = try cursor.writePath(void, &.{ 2218 | .{ .hash_map_get = .{ .value = hashInt("combo") } }, 2219 | .linked_array_list_append, 2220 | .{ .write = .{ .uint = 1 } }, 2221 | }); 2222 | } 2223 | } 2224 | }; 2225 | _ = try root_cursor.writePath(Ctx, &.{ 2226 | .array_list_init, 2227 | .array_list_append, 2228 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2229 | .{ .hash_map_init = .{} }, 2230 | .{ .ctx = .{ .allocator = allocator } }, 2231 | }); 2232 | } 2233 | 2234 | // append items to linked_array_list without setting their value 2235 | { 2236 | try clearStorage(db_kind, init_opts); 2237 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 2238 | var root_cursor = db.rootCursor(); 2239 | 2240 | // appending without setting any value should work 2241 | for (0..8) |_| { 2242 | _ = try root_cursor.writePath(void, &.{ 2243 | .array_list_init, 2244 | .array_list_append, 2245 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2246 | .linked_array_list_init, 2247 | .linked_array_list_append, 2248 | }); 2249 | } 2250 | 2251 | // explicitly writing a null slot should also work 2252 | for (0..8) |_| { 2253 | _ = try root_cursor.writePath(void, &.{ 2254 | .array_list_init, 2255 | .array_list_append, 2256 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2257 | .linked_array_list_init, 2258 | .linked_array_list_append, 2259 | .{ .write = .{ .slot = null } }, 2260 | }); 2261 | } 2262 | } 2263 | 2264 | // insert at beginning of linked_array_list many times 2265 | { 2266 | try clearStorage(db_kind, init_opts); 2267 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 2268 | var root_cursor = db.rootCursor(); 2269 | 2270 | _ = try root_cursor.writePath(void, &.{ 2271 | .array_list_init, 2272 | .array_list_append, 2273 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2274 | .linked_array_list_init, 2275 | .linked_array_list_append, 2276 | .{ .write = .{ .uint = 42 } }, 2277 | }); 2278 | 2279 | for (0..1_000) |i| { 2280 | _ = try root_cursor.writePath(void, &.{ 2281 | .array_list_init, 2282 | .array_list_append, 2283 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2284 | .linked_array_list_init, 2285 | .{ .linked_array_list_insert = 0 }, 2286 | .{ .write = .{ .uint = i } }, 2287 | }); 2288 | } 2289 | } 2290 | 2291 | // insert at end of linked_array_list many times 2292 | { 2293 | try clearStorage(db_kind, init_opts); 2294 | var db = try xitdb.Database(db_kind, HashInt).init(init_opts); 2295 | var root_cursor = db.rootCursor(); 2296 | 2297 | _ = try root_cursor.writePath(void, &.{ 2298 | .array_list_init, 2299 | .array_list_append, 2300 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2301 | .linked_array_list_init, 2302 | .linked_array_list_append, 2303 | .{ .write = .{ .uint = 42 } }, 2304 | }); 2305 | 2306 | for (0..1_000) |i| { 2307 | _ = try root_cursor.writePath(void, &.{ 2308 | .array_list_init, 2309 | .array_list_append, 2310 | .{ .write = .{ .slot = try root_cursor.readPathSlot(void, &.{.{ .array_list_get = -1 }}) } }, 2311 | .linked_array_list_init, 2312 | .{ .linked_array_list_insert = i }, 2313 | .{ .write = .{ .uint = i } }, 2314 | }); 2315 | } 2316 | } 2317 | } 2318 | --------------------------------------------------------------------------------