├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md └── tools ├── iface ├── Cargo.toml └── src │ ├── db.rs │ ├── db │ ├── heed.rs │ ├── persy.rs │ ├── rocksdb.rs │ └── sqlite.rs │ └── lib.rs └── migrate ├── Cargo.toml └── src └── main.rs /.gitignore: -------------------------------------------------------------------------------- 1 | target/ -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "ahash" 7 | version = "0.8.11" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" 10 | dependencies = [ 11 | "cfg-if", 12 | "once_cell", 13 | "version_check", 14 | "zerocopy", 15 | ] 16 | 17 | [[package]] 18 | name = "aho-corasick" 19 | version = "1.1.2" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" 22 | dependencies = [ 23 | "memchr", 24 | ] 25 | 26 | [[package]] 27 | name = "allocator-api2" 28 | version = "0.2.18" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" 31 | 32 | [[package]] 33 | name = "ansi_term" 34 | version = "0.12.1" 35 | source = "registry+https://github.com/rust-lang/crates.io-index" 36 | checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" 37 | dependencies = [ 38 | "winapi", 39 | ] 40 | 41 | [[package]] 42 | name = "anyhow" 43 | version = "1.0.80" 44 | source = "registry+https://github.com/rust-lang/crates.io-index" 45 | checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" 46 | 47 | [[package]] 48 | name = "atty" 49 | version = "0.2.14" 50 | source = "registry+https://github.com/rust-lang/crates.io-index" 51 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 52 | dependencies = [ 53 | "hermit-abi", 54 | "libc", 55 | "winapi", 56 | ] 57 | 58 | [[package]] 59 | name = "autocfg" 60 | version = "1.1.0" 61 | source = "registry+https://github.com/rust-lang/crates.io-index" 62 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 63 | 64 | [[package]] 65 | name = "bincode" 66 | version = "1.3.3" 67 | source = "registry+https://github.com/rust-lang/crates.io-index" 68 | checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" 69 | dependencies = [ 70 | "serde", 71 | ] 72 | 73 | [[package]] 74 | name = "bindgen" 75 | version = "0.69.4" 76 | source = "registry+https://github.com/rust-lang/crates.io-index" 77 | checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" 78 | dependencies = [ 79 | "bitflags 2.5.0", 80 | "cexpr", 81 | "clang-sys", 82 | "itertools", 83 | "lazy_static", 84 | "lazycell", 85 | "proc-macro2", 86 | "quote", 87 | "regex", 88 | "rustc-hash", 89 | "shlex", 90 | "syn", 91 | ] 92 | 93 | [[package]] 94 | name = "bitflags" 95 | version = "1.3.2" 96 | source = "registry+https://github.com/rust-lang/crates.io-index" 97 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 98 | 99 | [[package]] 100 | name = "bitflags" 101 | version = "2.5.0" 102 | source = "registry+https://github.com/rust-lang/crates.io-index" 103 | checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" 104 | 105 | [[package]] 106 | name = "bytemuck" 107 | version = "1.14.3" 108 | source = "registry+https://github.com/rust-lang/crates.io-index" 109 | checksum = "a2ef034f05691a48569bd920a96c81b9d91bbad1ab5ac7c4616c1f6ef36cb79f" 110 | 111 | [[package]] 112 | name = "byteorder" 113 | version = "1.5.0" 114 | source = "registry+https://github.com/rust-lang/crates.io-index" 115 | checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" 116 | 117 | [[package]] 118 | name = "bzip2-sys" 119 | version = "0.1.11+1.0.8" 120 | source = "registry+https://github.com/rust-lang/crates.io-index" 121 | checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" 122 | dependencies = [ 123 | "cc", 124 | "libc", 125 | "pkg-config", 126 | ] 127 | 128 | [[package]] 129 | name = "cc" 130 | version = "1.0.88" 131 | source = "registry+https://github.com/rust-lang/crates.io-index" 132 | checksum = "02f341c093d19155a6e41631ce5971aac4e9a868262212153124c15fa22d1cdc" 133 | dependencies = [ 134 | "libc", 135 | ] 136 | 137 | [[package]] 138 | name = "cexpr" 139 | version = "0.6.0" 140 | source = "registry+https://github.com/rust-lang/crates.io-index" 141 | checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" 142 | dependencies = [ 143 | "nom", 144 | ] 145 | 146 | [[package]] 147 | name = "cfg-if" 148 | version = "1.0.0" 149 | source = "registry+https://github.com/rust-lang/crates.io-index" 150 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 151 | 152 | [[package]] 153 | name = "clang-sys" 154 | version = "1.7.0" 155 | source = "registry+https://github.com/rust-lang/crates.io-index" 156 | checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" 157 | dependencies = [ 158 | "glob", 159 | "libc", 160 | "libloading", 161 | ] 162 | 163 | [[package]] 164 | name = "clap" 165 | version = "2.34.0" 166 | source = "registry+https://github.com/rust-lang/crates.io-index" 167 | checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" 168 | dependencies = [ 169 | "ansi_term", 170 | "atty", 171 | "bitflags 1.3.2", 172 | "strsim", 173 | "textwrap", 174 | "unicode-width", 175 | "vec_map", 176 | ] 177 | 178 | [[package]] 179 | name = "conduit_iface" 180 | version = "0.1.0" 181 | dependencies = [ 182 | "anyhow", 183 | "heed", 184 | "itertools", 185 | "persy", 186 | "rusqlite", 187 | "rust-rocksdb", 188 | "thiserror", 189 | ] 190 | 191 | [[package]] 192 | name = "conduit_migrate" 193 | version = "0.1.0" 194 | dependencies = [ 195 | "anyhow", 196 | "clap", 197 | "conduit_iface", 198 | "thiserror", 199 | ] 200 | 201 | [[package]] 202 | name = "crc" 203 | version = "3.0.1" 204 | source = "registry+https://github.com/rust-lang/crates.io-index" 205 | checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" 206 | dependencies = [ 207 | "crc-catalog", 208 | ] 209 | 210 | [[package]] 211 | name = "crc-catalog" 212 | version = "2.4.0" 213 | source = "registry+https://github.com/rust-lang/crates.io-index" 214 | checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" 215 | 216 | [[package]] 217 | name = "crossbeam-queue" 218 | version = "0.3.11" 219 | source = "registry+https://github.com/rust-lang/crates.io-index" 220 | checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" 221 | dependencies = [ 222 | "crossbeam-utils", 223 | ] 224 | 225 | [[package]] 226 | name = "crossbeam-utils" 227 | version = "0.8.19" 228 | source = "registry+https://github.com/rust-lang/crates.io-index" 229 | checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" 230 | 231 | [[package]] 232 | name = "data-encoding" 233 | version = "2.5.0" 234 | source = "registry+https://github.com/rust-lang/crates.io-index" 235 | checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" 236 | 237 | [[package]] 238 | name = "either" 239 | version = "1.10.0" 240 | source = "registry+https://github.com/rust-lang/crates.io-index" 241 | checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" 242 | 243 | [[package]] 244 | name = "fallible-iterator" 245 | version = "0.3.0" 246 | source = "registry+https://github.com/rust-lang/crates.io-index" 247 | checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" 248 | 249 | [[package]] 250 | name = "fallible-streaming-iterator" 251 | version = "0.1.9" 252 | source = "registry+https://github.com/rust-lang/crates.io-index" 253 | checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" 254 | 255 | [[package]] 256 | name = "form_urlencoded" 257 | version = "1.2.1" 258 | source = "registry+https://github.com/rust-lang/crates.io-index" 259 | checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" 260 | dependencies = [ 261 | "percent-encoding", 262 | ] 263 | 264 | [[package]] 265 | name = "fs2" 266 | version = "0.4.3" 267 | source = "registry+https://github.com/rust-lang/crates.io-index" 268 | checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" 269 | dependencies = [ 270 | "libc", 271 | "winapi", 272 | ] 273 | 274 | [[package]] 275 | name = "getrandom" 276 | version = "0.2.12" 277 | source = "registry+https://github.com/rust-lang/crates.io-index" 278 | checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" 279 | dependencies = [ 280 | "cfg-if", 281 | "libc", 282 | "wasi", 283 | ] 284 | 285 | [[package]] 286 | name = "glob" 287 | version = "0.3.1" 288 | source = "registry+https://github.com/rust-lang/crates.io-index" 289 | checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" 290 | 291 | [[package]] 292 | name = "hashbrown" 293 | version = "0.14.5" 294 | source = "registry+https://github.com/rust-lang/crates.io-index" 295 | checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" 296 | dependencies = [ 297 | "ahash", 298 | "allocator-api2", 299 | ] 300 | 301 | [[package]] 302 | name = "hashlink" 303 | version = "0.9.0" 304 | source = "registry+https://github.com/rust-lang/crates.io-index" 305 | checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee" 306 | dependencies = [ 307 | "hashbrown", 308 | ] 309 | 310 | [[package]] 311 | name = "heed" 312 | version = "0.10.6" 313 | source = "git+https://github.com/timokoesters/heed.git?rev=f6f825da7fb2c758867e05ad973ef800a6fe1d5d#f6f825da7fb2c758867e05ad973ef800a6fe1d5d" 314 | dependencies = [ 315 | "bytemuck", 316 | "byteorder", 317 | "heed-traits", 318 | "heed-types", 319 | "libc", 320 | "lmdb-rkv-sys", 321 | "once_cell", 322 | "page_size", 323 | "serde", 324 | "synchronoise", 325 | "url", 326 | ] 327 | 328 | [[package]] 329 | name = "heed-traits" 330 | version = "0.7.0" 331 | source = "git+https://github.com/timokoesters/heed.git?rev=f6f825da7fb2c758867e05ad973ef800a6fe1d5d#f6f825da7fb2c758867e05ad973ef800a6fe1d5d" 332 | 333 | [[package]] 334 | name = "heed-types" 335 | version = "0.7.2" 336 | source = "git+https://github.com/timokoesters/heed.git?rev=f6f825da7fb2c758867e05ad973ef800a6fe1d5d#f6f825da7fb2c758867e05ad973ef800a6fe1d5d" 337 | dependencies = [ 338 | "bincode", 339 | "bytemuck", 340 | "byteorder", 341 | "heed-traits", 342 | "serde", 343 | "serde_json", 344 | ] 345 | 346 | [[package]] 347 | name = "hermit-abi" 348 | version = "0.1.19" 349 | source = "registry+https://github.com/rust-lang/crates.io-index" 350 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" 351 | dependencies = [ 352 | "libc", 353 | ] 354 | 355 | [[package]] 356 | name = "idna" 357 | version = "0.5.0" 358 | source = "registry+https://github.com/rust-lang/crates.io-index" 359 | checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" 360 | dependencies = [ 361 | "unicode-bidi", 362 | "unicode-normalization", 363 | ] 364 | 365 | [[package]] 366 | name = "itertools" 367 | version = "0.10.5" 368 | source = "registry+https://github.com/rust-lang/crates.io-index" 369 | checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" 370 | dependencies = [ 371 | "either", 372 | ] 373 | 374 | [[package]] 375 | name = "itoa" 376 | version = "1.0.10" 377 | source = "registry+https://github.com/rust-lang/crates.io-index" 378 | checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" 379 | 380 | [[package]] 381 | name = "lazy_static" 382 | version = "1.4.0" 383 | source = "registry+https://github.com/rust-lang/crates.io-index" 384 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 385 | 386 | [[package]] 387 | name = "lazycell" 388 | version = "1.3.0" 389 | source = "registry+https://github.com/rust-lang/crates.io-index" 390 | checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" 391 | 392 | [[package]] 393 | name = "libc" 394 | version = "0.2.153" 395 | source = "registry+https://github.com/rust-lang/crates.io-index" 396 | checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" 397 | 398 | [[package]] 399 | name = "libloading" 400 | version = "0.8.1" 401 | source = "registry+https://github.com/rust-lang/crates.io-index" 402 | checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" 403 | dependencies = [ 404 | "cfg-if", 405 | "windows-sys", 406 | ] 407 | 408 | [[package]] 409 | name = "libsqlite3-sys" 410 | version = "0.28.0" 411 | source = "registry+https://github.com/rust-lang/crates.io-index" 412 | checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" 413 | dependencies = [ 414 | "cc", 415 | "pkg-config", 416 | "vcpkg", 417 | ] 418 | 419 | [[package]] 420 | name = "libz-sys" 421 | version = "1.1.16" 422 | source = "registry+https://github.com/rust-lang/crates.io-index" 423 | checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" 424 | dependencies = [ 425 | "cc", 426 | "pkg-config", 427 | "vcpkg", 428 | ] 429 | 430 | [[package]] 431 | name = "linked-hash-map" 432 | version = "0.5.6" 433 | source = "registry+https://github.com/rust-lang/crates.io-index" 434 | checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" 435 | 436 | [[package]] 437 | name = "lmdb-rkv-sys" 438 | version = "0.11.2" 439 | source = "registry+https://github.com/rust-lang/crates.io-index" 440 | checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe" 441 | dependencies = [ 442 | "cc", 443 | "libc", 444 | "pkg-config", 445 | ] 446 | 447 | [[package]] 448 | name = "lz4-sys" 449 | version = "1.9.4" 450 | source = "registry+https://github.com/rust-lang/crates.io-index" 451 | checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" 452 | dependencies = [ 453 | "cc", 454 | "libc", 455 | ] 456 | 457 | [[package]] 458 | name = "memchr" 459 | version = "2.7.1" 460 | source = "registry+https://github.com/rust-lang/crates.io-index" 461 | checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" 462 | 463 | [[package]] 464 | name = "minimal-lexical" 465 | version = "0.2.1" 466 | source = "registry+https://github.com/rust-lang/crates.io-index" 467 | checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" 468 | 469 | [[package]] 470 | name = "nom" 471 | version = "7.1.3" 472 | source = "registry+https://github.com/rust-lang/crates.io-index" 473 | checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" 474 | dependencies = [ 475 | "memchr", 476 | "minimal-lexical", 477 | ] 478 | 479 | [[package]] 480 | name = "num-traits" 481 | version = "0.2.18" 482 | source = "registry+https://github.com/rust-lang/crates.io-index" 483 | checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" 484 | dependencies = [ 485 | "autocfg", 486 | ] 487 | 488 | [[package]] 489 | name = "once_cell" 490 | version = "1.19.0" 491 | source = "registry+https://github.com/rust-lang/crates.io-index" 492 | checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" 493 | 494 | [[package]] 495 | name = "page_size" 496 | version = "0.4.2" 497 | source = "registry+https://github.com/rust-lang/crates.io-index" 498 | checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" 499 | dependencies = [ 500 | "libc", 501 | "winapi", 502 | ] 503 | 504 | [[package]] 505 | name = "percent-encoding" 506 | version = "2.3.1" 507 | source = "registry+https://github.com/rust-lang/crates.io-index" 508 | checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" 509 | 510 | [[package]] 511 | name = "persy" 512 | version = "1.4.7" 513 | source = "registry+https://github.com/rust-lang/crates.io-index" 514 | checksum = "3cd38c602b23c2f451842d89f27cd5e0d4b292176daf40feeda859c658dcdc76" 515 | dependencies = [ 516 | "crc", 517 | "data-encoding", 518 | "fs2", 519 | "linked-hash-map", 520 | "rand", 521 | "thiserror", 522 | "unsigned-varint", 523 | "zigzag", 524 | ] 525 | 526 | [[package]] 527 | name = "pkg-config" 528 | version = "0.3.30" 529 | source = "registry+https://github.com/rust-lang/crates.io-index" 530 | checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" 531 | 532 | [[package]] 533 | name = "ppv-lite86" 534 | version = "0.2.17" 535 | source = "registry+https://github.com/rust-lang/crates.io-index" 536 | checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" 537 | 538 | [[package]] 539 | name = "proc-macro2" 540 | version = "1.0.78" 541 | source = "registry+https://github.com/rust-lang/crates.io-index" 542 | checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" 543 | dependencies = [ 544 | "unicode-ident", 545 | ] 546 | 547 | [[package]] 548 | name = "quote" 549 | version = "1.0.35" 550 | source = "registry+https://github.com/rust-lang/crates.io-index" 551 | checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" 552 | dependencies = [ 553 | "proc-macro2", 554 | ] 555 | 556 | [[package]] 557 | name = "rand" 558 | version = "0.8.5" 559 | source = "registry+https://github.com/rust-lang/crates.io-index" 560 | checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" 561 | dependencies = [ 562 | "libc", 563 | "rand_chacha", 564 | "rand_core", 565 | ] 566 | 567 | [[package]] 568 | name = "rand_chacha" 569 | version = "0.3.1" 570 | source = "registry+https://github.com/rust-lang/crates.io-index" 571 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 572 | dependencies = [ 573 | "ppv-lite86", 574 | "rand_core", 575 | ] 576 | 577 | [[package]] 578 | name = "rand_core" 579 | version = "0.6.4" 580 | source = "registry+https://github.com/rust-lang/crates.io-index" 581 | checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" 582 | dependencies = [ 583 | "getrandom", 584 | ] 585 | 586 | [[package]] 587 | name = "regex" 588 | version = "1.10.3" 589 | source = "registry+https://github.com/rust-lang/crates.io-index" 590 | checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" 591 | dependencies = [ 592 | "aho-corasick", 593 | "memchr", 594 | "regex-automata", 595 | "regex-syntax", 596 | ] 597 | 598 | [[package]] 599 | name = "regex-automata" 600 | version = "0.4.5" 601 | source = "registry+https://github.com/rust-lang/crates.io-index" 602 | checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" 603 | dependencies = [ 604 | "aho-corasick", 605 | "memchr", 606 | "regex-syntax", 607 | ] 608 | 609 | [[package]] 610 | name = "regex-syntax" 611 | version = "0.8.2" 612 | source = "registry+https://github.com/rust-lang/crates.io-index" 613 | checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" 614 | 615 | [[package]] 616 | name = "rusqlite" 617 | version = "0.31.0" 618 | source = "registry+https://github.com/rust-lang/crates.io-index" 619 | checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" 620 | dependencies = [ 621 | "bitflags 2.5.0", 622 | "fallible-iterator", 623 | "fallible-streaming-iterator", 624 | "hashlink", 625 | "libsqlite3-sys", 626 | "smallvec", 627 | ] 628 | 629 | [[package]] 630 | name = "rust-librocksdb-sys" 631 | version = "0.21.0+9.1.1" 632 | source = "registry+https://github.com/rust-lang/crates.io-index" 633 | checksum = "75cb7b9cd5ce3b3ce0757ceab2240f7471826780b8700845c0cfd418cb7e398d" 634 | dependencies = [ 635 | "bindgen", 636 | "bzip2-sys", 637 | "cc", 638 | "glob", 639 | "libc", 640 | "libz-sys", 641 | "lz4-sys", 642 | "zstd-sys", 643 | ] 644 | 645 | [[package]] 646 | name = "rust-rocksdb" 647 | version = "0.25.0" 648 | source = "registry+https://github.com/rust-lang/crates.io-index" 649 | checksum = "2bcfb31b5bf2e3274686ebfdf9a946e9a327a3bc54adc7e5cda9f4fdcc4b55f1" 650 | dependencies = [ 651 | "libc", 652 | "rust-librocksdb-sys", 653 | ] 654 | 655 | [[package]] 656 | name = "rustc-hash" 657 | version = "1.1.0" 658 | source = "registry+https://github.com/rust-lang/crates.io-index" 659 | checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" 660 | 661 | [[package]] 662 | name = "ryu" 663 | version = "1.0.17" 664 | source = "registry+https://github.com/rust-lang/crates.io-index" 665 | checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" 666 | 667 | [[package]] 668 | name = "serde" 669 | version = "1.0.197" 670 | source = "registry+https://github.com/rust-lang/crates.io-index" 671 | checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" 672 | dependencies = [ 673 | "serde_derive", 674 | ] 675 | 676 | [[package]] 677 | name = "serde_derive" 678 | version = "1.0.197" 679 | source = "registry+https://github.com/rust-lang/crates.io-index" 680 | checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" 681 | dependencies = [ 682 | "proc-macro2", 683 | "quote", 684 | "syn", 685 | ] 686 | 687 | [[package]] 688 | name = "serde_json" 689 | version = "1.0.114" 690 | source = "registry+https://github.com/rust-lang/crates.io-index" 691 | checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" 692 | dependencies = [ 693 | "itoa", 694 | "ryu", 695 | "serde", 696 | ] 697 | 698 | [[package]] 699 | name = "shlex" 700 | version = "1.3.0" 701 | source = "registry+https://github.com/rust-lang/crates.io-index" 702 | checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" 703 | 704 | [[package]] 705 | name = "smallvec" 706 | version = "1.13.1" 707 | source = "registry+https://github.com/rust-lang/crates.io-index" 708 | checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" 709 | 710 | [[package]] 711 | name = "strsim" 712 | version = "0.8.0" 713 | source = "registry+https://github.com/rust-lang/crates.io-index" 714 | checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" 715 | 716 | [[package]] 717 | name = "syn" 718 | version = "2.0.50" 719 | source = "registry+https://github.com/rust-lang/crates.io-index" 720 | checksum = "74f1bdc9872430ce9b75da68329d1c1746faf50ffac5f19e02b71e37ff881ffb" 721 | dependencies = [ 722 | "proc-macro2", 723 | "quote", 724 | "unicode-ident", 725 | ] 726 | 727 | [[package]] 728 | name = "synchronoise" 729 | version = "1.0.1" 730 | source = "registry+https://github.com/rust-lang/crates.io-index" 731 | checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" 732 | dependencies = [ 733 | "crossbeam-queue", 734 | ] 735 | 736 | [[package]] 737 | name = "textwrap" 738 | version = "0.11.0" 739 | source = "registry+https://github.com/rust-lang/crates.io-index" 740 | checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" 741 | dependencies = [ 742 | "unicode-width", 743 | ] 744 | 745 | [[package]] 746 | name = "thiserror" 747 | version = "1.0.57" 748 | source = "registry+https://github.com/rust-lang/crates.io-index" 749 | checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" 750 | dependencies = [ 751 | "thiserror-impl", 752 | ] 753 | 754 | [[package]] 755 | name = "thiserror-impl" 756 | version = "1.0.57" 757 | source = "registry+https://github.com/rust-lang/crates.io-index" 758 | checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" 759 | dependencies = [ 760 | "proc-macro2", 761 | "quote", 762 | "syn", 763 | ] 764 | 765 | [[package]] 766 | name = "tinyvec" 767 | version = "1.6.0" 768 | source = "registry+https://github.com/rust-lang/crates.io-index" 769 | checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" 770 | dependencies = [ 771 | "tinyvec_macros", 772 | ] 773 | 774 | [[package]] 775 | name = "tinyvec_macros" 776 | version = "0.1.1" 777 | source = "registry+https://github.com/rust-lang/crates.io-index" 778 | checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" 779 | 780 | [[package]] 781 | name = "unicode-bidi" 782 | version = "0.3.15" 783 | source = "registry+https://github.com/rust-lang/crates.io-index" 784 | checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" 785 | 786 | [[package]] 787 | name = "unicode-ident" 788 | version = "1.0.12" 789 | source = "registry+https://github.com/rust-lang/crates.io-index" 790 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 791 | 792 | [[package]] 793 | name = "unicode-normalization" 794 | version = "0.1.23" 795 | source = "registry+https://github.com/rust-lang/crates.io-index" 796 | checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" 797 | dependencies = [ 798 | "tinyvec", 799 | ] 800 | 801 | [[package]] 802 | name = "unicode-width" 803 | version = "0.1.11" 804 | source = "registry+https://github.com/rust-lang/crates.io-index" 805 | checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" 806 | 807 | [[package]] 808 | name = "unsigned-varint" 809 | version = "0.7.2" 810 | source = "registry+https://github.com/rust-lang/crates.io-index" 811 | checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" 812 | 813 | [[package]] 814 | name = "url" 815 | version = "2.5.0" 816 | source = "registry+https://github.com/rust-lang/crates.io-index" 817 | checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" 818 | dependencies = [ 819 | "form_urlencoded", 820 | "idna", 821 | "percent-encoding", 822 | ] 823 | 824 | [[package]] 825 | name = "vcpkg" 826 | version = "0.2.15" 827 | source = "registry+https://github.com/rust-lang/crates.io-index" 828 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 829 | 830 | [[package]] 831 | name = "vec_map" 832 | version = "0.8.2" 833 | source = "registry+https://github.com/rust-lang/crates.io-index" 834 | checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" 835 | 836 | [[package]] 837 | name = "version_check" 838 | version = "0.9.4" 839 | source = "registry+https://github.com/rust-lang/crates.io-index" 840 | checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" 841 | 842 | [[package]] 843 | name = "wasi" 844 | version = "0.11.0+wasi-snapshot-preview1" 845 | source = "registry+https://github.com/rust-lang/crates.io-index" 846 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 847 | 848 | [[package]] 849 | name = "winapi" 850 | version = "0.3.9" 851 | source = "registry+https://github.com/rust-lang/crates.io-index" 852 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 853 | dependencies = [ 854 | "winapi-i686-pc-windows-gnu", 855 | "winapi-x86_64-pc-windows-gnu", 856 | ] 857 | 858 | [[package]] 859 | name = "winapi-i686-pc-windows-gnu" 860 | version = "0.4.0" 861 | source = "registry+https://github.com/rust-lang/crates.io-index" 862 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 863 | 864 | [[package]] 865 | name = "winapi-x86_64-pc-windows-gnu" 866 | version = "0.4.0" 867 | source = "registry+https://github.com/rust-lang/crates.io-index" 868 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 869 | 870 | [[package]] 871 | name = "windows-sys" 872 | version = "0.48.0" 873 | source = "registry+https://github.com/rust-lang/crates.io-index" 874 | checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" 875 | dependencies = [ 876 | "windows-targets", 877 | ] 878 | 879 | [[package]] 880 | name = "windows-targets" 881 | version = "0.48.5" 882 | source = "registry+https://github.com/rust-lang/crates.io-index" 883 | checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" 884 | dependencies = [ 885 | "windows_aarch64_gnullvm", 886 | "windows_aarch64_msvc", 887 | "windows_i686_gnu", 888 | "windows_i686_msvc", 889 | "windows_x86_64_gnu", 890 | "windows_x86_64_gnullvm", 891 | "windows_x86_64_msvc", 892 | ] 893 | 894 | [[package]] 895 | name = "windows_aarch64_gnullvm" 896 | version = "0.48.5" 897 | source = "registry+https://github.com/rust-lang/crates.io-index" 898 | checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" 899 | 900 | [[package]] 901 | name = "windows_aarch64_msvc" 902 | version = "0.48.5" 903 | source = "registry+https://github.com/rust-lang/crates.io-index" 904 | checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" 905 | 906 | [[package]] 907 | name = "windows_i686_gnu" 908 | version = "0.48.5" 909 | source = "registry+https://github.com/rust-lang/crates.io-index" 910 | checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" 911 | 912 | [[package]] 913 | name = "windows_i686_msvc" 914 | version = "0.48.5" 915 | source = "registry+https://github.com/rust-lang/crates.io-index" 916 | checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" 917 | 918 | [[package]] 919 | name = "windows_x86_64_gnu" 920 | version = "0.48.5" 921 | source = "registry+https://github.com/rust-lang/crates.io-index" 922 | checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" 923 | 924 | [[package]] 925 | name = "windows_x86_64_gnullvm" 926 | version = "0.48.5" 927 | source = "registry+https://github.com/rust-lang/crates.io-index" 928 | checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" 929 | 930 | [[package]] 931 | name = "windows_x86_64_msvc" 932 | version = "0.48.5" 933 | source = "registry+https://github.com/rust-lang/crates.io-index" 934 | checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" 935 | 936 | [[package]] 937 | name = "zerocopy" 938 | version = "0.7.32" 939 | source = "registry+https://github.com/rust-lang/crates.io-index" 940 | checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" 941 | dependencies = [ 942 | "zerocopy-derive", 943 | ] 944 | 945 | [[package]] 946 | name = "zerocopy-derive" 947 | version = "0.7.32" 948 | source = "registry+https://github.com/rust-lang/crates.io-index" 949 | checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" 950 | dependencies = [ 951 | "proc-macro2", 952 | "quote", 953 | "syn", 954 | ] 955 | 956 | [[package]] 957 | name = "zigzag" 958 | version = "0.1.0" 959 | source = "registry+https://github.com/rust-lang/crates.io-index" 960 | checksum = "70b40401a28d86ce16a330b863b86fd7dbee4d7c940587ab09ab8c019f9e3fdf" 961 | dependencies = [ 962 | "num-traits", 963 | ] 964 | 965 | [[package]] 966 | name = "zstd-sys" 967 | version = "2.0.10+zstd.1.5.6" 968 | source = "registry+https://github.com/rust-lang/crates.io-index" 969 | checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" 970 | dependencies = [ 971 | "cc", 972 | "pkg-config", 973 | ] 974 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | 3 | members = [ 4 | "tools/*", 5 | ] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Jonathan de Jong 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🧰 Conduit Toolbox 🦀 2 | 3 | Helper tools for [Conduit](https://conduit.rs), a matrix homeserver in rust. 4 | 5 | This repository provides binaries to wrangle your conduit homeserver with. 6 | 7 | ## Tools 8 | 9 | ### `conduit_sled_to_sqlite` 10 | 11 | **This tool is no longer available on the main branch**, go to `with-sled` to download from there. 12 | 13 | ### `conduit_migrate` 14 | 15 | This tool provides generic migration between `heed`, `sqlite`, `persy`, and `rocksdb` conduit databases. 16 | 17 | ## Installing 18 | 19 | For the best experience, compile this toolbox locally on your server; 20 | 21 | 1. Download and install rust, see [rustup](https://rustup.rs/) for more. 22 | 2. Be sure that the rust executables are on your `$PATH` 23 | 3. You may want to have a compiler and build tools installed on your system, or else cargo will complain about not being able to "link" or "compile" with `cc`. 24 | - on debian/ubuntu-based systems you can install this with `sudo apt install build-essential` 25 | 4. `cargo install --locked --git https://github.com/shadowjonathan/conduit_toolbox conduit_migrate` 26 | 27 | (updating only requires running that last line again) 28 | -------------------------------------------------------------------------------- /tools/iface/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "conduit_iface" 3 | version = "0.1.0" 4 | edition = "2018" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | itertools = "0.10.1" 10 | thiserror = "1.0.26" 11 | anyhow = "1.0.42" 12 | 13 | rusqlite = { version = "0.31", features = ["bundled"], optional = true } 14 | heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } 15 | persy = { version = "1.2", optional = true } 16 | 17 | [dependencies.rocksdb] 18 | package = "rust-rocksdb" 19 | version = "0.25" 20 | optional = true 21 | features = [ 22 | "multi-threaded-cf", 23 | "zstd", 24 | "lz4", 25 | ] 26 | 27 | [features] 28 | default = [] 29 | 30 | sqlite = ["rusqlite"] 31 | -------------------------------------------------------------------------------- /tools/iface/src/db.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "heed")] 2 | pub mod heed; 3 | #[cfg(feature = "persy")] 4 | pub mod persy; 5 | #[cfg(feature = "rocksdb")] 6 | pub mod rocksdb; 7 | #[cfg(feature = "sqlite")] 8 | pub mod sqlite; 9 | 10 | use itertools::Itertools; 11 | 12 | pub type KVIter<'a> = Box, Vec)> + 'a>; 13 | 14 | pub type TreeKVIter<'a> = Box, KVIter<'a>)> + 'a>; 15 | 16 | #[derive(Clone, Copy)] 17 | pub struct Config { 18 | pub ignore_broken_rows: bool, 19 | } 20 | 21 | pub trait Database { 22 | fn names<'a>(&'a self) -> Vec>; 23 | 24 | fn segment<'a>(&'a mut self, name: Vec) -> Option>; // change return type to Result 25 | 26 | fn flush(&mut self); 27 | } 28 | 29 | pub trait Segment { 30 | fn batch_insert<'a>( 31 | &'a mut self, 32 | batch: Box, Vec)> + 'a>, 33 | ) -> anyhow::Result<()>; 34 | 35 | fn get_iter<'a>(&'a mut self) -> Box; 36 | } 37 | 38 | pub trait SegmentIter { 39 | fn iter<'a>(&'a mut self) -> KVIter<'a>; 40 | } 41 | 42 | pub fn copy_database( 43 | src: &mut dyn Database, 44 | dst: &mut dyn Database, 45 | chunk_size: usize, 46 | ) -> anyhow::Result<()> { 47 | // todo remove unwraps 48 | for seg_name in src.names() { 49 | drop(dbg!(String::from_utf8(seg_name.clone()))); 50 | 51 | let mut src_seg = src.segment(seg_name.clone()).unwrap(); 52 | 53 | let mut dst_seg = dst.segment(seg_name).unwrap(); 54 | 55 | let mut src_seg_iter = src_seg.get_iter(); 56 | 57 | let i = src_seg_iter.iter(); 58 | 59 | let mut x: usize = 0; 60 | 61 | for chunk in &i.chunks(chunk_size) { 62 | dbg!(&x); 63 | dst_seg.batch_insert(Box::new(chunk))?; 64 | x += chunk_size; 65 | } 66 | 67 | drop(dst_seg); 68 | drop(src_seg_iter); 69 | drop(src_seg); 70 | 71 | dst.flush(); 72 | } 73 | 74 | Ok(()) 75 | } 76 | -------------------------------------------------------------------------------- /tools/iface/src/db/heed.rs: -------------------------------------------------------------------------------- 1 | use super::{Database, KVIter, Segment, SegmentIter}; 2 | use heed::UntypedDatabase; 3 | use itertools::Itertools; 4 | use std::path::Path; 5 | use thiserror::Error; 6 | 7 | #[derive(Error, Debug)] 8 | #[error("There was a problem with the connection to the heed database: {0}")] 9 | pub struct HeedError(String); 10 | 11 | impl From for HeedError { 12 | fn from(err: heed::Error) -> Self { 13 | Self(err.to_string()) 14 | } 15 | } 16 | 17 | pub fn new_db>(path: P) -> Result { 18 | let mut env_builder = heed::EnvOpenOptions::new(); 19 | // env_builder.map_size(1024 * 1024 * 1024); // 1 Terabyte 20 | env_builder.max_readers(126); 21 | env_builder.max_dbs(128); 22 | 23 | Ok(env_builder.open(path)?) 24 | } 25 | 26 | pub struct HeedDB(heed::Env); 27 | 28 | impl HeedDB { 29 | pub fn new(env: heed::Env) -> Self { 30 | Self(env) 31 | } 32 | } 33 | 34 | impl Database for HeedDB { 35 | fn segment<'a>(&'a mut self, name: Vec) -> Option> { 36 | let name = String::from_utf8(name).ok()?; 37 | 38 | let db: UntypedDatabase = self.0.create_database(Some(name.as_str())).ok()?; 39 | 40 | Some(Box::new(HeedSegment { 41 | env: self.0.clone(), 42 | db, 43 | })) 44 | } 45 | 46 | fn names<'a>(&'a self) -> Vec> { 47 | let db: UntypedDatabase = self.0.open_database(None).unwrap().unwrap(); 48 | 49 | let txn = self.0.read_txn().unwrap(); 50 | 51 | db.iter(&txn) 52 | .unwrap() 53 | .filter_map(|r| -> Option<(Vec, UntypedDatabase)> { 54 | let (k, _) = r.ok()?; 55 | 56 | let name = String::from_utf8(k.to_vec()).ok()?; 57 | 58 | if let Some(db) = (self.0.open_database(Some(name.as_str()))).ok().flatten() { 59 | Some((k.to_vec(), db)) 60 | } else { 61 | None 62 | } 63 | }) 64 | .map(|(k, _)| k) 65 | .collect_vec() 66 | } 67 | 68 | fn flush(&mut self) { 69 | // NOOP 70 | } 71 | } 72 | pub struct HeedSegment { 73 | env: heed::Env, 74 | db: heed::UntypedDatabase, 75 | } 76 | 77 | impl Segment for HeedSegment { 78 | fn batch_insert<'a>( 79 | &'a mut self, 80 | batch: Box, Vec)> + 'a>, 81 | ) -> anyhow::Result<()> { 82 | let mut txn = self.env.write_txn().unwrap(); 83 | 84 | for (k, v) in batch { 85 | self.db.put(&mut txn, &k.as_slice(), &v.as_slice()).unwrap(); 86 | } 87 | 88 | txn.commit().unwrap(); 89 | 90 | Ok(()) 91 | } 92 | 93 | fn get_iter<'a>(&'a mut self) -> Box { 94 | todo!() 95 | } 96 | } 97 | 98 | struct HeedSegmentIter<'a>(heed::RoTxn<'a>, &'a heed::UntypedDatabase); 99 | 100 | impl SegmentIter for HeedSegmentIter<'_> { 101 | fn iter<'a>(&'a mut self) -> KVIter<'a> { 102 | Box::new(self.1.iter(&self.0).unwrap().filter_map(|r| { 103 | if let Ok(t) = r { 104 | Some((t.0.to_vec(), t.1.to_vec())) 105 | } else { 106 | None 107 | } 108 | })) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /tools/iface/src/db/persy.rs: -------------------------------------------------------------------------------- 1 | use super::{Database, KVIter, Segment, SegmentIter}; 2 | use persy::{ByteVec, Persy}; 3 | use std::path::Path; 4 | 5 | pub fn new_db>(path: P) -> anyhow::Result { 6 | let path = Path::new("./db.persy").join(path); 7 | 8 | let persy = persy::OpenOptions::new() 9 | .create(true) 10 | .config(persy::Config::new()) 11 | .open(&path)?; 12 | 13 | Ok(PersyDB { persy }) 14 | } 15 | 16 | pub struct PersyDB { 17 | persy: Persy, 18 | } 19 | 20 | impl Database for PersyDB { 21 | fn names<'a>(&'a self) -> Vec> { 22 | self.persy 23 | .list_indexes() 24 | .unwrap() 25 | .iter() 26 | .map(|(s, _)| s.as_bytes().to_vec()) 27 | .collect() 28 | } 29 | 30 | fn segment<'a>(&'a mut self, name: Vec) -> Option> { 31 | let string = String::from_utf8(name).unwrap(); 32 | 33 | if !self.persy.exists_index(&string).unwrap() { 34 | use persy::ValueMode; 35 | 36 | let mut tx = self.persy.begin().unwrap(); 37 | tx.create_index::(&string, ValueMode::Replace) 38 | .unwrap(); 39 | tx.prepare().unwrap().commit().unwrap(); 40 | } 41 | 42 | Some(Box::new(PersySeg { 43 | db: self, 44 | name: string, 45 | })) 46 | } 47 | 48 | fn flush(&mut self) { 49 | // NOOP 50 | } 51 | } 52 | 53 | pub struct PersySeg<'a> { 54 | db: &'a mut PersyDB, 55 | name: String, 56 | } 57 | 58 | impl<'r> Segment for PersySeg<'r> { 59 | fn batch_insert<'a>( 60 | &'a mut self, 61 | batch: Box, Vec)> + 'a>, 62 | ) -> anyhow::Result<()> { 63 | let mut tx = self.db.persy.begin()?; 64 | for (key, value) in batch { 65 | tx.put::( 66 | &self.name, 67 | ByteVec::from(key.clone()), 68 | ByteVec::from(value), 69 | )?; 70 | } 71 | tx.prepare()?.commit()?; 72 | 73 | Ok(()) 74 | } 75 | 76 | fn get_iter<'a>(&'a mut self) -> Box { 77 | Box::new(PersySegIter(self, &self.name)) 78 | } 79 | } 80 | 81 | pub struct PersySegIter<'a>(&'a PersySeg<'a>, &'a str); 82 | 83 | impl<'r> SegmentIter for PersySegIter<'r> { 84 | fn iter<'a>(&'a mut self) -> KVIter<'a> { 85 | Box::new( 86 | self.0 87 | .db 88 | .persy 89 | .range::(self.1, ..) 90 | .unwrap() 91 | .filter_map(|(k, v)| { 92 | v.into_iter() 93 | .map(|val| ((*k).to_owned().into(), (*val).to_owned().into())) 94 | .next() 95 | }), 96 | ) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /tools/iface/src/db/rocksdb.rs: -------------------------------------------------------------------------------- 1 | use std::{path::Path, sync::Arc}; 2 | 3 | use super::{Database, Segment}; 4 | use rocksdb::{DBWithThreadMode, MultiThreaded}; 5 | 6 | pub fn options() -> rocksdb::Options { 7 | let mut db_opts = rocksdb::Options::default(); 8 | 9 | db_opts.create_if_missing(true); 10 | db_opts.set_max_open_files(512); 11 | db_opts.set_compression_type(rocksdb::DBCompressionType::Lz4); 12 | db_opts.set_bottommost_compression_type(rocksdb::DBCompressionType::Zstd); 13 | db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level); 14 | db_opts.set_level_compaction_dynamic_level_bytes(true); 15 | db_opts.set_bytes_per_sync(1048576); 16 | db_opts.set_keep_log_file_num(100); 17 | db_opts.set_wal_recovery_mode(rocksdb::DBRecoveryMode::TolerateCorruptedTailRecords); 18 | 19 | let mut block_based_options = rocksdb::BlockBasedOptions::default(); 20 | block_based_options.set_bloom_filter(10.0, false); 21 | block_based_options.set_block_size(4 * 1024); 22 | block_based_options.set_cache_index_and_filter_blocks(true); 23 | block_based_options.set_pin_l0_filter_and_index_blocks_in_cache(true); 24 | block_based_options.set_optimize_filters_for_memory(true); 25 | 26 | db_opts.set_block_based_table_factory(&block_based_options); 27 | 28 | db_opts 29 | } 30 | 31 | pub fn new_conn>(path: P) -> Result { 32 | let opts = options(); 33 | 34 | let cfs = DBWithThreadMode::::list_cf(&opts, &path).unwrap_or_default(); 35 | 36 | let db = DBWithThreadMode::::open_cf_descriptors( 37 | &opts, 38 | &path, 39 | cfs.iter().map(|name| { 40 | let mut options = opts.clone(); 41 | let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1); 42 | options.set_prefix_extractor(prefix_extractor); 43 | 44 | rocksdb::ColumnFamilyDescriptor::new(name, options) 45 | }), 46 | )?; 47 | 48 | Ok(RocksDB { 49 | rocks: db, 50 | old_cfs: cfs, 51 | }) 52 | } 53 | 54 | pub struct RocksDB { 55 | rocks: DBWithThreadMode, 56 | old_cfs: Vec, 57 | } 58 | 59 | impl Database for RocksDB { 60 | fn segment<'a>(&'a mut self, name: Vec) -> Option> { 61 | let string = String::from_utf8(name).unwrap(); 62 | 63 | // Create if it didn't exist 64 | if !self.old_cfs.contains(&string) { 65 | let mut options = options(); 66 | 67 | let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1); 68 | options.set_prefix_extractor(prefix_extractor); 69 | 70 | let _ = self.rocks.create_cf(&string, &options); 71 | println!("created cf"); 72 | } 73 | 74 | Some(Box::new(RocksDBCF { 75 | db: self, 76 | name: string, 77 | })) 78 | } 79 | 80 | fn names<'a>(&'a self) -> Vec> { 81 | self.old_cfs 82 | .iter() 83 | .filter(|&v| &*v != "default") 84 | .map(|v| v.as_bytes().to_vec()) 85 | .collect() 86 | } 87 | 88 | fn flush(&mut self) { 89 | self.rocks.flush().unwrap() 90 | } 91 | } 92 | 93 | impl Drop for RocksDB { 94 | #[allow(unused_must_use)] 95 | fn drop(&mut self) { 96 | self.rocks.flush(); 97 | } 98 | } 99 | 100 | pub struct RocksDBCF<'a> { 101 | db: &'a mut RocksDB, 102 | name: String, 103 | } 104 | 105 | impl RocksDBCF<'_> { 106 | fn cf(&self) -> Arc> { 107 | self.db.rocks.cf_handle(&self.name).unwrap() 108 | } 109 | } 110 | 111 | impl<'r> Segment for RocksDBCF<'r> { 112 | fn batch_insert<'a>( 113 | &'a mut self, 114 | batch: Box, Vec)> + 'a>, 115 | ) -> anyhow::Result<()> { 116 | let cf = self.cf(); 117 | for (key, value) in batch { 118 | self.db.rocks.put_cf(&cf, key, value)?; 119 | } 120 | 121 | Ok(()) 122 | } 123 | 124 | fn get_iter(&mut self) -> Box { 125 | Box::new(RocksDBCFIter(self)) 126 | } 127 | } 128 | 129 | pub struct RocksDBCFIter<'a>(&'a RocksDBCF<'a>); 130 | 131 | impl super::SegmentIter for RocksDBCFIter<'_> { 132 | fn iter<'a>(&'a mut self) -> super::KVIter<'a> { 133 | Box::new( 134 | self.0 135 | .db 136 | .rocks 137 | .iterator_cf(&self.0.cf(), rocksdb::IteratorMode::Start) 138 | .map(|r| { 139 | let (k, v) = r.expect("we expect rocksdb to give us good rows only"); 140 | (Vec::from(k), Vec::from(v)) 141 | }), 142 | ) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /tools/iface/src/db/sqlite.rs: -------------------------------------------------------------------------------- 1 | use itertools::Itertools; 2 | use rusqlite::{self, Connection, DatabaseName::Main, Statement}; 3 | use std::{collections::HashSet, iter::FromIterator, path::Path}; 4 | 5 | use super::{Config, Database, KVIter, Segment, SegmentIter}; 6 | 7 | pub fn new_conn>(path: P) -> rusqlite::Result { 8 | let path = path.as_ref().join("conduit.db"); 9 | let conn = Connection::open(path)?; 10 | 11 | conn.pragma_update(Some(Main), "journal_mode", &"WAL".to_owned())?; 12 | 13 | Ok(conn) 14 | } 15 | 16 | pub struct SqliteDB { 17 | conn: Connection, 18 | config: Config, 19 | } 20 | 21 | const CORRECT_TABLE_SET: &[&str] = &["key", "value"]; 22 | 23 | impl<'a> SqliteDB { 24 | pub fn new(conn: Connection, config: Config) -> Self { 25 | Self { conn, config } 26 | } 27 | 28 | fn valid_tables(&self) -> Vec { 29 | self.conn 30 | .prepare("SELECT name FROM sqlite_master WHERE type='table'") 31 | .unwrap() 32 | .query_map([], |row| row.get(0)) 33 | .unwrap() 34 | .map(|r| r.unwrap()) 35 | .filter(|a| self.test_table(a)) 36 | .collect() 37 | } 38 | 39 | fn test_table(&self, table: &String) -> bool { 40 | let set: HashSet = self 41 | .conn 42 | .prepare("SELECT name FROM pragma_table_info(?)") 43 | .unwrap() 44 | .query_map([table], |row| row.get(0)) 45 | .unwrap() 46 | .map(|r| r.unwrap()) 47 | .collect(); 48 | 49 | set == HashSet::from_iter(CORRECT_TABLE_SET.iter().map(|s| s.to_string())) 50 | } 51 | } 52 | 53 | impl Database for SqliteDB { 54 | fn names<'a>(&'a self) -> Vec> { 55 | self.valid_tables().into_iter().map_into().collect_vec() 56 | } 57 | 58 | fn segment<'a>(&'a mut self, name: Vec) -> Option> { 59 | let string = String::from_utf8(name).unwrap(); 60 | // taken from src/database/abstraction/sqlite.rs 61 | self.conn.execute(format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", &string).as_str(), []).unwrap(); 62 | 63 | Some(Box::new(SqliteSegment { 64 | conn: &mut self.conn, 65 | name: string, 66 | config: self.config, 67 | })) 68 | } 69 | 70 | fn flush(&mut self) { 71 | // NOOP 72 | } 73 | } 74 | 75 | pub struct SqliteSegment<'a> { 76 | conn: &'a mut Connection, 77 | name: String, 78 | config: Config, 79 | } 80 | 81 | impl Segment for SqliteSegment<'_> { 82 | fn batch_insert( 83 | &mut self, 84 | batch: Box, Vec)> + '_>, 85 | ) -> anyhow::Result<()> { 86 | let tx = self.conn.transaction()?; 87 | let sql_s = format!("INSERT INTO {} (key, value) VALUES (?, ?)", &self.name); 88 | let sql = sql_s.as_str(); 89 | 90 | for (k, v) in batch { 91 | tx.execute(sql, rusqlite::params![k, v])?; 92 | } 93 | 94 | tx.commit().map_err(Into::into) 95 | } 96 | 97 | fn get_iter(&mut self) -> Box { 98 | Box::new(SqliteSegmentIter { 99 | statement: self 100 | .conn 101 | .prepare(format!("SELECT key, value FROM {}", self.name).as_str()) 102 | .unwrap(), 103 | config: self.config, 104 | }) 105 | } 106 | } 107 | 108 | struct SqliteSegmentIter<'a> { 109 | statement: Statement<'a>, 110 | config: Config, 111 | } 112 | 113 | impl SegmentIter for SqliteSegmentIter<'_> { 114 | fn iter<'f>(&'f mut self) -> KVIter<'f> { 115 | let config = self.config; 116 | 117 | Box::new( 118 | self.statement 119 | .query_map([], |row| Ok((row.get(0), row.get(1)))) 120 | .unwrap() 121 | .map(|x| x.unwrap()) 122 | .filter_map(move |(k, v)| { 123 | let advice = "You could try using `--ignore-broken-rows` to complete the migration, but take note of its caveats."; 124 | let Ok(k) = k else { 125 | if config.ignore_broken_rows { 126 | println!("ignored a row because its key is malformed"); 127 | } else { 128 | panic!("This row has a malformed key. {}", advice); 129 | } 130 | return None; 131 | }; 132 | 133 | let Ok(v) = v else { 134 | if config.ignore_broken_rows { 135 | println!("ignored a row because its value is malformed"); 136 | } else { 137 | panic!("This row has a malformed value. {}", advice); 138 | } 139 | return None; 140 | }; 141 | 142 | Some((k, v)) 143 | }), 144 | ) 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /tools/iface/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod db; 2 | -------------------------------------------------------------------------------- /tools/migrate/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "conduit_migrate" 3 | version = "0.1.0" 4 | edition = "2018" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | clap = "2.33.3" 10 | anyhow = "1.0.41" 11 | conduit_iface = { path = "../iface/", default-features = false } 12 | thiserror = "1.0.26" 13 | 14 | [features] 15 | default = ["sqlite", "rocksdb"] 16 | 17 | persy = ["conduit_iface/persy"] 18 | heed = ["conduit_iface/heed"] 19 | sqlite = ["conduit_iface/sqlite"] 20 | rocksdb = ["conduit_iface/rocksdb"] 21 | -------------------------------------------------------------------------------- /tools/migrate/src/main.rs: -------------------------------------------------------------------------------- 1 | use clap::{App, Arg}; 2 | use conduit_iface::db::{self, copy_database, Config}; 3 | use std::{ 4 | ops::{Deref, DerefMut}, 5 | path::{Path, PathBuf}, 6 | }; 7 | 8 | enum Database { 9 | #[cfg(feature = "heed")] 10 | Heed(db::heed::HeedDB), 11 | #[cfg(feature = "sqlite")] 12 | Sqlite(db::sqlite::SqliteDB), 13 | #[cfg(feature = "rocksdb")] 14 | Rocks(db::rocksdb::RocksDB), 15 | #[cfg(feature = "persy")] 16 | Persy(db::persy::PersyDB), 17 | } 18 | 19 | impl Database { 20 | fn new(name: &str, path: PathBuf, config: Config) -> anyhow::Result { 21 | Ok(match name { 22 | #[cfg(feature = "heed")] 23 | "heed" => Self::Heed(db::heed::HeedDB::new(db::heed::new_db(path)?)), 24 | #[cfg(feature = "sqlite")] 25 | "sqlite" => Self::Sqlite(db::sqlite::SqliteDB::new( 26 | db::sqlite::new_conn(path)?, 27 | config, 28 | )), 29 | #[cfg(feature = "rocksdb")] 30 | "rocks" => Self::Rocks(db::rocksdb::new_conn(path)?), 31 | #[cfg(feature = "persy")] 32 | "persy" => Self::Persy(db::persy::new_db(path)?), 33 | _ => panic!("unknown database type: {}", name), 34 | }) 35 | } 36 | } 37 | 38 | impl Deref for Database { 39 | type Target = dyn db::Database; 40 | 41 | fn deref(&self) -> &Self::Target { 42 | match self { 43 | #[cfg(feature = "heed")] 44 | Database::Heed(db) => db, 45 | #[cfg(feature = "sqlite")] 46 | Database::Sqlite(db) => db, 47 | #[cfg(feature = "rocksdb")] 48 | Database::Rocks(db) => db, 49 | #[cfg(feature = "persy")] 50 | Database::Persy(db) => db, 51 | } 52 | } 53 | } 54 | 55 | impl DerefMut for Database { 56 | fn deref_mut(&mut self) -> &mut Self::Target { 57 | match self { 58 | #[cfg(feature = "heed")] 59 | Database::Heed(db) => db, 60 | #[cfg(feature = "sqlite")] 61 | Database::Sqlite(db) => db, 62 | #[cfg(feature = "rocksdb")] 63 | Database::Rocks(db) => db, 64 | #[cfg(feature = "persy")] 65 | Database::Persy(db) => db, 66 | } 67 | } 68 | } 69 | 70 | const DATABASES: &[&str] = &[ 71 | #[cfg(feature = "heed")] 72 | "heed", 73 | #[cfg(feature = "sqlite")] 74 | "sqlite", 75 | #[cfg(feature = "rocksdb")] 76 | "rocks", 77 | #[cfg(feature = "persy")] 78 | "persy", 79 | ]; 80 | 81 | fn main() -> anyhow::Result<()> { 82 | let matches = App::new("Conduit Generic Migrator") 83 | .arg( 84 | Arg::with_name("from_dir") 85 | .short("s") 86 | .long("from-dir") 87 | .takes_value(true) 88 | .long_help("Sets the directory to grab the database from\nWill default to \".\""), 89 | ) 90 | .arg( 91 | Arg::with_name("to_dir") 92 | .short("d") 93 | .long("to-dir") 94 | .takes_value(true) 95 | .long_help("Sets the destination directory\nWill default to from_dir"), 96 | ) 97 | .arg( 98 | Arg::with_name("from") 99 | .short("f") 100 | .long("from") 101 | .long_help( 102 | format!( 103 | "The type of database to convert from\nExample: {}", 104 | DATABASES.join(", ") 105 | ) 106 | .as_str(), 107 | ) 108 | .takes_value(true) 109 | .required(true), 110 | ) 111 | .arg( 112 | Arg::with_name("to") 113 | .short("t") 114 | .long("to") 115 | .long_help( 116 | format!( 117 | "The type of database to convert to\nExample: {}", 118 | DATABASES.join(", ") 119 | ) 120 | .as_str(), 121 | ) 122 | .takes_value(true) 123 | .required(true), 124 | ) 125 | .arg( 126 | Arg::with_name("ignore_broken_rows") 127 | .long("ignore-broken-rows") 128 | .long_help("Lossy migration methodology if parts of the database are malformed due to e.g. improper manual database surgery. Currently only applies to SQLite.") 129 | ) 130 | .get_matches(); 131 | 132 | let src_dir = matches.value_of("from_dir").unwrap_or("."); 133 | 134 | let dst_dir = matches.value_of("to_dir"); 135 | 136 | let src_dir = Path::new(src_dir).canonicalize()?; 137 | 138 | if !src_dir.is_dir() { 139 | return Err(anyhow::anyhow!("source path must be directory")); 140 | } 141 | 142 | let dst_dir = match dst_dir { 143 | None => Ok(src_dir.clone()), 144 | Some(dir) => { 145 | let p = Path::new(dir).canonicalize()?; 146 | if !p.is_dir() { 147 | Err(anyhow::anyhow!("destination path must be directory")) 148 | } else { 149 | Ok(p) 150 | } 151 | } 152 | }?; 153 | 154 | dbg!(&src_dir, &dst_dir); 155 | 156 | let ignore_broken_rows = matches.is_present("ignore_broken_rows"); 157 | 158 | let config = Config { ignore_broken_rows }; 159 | 160 | let mut src_db = Database::new(matches.value_of("from").unwrap(), src_dir, config)?; 161 | 162 | let mut dst_db = Database::new(matches.value_of("to").unwrap(), dst_dir, config)?; 163 | 164 | copy_database(&mut *src_db, &mut *dst_db, 1000)?; 165 | 166 | Ok(()) 167 | } 168 | --------------------------------------------------------------------------------