├── .github └── workflows │ └── rust_test.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE ├── README.md ├── format.md ├── script └── coverage.sh └── src ├── library ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE ├── README.md ├── examples │ ├── benchmark.rs │ ├── cursor.rs │ ├── get_put_remove.rs │ ├── insert_simulator.rs │ ├── queue_simulator.rs │ ├── serde.rs │ └── uuid_simulator.rs ├── src │ ├── error.rs │ ├── export.rs │ ├── format.rs │ ├── lib.rs │ ├── lru.rs │ ├── page.rs │ ├── system.rs │ ├── tree.rs │ └── vfs.rs └── tests │ ├── common.rs │ ├── common_traits.rs │ ├── cursor.rs │ ├── export.rs │ ├── flush.rs │ ├── get_put_remove.rs │ ├── metadata.rs │ ├── options.rs │ ├── random_operations.rs │ ├── remove.rs │ ├── sync.rs │ └── vfs_crash.rs └── tool ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE ├── README.md └── src ├── export.rs ├── main.rs ├── repl ├── encoding.rs └── mod.rs └── verify.rs /.github/workflows/rust_test.yml: -------------------------------------------------------------------------------- 1 | name: Rust test 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Build 20 | run: cargo build --verbose 21 | - name: Run tests 22 | run: cargo test --verbose 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Rust output 2 | /target/ 3 | /Cargo.lock 4 | 5 | ## Visual Studio Code 6 | /.vscode/ 7 | 8 | ## Data from example programs 9 | /grebedb_example_data/ 10 | 11 | ## Coverage report output 12 | /grebedb.profdata 13 | /grebedb_coverage_report.html 14 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # grebedb change log 2 | 3 | (This log only contains changes for the library. Changes for the grebedb-tool crate are located in its own file.) 4 | 5 | ## Unreleased 6 | 7 | * Updated dependencies. 8 | 9 | ## 1.0.0 (2021-06-04) 10 | 11 | * Updated dependencies. 12 | * API considered stable. 13 | 14 | ## 0.3.0 (2021-04-20) 15 | 16 | ### General 17 | 18 | * When attempting to open a database in read-only or load-only mode, a lock file is no longer written to a directory if does not actually contain a database. 19 | 20 | ### API 21 | 22 | * Added `CompressionLevel::VeryLow`. 23 | * Added the `export` module, from the command line tool, to allow programmatic backups. 24 | 25 | ## 0.2.0 (2021-04-07) 26 | 27 | ### General 28 | 29 | * Internal file sync_data/sync_all() now only occurs at once during `flush()`, rather than at all the time. This is intended to perform better with OS filesystem buffers. 30 | 31 | ### API 32 | 33 | * `Cursor::set_end_range()` was removed and replaced with `Cursor::set_range()` which accepts a range. 34 | * `Database::cursor_range()` changed to accept a range. 35 | * `Database::cursor()` changed to return a `Result` instead of `Cursor` to better match `Database::cursor_range()`. 36 | * Added `Database::verify()`. 37 | * Added `Vfs::sync_file()`. 38 | 39 | ### Format 40 | 41 | * Added revision 3 for node filenames. (Backwards compatible.) 42 | 43 | ## 0.1.0 (2021-03-28) 44 | 45 | Initial version. 46 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "src/library", 4 | "src/tool", 5 | ] 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GrebeDB 2 | 3 | GrebeDB is a Rust library that provides a lightweight embedded key-value store/database backed by files in a virtual file system interface. It is intended for single-process applications that prefer a key-value database-like interface to data instead operating on file formats directly. 4 | 5 | [![Crates.io](https://img.shields.io/crates/v/grebedb)](https://crates.io/crates/grebedb) [![docs.rs](https://img.shields.io/docsrs/grebedb)](https://docs.rs/grebedb) 6 | 7 | **Note:** While the library is in a very usable state and a decent amount of effort has been made to ensure it is correct and robust, it has not been extensively tested or used in production. Use with caution and make backups regularly of important data. 8 | 9 | ## Design summary (limitations and guarantees) 10 | 11 | Since there are too many key-value stores, the design of the database is immediately described upfront for you to decide whether GrebeDB is fit for your use: 12 | 13 | * The database is implemented as a B+ tree with each node saved to a file. 14 | * Both keys and values are treated as binary data. Keys with prefixes are not optimized. 15 | * Values are stored inline with the leaf nodes. 16 | * Lazy deletion is performed. 17 | * The size of each file is not fixed and can vary significantly depending on the data stored and configuration options. 18 | * Files are stored using a virtual file system interface. The implementation can be in memory, on a real disk, or your own implementation. Performance and durability is dependent on the file system. 19 | * Operations such as `get`, `put`, `remove`, and `cursor` are provided. There's no support for traditional transactions, but the `flush` operation provides atomic saving of data. Internal consistency is provided by file copy-on-writes, incrementing revision counters, and atomic file renames. 20 | * Compression of the file with Zstandard can be used. 21 | * Concurrency is not supported. No threads are used for background tasks. 22 | 23 | For details about the file format, see [format.md](https://github.com/chfoo/grebedb/blob/main/format.md). 24 | 25 | ## Getting started 26 | 27 | Remember to add the `grebedb` crate dependency to your Cargo.toml. 28 | 29 | A GrebeDB database is stored as multiple files in a directory. The following creates a database using the given path and default options: 30 | 31 | ```rust 32 | let options = Options::default(); 33 | let mut db = Database::open_path("path/to/empty/directory/", options)?; 34 | ``` 35 | 36 | Storing, retrieving, and deleting keys is as simple as using the `get()`, `put()`, and `remove()` functions: 37 | 38 | ```rust 39 | db.put("my_key", "hello world")?; 40 | 41 | println!("The value of my_key is {:?}", db.get("my_key")?); 42 | 43 | db.remove("my_key")?; 44 | 45 | println!("The value of my_key is now {:?}", db.get("my_key")?); 46 | ``` 47 | 48 | To get all the key-values, use `cursor()`: 49 | 50 | ```rust 51 | for (key, value) in db.cursor() { 52 | println!("key = {}, value = {}", key, value); 53 | } 54 | ``` 55 | 56 | The database uses an internal cache and automatically delays writing data to the file system. If you want to ensure all changes has been persisted to the file system at a certain point, use `flush()`. This operation saves the data with atomicity, effectively emulating a transaction, before returning: 57 | 58 | ```rust 59 | db.flush()?; 60 | ``` 61 | 62 | Tip: When inserting a large amount of items, insert keys in sorted order for best performance. Inserting many random keys will be slow as it will require opening, writing, and closing the file that contains the position for it. If you need sorted ID generation, try algorithms based on Twitter Snowflake, MongoDB Object ID, or ULID. 63 | 64 | For more information, check the [examples](https://github.com/chfoo/grebedb/tree/main/src/library/examples) directory and the [API reference on docs.rs](https://docs.rs/grebedb). 65 | 66 | ### Features 67 | 68 | By default, the features are enabled: 69 | 70 | * `compression`: `zstd` crate is enabled for compression 71 | * `file_locking`: `fslock` is for cross-platform file locking 72 | * `system`: `getrandom` is a dependency for `uuid` 73 | 74 | To disable them, use `default-features = false` in your Cargo.toml file. 75 | 76 | ### Tool 77 | 78 | For a command-line tool to provide basic manipulation (such as import & export for backup) and debugging, see [grebedb-tool](https://github.com/chfoo/grebedb/tree/main/src/tool). 79 | 80 | ## Contributing 81 | 82 | Please use the GitHub Issues, Pull Requests, and Discussions if you have any problems, bug fixes, or suggestions. 83 | 84 | ### Roadmap 85 | 86 | Possible improvements: 87 | 88 | * Better API for buffer arguments. The `&mut Vec` type might be inflexible. 89 | * Reduce internal memory allocation and copying. 90 | * Reduce disk IO with smarter caching. 91 | * More tests for error cases. 92 | 93 | ### Not in scope 94 | 95 | The following non-exhaustive features are *not* in scope and likely won't be implemented: 96 | 97 | * Additions or modifications of data structures that increase complexity (such as supporting reverse cursor direction). 98 | * Traditional grouping of operations into transactions with commit and rollback. 99 | * Threads for performing automatic background work. 100 | * Async/.await support. 101 | 102 | If you need more features or require better performance, it's likely you need a more powerful database with a Rust binding such as RocksDB or a traditional database like SQLite. 103 | 104 | ## License 105 | 106 | Copyright 2021 Christopher Foo. Licensed under Mozilla Public License 2.0. 107 | -------------------------------------------------------------------------------- /format.md: -------------------------------------------------------------------------------- 1 | # GrebeDB file format 2 | 3 | This file describes the GrebeDB file format. 4 | 5 | Note: This document may not be complete and accurate. 6 | 7 | All GrebeDB files use the same format: 8 | 9 | 1. Magic bytes `0xFE 0xC7 0xF2 0xE5 0xE2 0xE5 0x00 0x00`. 10 | 2. Compression flag (1 byte) for the Page. 11 | 12 | * `0x00`: none (no compression) 13 | * `0x01`: compressed 14 | 15 | 3. Page: contains encapsulated data. 16 | 17 | * Optionally compressed using the Compression Flag. 18 | * If compressed, only Zstandard format is supported. Format is detected using magic bytes specified by the format. 19 | 20 | ## Page 21 | 22 | The Page contains the format: 23 | 24 | 1. Payload size: 8 bytes of a 64-bit big-endian unsigned integer indicating the length of the Payload. 25 | 2. Payload: MessagePack encoded data 26 | 3. Checksum: CRC-32C (Castagnoli) checksum of the Payload in 4 bytes of a 32-bit big-endian unsigned integer. 27 | 28 | ## Payload 29 | 30 | Page is MessagePack encoded data. The object is always a map with keys as strings. 31 | 32 | The metadata page has the key-value pairs: 33 | 34 | * `uuid` (16 byte binary): UUID for the instance of the database. This value is used to prevent mix up of files belonging to other instances. 35 | * `revision` (u64): Revision counter. 36 | * `id_counter` (u64): Page ID counter. It is incremented when a new ID is required. 37 | * `free_id_list` (u64 array): Unused page IDs. 38 | * `root_id` (u64, optional): Page ID containing the root node. 39 | * `auxiliary` (optional): Auxiliary metadata. 40 | 41 | The content page has the key-value pairs: 42 | 43 | * `uuid` (16 byte binary): Same usage as above. 44 | * `id` (u64): Page ID. 45 | * `revision` (u64): Revision ID. It must be equal or less to the value in the metadata page. A larger value indicates the page was not committed and should be discarded. 46 | * `deleted` (boolean): If true, there is no content and this page ID can be reused. 47 | * `content` (optional): Node data. 48 | 49 | The auxiliary metadata is a map with string keys: 50 | 51 | * `key_value_count` (u64): Number of key-value pairs stored in the tree. 52 | 53 | ## Filename 54 | 55 | The lock file uses the filename `grebedb_lock.lock`. 56 | 57 | The metadata file uses the filename `grebedb_meta.grebedb`. A copy is saved to `grebedb_meta_copy.grebedb` and a previous copy to `grebedb_meta_prev.grebedb`. 58 | 59 | For node files, filenames use the format `ID_PATH/grebedb_ID_REVISION.grebedb` where: 60 | 61 | * `ID` (16 character string): lowercase hexadecimal encoded 64-bit big-endian unsigned integer. 62 | * `REVISION` (1 character string): digit `0`, `1`, `2`. Implementations use the page that contains the greatest valid revision ID. 63 | * `ID_PATH`: the first 14 characters of ID split into 7 directories (for example, `ab/cd/ef/01/23/45/67`). 64 | 65 | ## Node 66 | 67 | Nodes are an externally tagged enum represented as one of: 68 | 69 | * String `empty_root` indicating an empty tree. 70 | * Map with one of the string keys: 71 | * `internal`: Internal node 72 | * `leaf`: Leaf node 73 | 74 | ### Internal node 75 | 76 | Internal nodes are a map with key-value pairs: 77 | 78 | * `keys` (array of binary): Keys in an B+ tree internal node. 79 | * `children` (array of u64): Array of child node page IDs. 80 | 81 | ### Leaf node 82 | 83 | Leaf nodes are a map with key-value pairs: 84 | 85 | * `keys` (array of binary): Keys in a B+ tree leaf node. 86 | * `values` (array of binary): Contains the values. 87 | -------------------------------------------------------------------------------- /script/coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## Runs code coverage using the tests. 3 | 4 | set -e 5 | shopt -s globstar 6 | 7 | ## Before running, install the needed tools: 8 | 9 | # rustup +nightly component add llvm-tools-preview 10 | # cargo +nightly install cargo-binutils 11 | # cargo +nightly install rustfilt 12 | 13 | cargo +nightly profdata -- --help 14 | 15 | RUSTFLAGS="-Zinstrument-coverage" \ 16 | LLVM_PROFILE_FILE="grebedb-%m.profraw" \ 17 | cargo +nightly test --tests 18 | 19 | cargo +nightly profdata -- merge \ 20 | -sparse **/grebedb-*.profraw -o grebedb.profdata 21 | 22 | rm **/grebedb-*.profraw 23 | 24 | FILES=`RUSTFLAGS="-Zinstrument-coverage" \ 25 | cargo +nightly test --tests --no-run --message-format=json \ 26 | | jq -r "select(.profile.test == true) | .filenames[]" \ 27 | | grep -v dSYM` 28 | 29 | FILE_ARGS="" 30 | 31 | for file in $FILES; do 32 | FILE_ARGS+=" --object $file" 33 | done 34 | 35 | cargo +nightly cov -- report \ 36 | --use-color --ignore-filename-regex='/.cargo/registry' \ 37 | --instr-profile=grebedb.profdata \ 38 | $FILE_ARGS 39 | 40 | cargo +nightly cov -- show \ 41 | --use-color --ignore-filename-regex='/.cargo/registry' \ 42 | --instr-profile=grebedb.profdata \ 43 | $FILE_ARGS \ 44 | --show-instantiations \ 45 | --show-line-counts-or-regions \ 46 | --Xdemangler=rustfilt --format html > grebedb_coverage_report.html 47 | 48 | echo "Report placed in grebedb_coverage_report.html" 49 | -------------------------------------------------------------------------------- /src/library/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ../../CHANGELOG.md -------------------------------------------------------------------------------- /src/library/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "grebedb" 3 | version = "1.0.0" 4 | authors = ["Christopher Foo "] 5 | edition = "2021" 6 | description = "Lightweight embedded key-value store/database backed by files in a virtual file system interface" 7 | repository = "https://github.com/chfoo/grebedb/" 8 | license = "MPL-2.0" 9 | keywords = ["database", "key-value-store"] 10 | categories = ["database-implementations"] 11 | 12 | [features] 13 | default = ["compression", "file_locking", "system"] 14 | compression = ["zstd"] 15 | file_locking = ["fslock"] 16 | system = ["uuid/v4"] 17 | 18 | [dependencies] 19 | crc32c = "0.6" 20 | data-encoding = "2.3" 21 | relative-path = "1.4" 22 | rmp-serde = "0.15" 23 | serde = { version = "1.0", features = ["derive"] } 24 | serde_bytes = "0.11" 25 | serde_json = "1.0" 26 | thiserror = "1.0" 27 | uuid = { version = "0.8", features = ["serde"] } 28 | vfs = "0.5" 29 | zstd = { version = "0.9", optional = true } 30 | 31 | [target.'cfg(unix)'.dependencies] 32 | fslock = { version = "0.2", optional = true } 33 | 34 | [target.'cfg(windows)'.dependencies] 35 | fslock = { version = "0.2", optional = true } 36 | 37 | [dev-dependencies] 38 | anyhow = "1.0" 39 | clap = "2.33" 40 | indexmap = "1.6" 41 | paste = "1.0" 42 | rand = "0.8" 43 | rand_xorshift = "0.3" 44 | statrs = "0.15" 45 | strkey = "0.1" 46 | tempfile = "3.2" 47 | -------------------------------------------------------------------------------- /src/library/LICENSE: -------------------------------------------------------------------------------- 1 | ../../LICENSE -------------------------------------------------------------------------------- /src/library/README.md: -------------------------------------------------------------------------------- 1 | ../../README.md -------------------------------------------------------------------------------- /src/library/examples/benchmark.rs: -------------------------------------------------------------------------------- 1 | // Test database operation speeds 2 | 3 | use std::time::{Duration, Instant}; 4 | 5 | use clap::{App, Arg}; 6 | use grebedb::{CompressionLevel, Database, Error, Options, SyncOption}; 7 | use rand::{Rng, RngCore, SeedableRng}; 8 | use rand_xorshift::XorShiftRng; 9 | use statrs::statistics::{Data, Median}; 10 | 11 | fn main() -> Result<(), Error> { 12 | let args = App::new("GrebeDB benchmark") 13 | .arg( 14 | Arg::with_name("sync") 15 | .long("sync") 16 | .possible_values(&["none", "data", "all"]) 17 | .default_value("data"), 18 | ) 19 | .arg( 20 | Arg::with_name("store") 21 | .long("store") 22 | .possible_values(&["disk", "memory"]) 23 | .default_value("disk"), 24 | ) 25 | .arg( 26 | Arg::with_name("page_cache_size") 27 | .long("page-cache-size") 28 | .default_value("64"), 29 | ) 30 | .arg( 31 | Arg::with_name("keys_per_node") 32 | .long("keys-per-node") 33 | .default_value("1024"), 34 | ) 35 | .arg( 36 | Arg::with_name("compression_level") 37 | .long("compression-level") 38 | .possible_values(&["none", "verylow", "low", "medium", "high"]) 39 | .default_value("low"), 40 | ) 41 | .arg( 42 | Arg::with_name("max_batch_size") 43 | .long("max-batch-size") 44 | .default_value("2000"), 45 | ) 46 | .arg( 47 | Arg::with_name("batches") 48 | .long("batches") 49 | .default_value("100"), 50 | ) 51 | .arg(Arg::with_name("rounds").long("rounds").default_value("3")) 52 | .get_matches(); 53 | 54 | let options = Options { 55 | automatic_flush: false, 56 | file_sync: match args.value_of("sync").unwrap() { 57 | "none" => SyncOption::None, 58 | "data" => SyncOption::Data, 59 | "all" => SyncOption::All, 60 | _ => unreachable!(), 61 | }, 62 | page_cache_size: args.value_of("page_cache_size").unwrap().parse().unwrap(), 63 | keys_per_node: args.value_of("keys_per_node").unwrap().parse().unwrap(), 64 | compression_level: match args.value_of("compression_level").unwrap() { 65 | "none" => CompressionLevel::None, 66 | "verylow" => CompressionLevel::VeryLow, 67 | "low" => CompressionLevel::Low, 68 | "medium" => CompressionLevel::Medium, 69 | "high" => CompressionLevel::High, 70 | _ => unreachable!(), 71 | }, 72 | ..Default::default() 73 | }; 74 | 75 | let (mut db_sequential, mut db_random) = if args.value_of("store").unwrap() == "disk" { 76 | let path_sequential = 77 | std::path::PathBuf::from("grebedb_example_data/benchmark/db_sequential"); 78 | let path_random = std::path::PathBuf::from("grebedb_example_data/benchmark/db_random"); 79 | 80 | std::fs::create_dir_all(&path_sequential)?; 81 | std::fs::create_dir_all(&path_random)?; 82 | 83 | ( 84 | Database::open_path(path_sequential, options.clone())?, 85 | Database::open_path(path_random, options)?, 86 | ) 87 | } else { 88 | ( 89 | Database::open_memory(options.clone())?, 90 | Database::open_memory(options)?, 91 | ) 92 | }; 93 | 94 | let max_batch_size: usize = args.value_of("max_batch_size").unwrap().parse().unwrap(); 95 | let batches: usize = args.value_of("batches").unwrap().parse().unwrap(); 96 | let rounds: usize = args.value_of("rounds").unwrap().parse().unwrap(); 97 | let mut id = 0u64; 98 | let mut seed = 1u64; 99 | 100 | for batch_id in 0usize..batches { 101 | let batch_size = ((batch_id + 1).pow(4)).min(max_batch_size); 102 | let mut stats_sequential = Stats::default(); 103 | 104 | for _ in 0..rounds { 105 | insert_sequential(&mut db_sequential, batch_size, id, &mut stats_sequential)?; 106 | id += batch_size as u64; 107 | } 108 | 109 | let mut stats_random = Stats::default(); 110 | 111 | for _ in 0..rounds { 112 | insert_random(&mut db_random, batch_size, seed, &mut stats_random)?; 113 | seed ^= mix(seed); 114 | } 115 | 116 | println!( 117 | "Batch size {}, rounds {}, sequential total {}, random total {}", 118 | batch_size, 119 | rounds, 120 | db_sequential.metadata().key_value_count(), 121 | db_random.metadata().key_value_count(), 122 | ); 123 | println!(" Sequential"); 124 | 125 | let median = stats_sequential.insert_median(); 126 | let rate = 1.0 / median * batch_size as f64; 127 | println!(" insert\t{:.6} s/batch\t{:.2} pairs/s", median, rate); 128 | 129 | let median = stats_sequential.flush_median(); 130 | let rate = 1.0 / median * batch_size as f64; 131 | println!(" flush\t{:.6} s/batch\t{:.2} pairs/s", median, rate); 132 | 133 | let median = stats_sequential.read_median(); 134 | let rate = 1.0 / median * batch_size as f64; 135 | println!(" read\t{:.6} s/batch\t{:.2} pairs/s", median, rate); 136 | 137 | println!(" Random"); 138 | 139 | let median = stats_random.insert_median(); 140 | let rate = 1.0 / median * batch_size as f64; 141 | println!(" insert\t{:.6} s/batch\t{:.2} pairs/s", median, rate); 142 | 143 | let median = stats_random.flush_median(); 144 | let rate = 1.0 / median * batch_size as f64; 145 | println!(" flush\t{:.6} s/batch\t{:.2} pairs/s", median, rate); 146 | 147 | let median = stats_random.read_median(); 148 | let rate = 1.0 / median * batch_size as f64; 149 | println!(" read\t{:.6} s/batch\t{:.2} pairs/s", median, rate); 150 | 151 | std::thread::sleep(Duration::from_secs_f32(0.1)); 152 | } 153 | 154 | Ok(()) 155 | } 156 | 157 | #[derive(Default)] 158 | struct Stats { 159 | insert: Vec, 160 | read: Vec, 161 | flush: Vec, 162 | } 163 | 164 | impl Stats { 165 | fn insert_median(&mut self) -> f64 { 166 | self.insert.sort_unstable(); 167 | let times: Vec = self.insert.iter().map(|item| item.as_secs_f64()).collect(); 168 | let times = Data::new(times); 169 | times.median() 170 | } 171 | 172 | fn read_median(&mut self) -> f64 { 173 | self.read.sort_unstable(); 174 | let times: Vec = self.read.iter().map(|item| item.as_secs_f64()).collect(); 175 | let times = Data::new(times); 176 | times.median() 177 | } 178 | 179 | fn flush_median(&mut self) -> f64 { 180 | self.flush.sort_unstable(); 181 | let times: Vec = self.flush.iter().map(|item| item.as_secs_f64()).collect(); 182 | let times = Data::new(times); 183 | times.median() 184 | } 185 | } 186 | 187 | fn insert_sequential( 188 | db: &mut Database, 189 | batch_size: usize, 190 | id_offset: u64, 191 | stats: &mut Stats, 192 | ) -> Result<(), Error> { 193 | let time_start = Instant::now(); 194 | 195 | for id in 0..batch_size { 196 | let id = id as u64 + id_offset; 197 | let mut rng = XorShiftRng::seed_from_u64(id); 198 | 199 | let key = format!("{:016x}", id); 200 | let mut value = vec![0u8; 1024]; 201 | rng.fill_bytes(&mut value); 202 | 203 | db.put(key, value)?; 204 | } 205 | 206 | stats.insert.push(time_start.elapsed()); 207 | 208 | let time_start = Instant::now(); 209 | 210 | db.flush()?; 211 | 212 | stats.flush.push(time_start.elapsed()); 213 | 214 | let time_start = Instant::now(); 215 | 216 | for id in 0..batch_size { 217 | let id = id as u64 + id_offset; 218 | 219 | let key = format!("{:016x}", id); 220 | 221 | db.get(key)?; 222 | } 223 | 224 | stats.read.push(time_start.elapsed()); 225 | 226 | Ok(()) 227 | } 228 | 229 | fn insert_random( 230 | db: &mut Database, 231 | batch_size: usize, 232 | seed: u64, 233 | stats: &mut Stats, 234 | ) -> Result<(), Error> { 235 | let time_start = Instant::now(); 236 | 237 | for id in 0..batch_size { 238 | let id = id as u64; 239 | let mut rng = XorShiftRng::seed_from_u64(mix(id) ^ seed); 240 | 241 | let key = format!("{:016x}", rng.gen::()); 242 | let mut value = vec![0u8; 1024]; 243 | rng.fill_bytes(&mut value); 244 | 245 | db.put(key, value)?; 246 | } 247 | 248 | stats.insert.push(time_start.elapsed()); 249 | 250 | let time_start = Instant::now(); 251 | 252 | db.flush()?; 253 | 254 | stats.flush.push(time_start.elapsed()); 255 | 256 | let time_start = Instant::now(); 257 | 258 | for id in 0..batch_size { 259 | let id = id as u64; 260 | let mut rng = XorShiftRng::seed_from_u64(mix(id) ^ seed); 261 | 262 | let key = format!("{:016x}", rng.gen::()); 263 | 264 | db.get(key)?; 265 | } 266 | 267 | stats.read.push(time_start.elapsed()); 268 | 269 | Ok(()) 270 | } 271 | 272 | fn mix(mut value: u64) -> u64 { 273 | value ^= 0xc001cafe; 274 | value <<= 2; 275 | value = value.wrapping_mul(0xc001cafe); 276 | value = value.wrapping_add(1); 277 | value 278 | } 279 | -------------------------------------------------------------------------------- /src/library/examples/cursor.rs: -------------------------------------------------------------------------------- 1 | // Demonstrates the use of a cursor. 2 | 3 | use grebedb::{Database, Options}; 4 | 5 | fn main() -> Result<(), grebedb::Error> { 6 | let options = Options::default(); 7 | let mut db = Database::open_memory(options)?; 8 | 9 | for number in 0..10 { 10 | db.put( 11 | format!("key:{:04x}", number), 12 | format!("hello world {}!", number), 13 | )?; 14 | } 15 | 16 | println!("Printing all the key-values..."); 17 | 18 | for (key, value) in db.cursor()? { 19 | println!( 20 | "Cursor key = {}, value = {}", 21 | std::str::from_utf8(&key).unwrap(), 22 | std::str::from_utf8(&value).unwrap() 23 | ); 24 | } 25 | 26 | println!("Printing all the key-values starting from [key:0004, key:0008) ..."); 27 | 28 | let cursor = db.cursor_range("key:0004".."key:0008")?; 29 | 30 | for (key, value) in cursor { 31 | println!( 32 | "Cursor key = {}, value = {}", 33 | std::str::from_utf8(&key).unwrap(), 34 | std::str::from_utf8(&value).unwrap() 35 | ); 36 | } 37 | 38 | Ok(()) 39 | } 40 | -------------------------------------------------------------------------------- /src/library/examples/get_put_remove.rs: -------------------------------------------------------------------------------- 1 | // Demonstrates key-value operations. 2 | 3 | use grebedb::{Database, Options}; 4 | 5 | fn main() -> Result<(), grebedb::Error> { 6 | // A directory is used to store a GrebeDB database. 7 | let path = std::path::PathBuf::from("grebedb_example_data/get_put_remove/"); 8 | 9 | std::fs::create_dir_all(&path)?; 10 | 11 | let options = Options::default(); 12 | let mut db = Database::open_path(path, options)?; 13 | 14 | // Store some key-values 15 | db.put("key:1", "hello world 1!")?; 16 | db.put("key:2", "hello world 2!")?; 17 | db.put("key:3", "hello world 3!")?; 18 | 19 | // Getting some values 20 | println!("The value of key1 is {:?}", db.get("key:1")?); 21 | println!("The value of key2 is {:?}", db.get("key:2")?); 22 | println!("The value of key3 is {:?}", db.get("key:3")?); 23 | 24 | // Overwrite the value 25 | db.put("key:2", "new value")?; 26 | 27 | println!("The value of key2 is {:?}", db.get("key:2")?); 28 | 29 | // Deleting a key-value 30 | db.remove("key:2")?; 31 | 32 | println!("The value of key2 is {:?}", db.get("key:2")?); 33 | 34 | // Data stored in internal cache is automatically written to the 35 | // file system when needed, but this only happens when a database 36 | // operation function is called, with automatic flushing, 37 | // or when the database is dropped. 38 | // 39 | // If you need to ensure all data is persisted at a given time, 40 | // you can call flush() manually. This method effectively emulates 41 | // a transaction as it saves the database changes with atomicity 42 | // before returning. 43 | db.flush()?; 44 | 45 | Ok(()) 46 | } 47 | -------------------------------------------------------------------------------- /src/library/examples/insert_simulator.rs: -------------------------------------------------------------------------------- 1 | // Sample program that inserts monotonic-like key-values infinitely. 2 | 3 | use std::time::{Duration, SystemTime}; 4 | 5 | use clap::{App, Arg}; 6 | use grebedb::{Database, Options}; 7 | use rand::{RngCore, SeedableRng}; 8 | use rand_xorshift::XorShiftRng; 9 | 10 | fn main() -> Result<(), grebedb::Error> { 11 | let matches = App::new("GrebeDB monotonic-like insert simulator") 12 | .arg( 13 | Arg::with_name("delay") 14 | .long("delay") 15 | .takes_value(true) 16 | .default_value("0.5") 17 | .help("Delay between insert batches in seconds."), 18 | ) 19 | .get_matches(); 20 | 21 | let delay = matches.value_of("delay").unwrap(); 22 | let delay = delay.parse::().unwrap(); 23 | 24 | let path = std::path::PathBuf::from("grebedb_example_data/insert_simulator/"); 25 | 26 | std::fs::create_dir_all(&path)?; 27 | 28 | let options = Options::default(); 29 | let mut db = Database::open_path(path, options)?; 30 | 31 | let mut counter = 0u64; 32 | 33 | loop { 34 | for _ in 0..100 { 35 | let duration = SystemTime::now() 36 | .duration_since(SystemTime::UNIX_EPOCH) 37 | .unwrap(); 38 | let ms = duration.as_micros() as u64; 39 | let key = format!("{:016x}{:016x}", ms, counter); 40 | 41 | let mut rng = XorShiftRng::seed_from_u64(counter); 42 | let mut buffer = vec![0u8; 1024]; 43 | rng.fill_bytes(&mut buffer); 44 | 45 | db.put(key, buffer)?; 46 | 47 | counter += 1; 48 | } 49 | 50 | if delay > 0.0 { 51 | std::thread::sleep(Duration::from_secs_f32(delay)) 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/library/examples/queue_simulator.rs: -------------------------------------------------------------------------------- 1 | // Sample program that inserts and removes key-values like a deque. 2 | 3 | use std::time::{Duration, SystemTime}; 4 | 5 | use clap::{App, Arg}; 6 | use rand::{Rng, RngCore, SeedableRng}; 7 | use rand_xorshift::XorShiftRng; 8 | 9 | fn main() -> Result<(), grebedb::Error> { 10 | let matches = App::new("GrebeDB insert simulator") 11 | .arg( 12 | Arg::with_name("delay") 13 | .long("delay") 14 | .takes_value(true) 15 | .default_value("0.5") 16 | .help("Delay between insert/remove batches in seconds."), 17 | ) 18 | .get_matches(); 19 | 20 | let delay = matches.value_of("delay").unwrap(); 21 | let delay = delay.parse::().unwrap(); 22 | 23 | let path = std::path::PathBuf::from("grebedb_example_data/queue_simulator/"); 24 | 25 | std::fs::create_dir_all(&path)?; 26 | 27 | let options = grebedb::Options::default(); 28 | let mut db = grebedb::Database::open_path(path, options)?; 29 | 30 | let mut counter = 0u64; 31 | 32 | loop { 33 | let mut rng = XorShiftRng::seed_from_u64(counter); 34 | 35 | for _ in 0..rng.gen_range(50..150) { 36 | let duration = SystemTime::now() 37 | .duration_since(SystemTime::UNIX_EPOCH) 38 | .unwrap(); 39 | let ms = duration.as_micros() as u64; 40 | let key = format!("{:016x}{:016x}", ms, counter); 41 | 42 | let mut rng = XorShiftRng::seed_from_u64(counter); 43 | let mut buffer = vec![0u8; 1024]; 44 | rng.fill_bytes(&mut buffer); 45 | 46 | db.put(key, buffer)?; 47 | 48 | counter += 1; 49 | } 50 | 51 | let mut keys_to_remove = Vec::new(); 52 | let mut cursor = db.cursor()?; 53 | 54 | for _ in 0..100 { 55 | if let Some((key, _value)) = cursor.next() { 56 | keys_to_remove.push(key); 57 | } else { 58 | break; 59 | } 60 | } 61 | 62 | for key in keys_to_remove { 63 | db.remove(key)?; 64 | } 65 | 66 | if delay > 0.0 { 67 | std::thread::sleep(Duration::from_secs_f32(delay)) 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/library/examples/serde.rs: -------------------------------------------------------------------------------- 1 | // Example showing how to store data using Serde. 2 | // 3 | // In this example, standalone functions are used. For simple applications, 4 | // this is usually sufficient. In more complex cases, you may choose to write 5 | // a wrapper around Database and Cursor instead. 6 | // 7 | // For details about Serde, please read the documentation at https://serde.rs/ 8 | 9 | use grebedb::{Database, Options}; 10 | 11 | fn main() -> anyhow::Result<()> { 12 | let path = std::path::PathBuf::from("grebedb_example_data/serde/"); 13 | 14 | std::fs::create_dir_all(&path)?; 15 | 16 | let options = Options::default(); 17 | let mut db = Database::open_path(path, options)?; 18 | 19 | put_serde( 20 | &mut db, 21 | ("document", 1u32), 22 | MyDocument { 23 | id: 1, 24 | name: "hello".to_string(), 25 | content: "Hello world!".to_string(), 26 | }, 27 | )?; 28 | 29 | let value = get_serde::<_, MyDocument>(&mut db, ("document", 1u32))?; 30 | 31 | println!("The value of document 1 is {:?}", value); 32 | 33 | db.flush()?; 34 | 35 | Ok(()) 36 | } 37 | 38 | fn put_serde(database: &mut Database, key: K, value: V) -> anyhow::Result<()> 39 | where 40 | K: serde::Serialize, 41 | V: serde::Serialize, 42 | { 43 | let key = strkey::to_vec(&key)?; 44 | let value = serde_json::to_vec(&value)?; 45 | 46 | database.put(key, value)?; 47 | 48 | Ok(()) 49 | } 50 | 51 | fn get_serde(database: &mut Database, key: K) -> anyhow::Result> 52 | where 53 | K: serde::Serialize, 54 | V: serde::de::DeserializeOwned, 55 | { 56 | let key = strkey::to_vec(&key)?; 57 | let value = database.get(key)?; 58 | 59 | if let Some(value) = value { 60 | Ok(Some(serde_json::from_slice(&value)?)) 61 | } else { 62 | Ok(None) 63 | } 64 | } 65 | 66 | #[derive(Debug, serde::Serialize, serde::Deserialize)] 67 | struct MyDocument { 68 | id: u32, 69 | name: String, 70 | content: String, 71 | } 72 | -------------------------------------------------------------------------------- /src/library/examples/uuid_simulator.rs: -------------------------------------------------------------------------------- 1 | // Sample program that inserts UUID (random ordered) key-values infinitely. 2 | 3 | use std::time::Duration; 4 | 5 | use clap::{App, Arg}; 6 | use grebedb::{Database, Options}; 7 | use rand::{RngCore, SeedableRng}; 8 | use rand_xorshift::XorShiftRng; 9 | use uuid::Uuid; 10 | 11 | fn main() -> Result<(), grebedb::Error> { 12 | let matches = App::new("GrebeDB UUID insert simulator") 13 | .arg( 14 | Arg::with_name("delay") 15 | .long("delay") 16 | .takes_value(true) 17 | .default_value("0.5") 18 | .help("Delay between insert batches in seconds."), 19 | ) 20 | .get_matches(); 21 | 22 | let delay = matches.value_of("delay").unwrap(); 23 | let delay = delay.parse::().unwrap(); 24 | 25 | let path = std::path::PathBuf::from("grebedb_example_data/uuid_simulator/"); 26 | 27 | std::fs::create_dir_all(&path)?; 28 | 29 | let options = Options::default(); 30 | let mut db = Database::open_path(path, options)?; 31 | 32 | let mut counter = 0u64; 33 | 34 | loop { 35 | for _ in 0..100 { 36 | let id = Uuid::new_v4(); 37 | let key = id.as_bytes(); 38 | 39 | let mut rng = XorShiftRng::seed_from_u64(counter); 40 | let mut buffer = vec![0u8; 1024]; 41 | rng.fill_bytes(&mut buffer); 42 | 43 | db.put(key.to_vec(), buffer)?; 44 | 45 | counter += 1; 46 | } 47 | 48 | if delay > 0.0 { 49 | std::thread::sleep(Duration::from_secs_f32(delay)) 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/library/src/error.rs: -------------------------------------------------------------------------------- 1 | //! Errors 2 | 3 | /// Error type returned by database operations. 4 | #[derive(thiserror::Error, Debug)] 5 | pub enum Error { 6 | /// Support for compression is not available due to a disabled feature. 7 | #[error("compression support not available")] 8 | CompressionUnavailable, 9 | 10 | /// Support for file locking is not available due to a disabled feature. 11 | #[error("file locking support not available")] 12 | FileLockingUnavailable, 13 | 14 | /// Provided configuration is invalid. 15 | #[error("invalid configuration: {message}")] 16 | InvalidConfig { 17 | /// Custom message. 18 | message: &'static str, 19 | }, 20 | 21 | /// A calculated checksum does not match. 22 | #[error("bad checksum: {path}")] 23 | BadChecksum { 24 | /// Path to file with bad checksum. 25 | path: String, 26 | }, 27 | 28 | /// A file contained unexpected data or is not a database file. 29 | #[error("invalid file format: {message}, {path}")] 30 | InvalidFileFormat { 31 | /// Path to file. 32 | path: String, 33 | /// Custom message. 34 | message: &'static str, 35 | }, 36 | 37 | /// The metadata file contains invalid data. 38 | #[error("invalid page metadata: {message}")] 39 | InvalidMetadata { 40 | /// Custom message. 41 | message: &'static str, 42 | }, 43 | 44 | /// A page file contains invalid data. 45 | #[error("invalid page data: {message}, {page}")] 46 | InvalidPageData { 47 | /// Page ID. 48 | page: u64, 49 | /// Custom message 50 | message: &'static str, 51 | }, 52 | 53 | /// An execution or resource limit was exceeded. 54 | /// 55 | /// This error occurs if the tree is corrupted in such a way that it 56 | /// causes infinite loops. 57 | #[error("execution or resource limit exceeded")] 58 | LimitExceeded, 59 | 60 | /// Database is closed. 61 | /// 62 | /// This occurs if the database experienced an error and will refuse to 63 | /// process future operations to prevent further corruption. 64 | #[error("database closed")] 65 | Closed, 66 | 67 | /// Database is locked. 68 | /// 69 | /// Another process has locked the database for reading/writing. 70 | #[error("database locked")] 71 | Locked, 72 | 73 | /// A modification to a database opened in read-only mode was requested. 74 | #[error("database read only")] 75 | ReadOnly, 76 | 77 | /// Other std IO error. 78 | #[error(transparent)] 79 | Io(#[from] std::io::Error), 80 | 81 | /// Other internal errors. 82 | #[error(transparent)] 83 | Other(Box), 84 | } 85 | 86 | impl From for Error { 87 | fn from(error: vfs::VfsError) -> Self { 88 | Self::Other(Box::new(error)) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/library/src/export.rs: -------------------------------------------------------------------------------- 1 | //! Export and import database key-value pairs. 2 | //! 3 | //! The functions allow saving database contents into another file 4 | //! which can be used for migrating data or for backup purposes. 5 | //! 6 | //! The export file format is a JSON text sequence (RFC 7464). 7 | 8 | const RECORD_SEPARATOR: u8 = 0x1e; 9 | const NEWLINE: u8 = 0x0a; 10 | 11 | #[derive(Serialize, Deserialize)] 12 | #[serde(rename_all = "snake_case")] 13 | enum Row { 14 | Metadata(MetadataRow), 15 | KeyValue(KeyValueRow), 16 | Eof, 17 | } 18 | 19 | use std::io::{BufRead, Write}; 20 | 21 | use serde::{Deserialize, Deserializer, Serialize, Serializer}; 22 | 23 | use crate::{Database, Error}; 24 | 25 | /// Import and export errors. 26 | #[derive(thiserror::Error, Debug)] 27 | pub enum ExportError { 28 | /// Missing record separator. 29 | /// 30 | /// File is not JSON text sequence formatted. 31 | #[error("missing record separator")] 32 | MissingRecordSeparator, 33 | 34 | /// Duplicate header. 35 | /// 36 | /// File unexpectedly contains another file. 37 | #[error("duplicate header")] 38 | DuplicateHeader, 39 | 40 | /// Header not found 41 | /// 42 | /// Beginning of the file is missing. 43 | #[error("header not found")] 44 | HeaderNotFound, 45 | 46 | /// Bad checksum. 47 | /// 48 | /// Data is corrupted. 49 | #[error("bad checksum, {column}, row = {row}")] 50 | BadChecksum { 51 | /// Located at key or value 52 | column: &'static str, 53 | /// Row index (0 based) 54 | row: u64, 55 | }, 56 | 57 | /// Duplicate footer. 58 | /// 59 | /// File unexpectedly contains another file. 60 | #[error("duplicate footer")] 61 | DuplicateFooter, 62 | 63 | /// Footer not found. 64 | /// 65 | /// The file is incomplete. 66 | #[error("footer not found")] 67 | FooterNotFound, 68 | 69 | /// Unexpected end of file. 70 | /// 71 | /// The file is incomplete. 72 | #[error("unexpected end of file")] 73 | UnexpectedEof, 74 | } 75 | 76 | impl From for Error { 77 | fn from(error: ExportError) -> Self { 78 | Self::Other(Box::new(error)) 79 | } 80 | } 81 | 82 | impl From for Error { 83 | fn from(error: serde_json::Error) -> Self { 84 | Self::Other(Box::new(error)) 85 | } 86 | } 87 | 88 | #[derive(Default, Serialize, Deserialize)] 89 | struct MetadataRow { 90 | pub key_value_count: u64, 91 | } 92 | 93 | #[derive(Default, Serialize, Deserialize)] 94 | struct KeyValueRow { 95 | #[serde(serialize_with = "vec_to_hex")] 96 | #[serde(deserialize_with = "hex_to_vec")] 97 | pub key: Vec, 98 | 99 | #[serde(serialize_with = "vec_to_hex")] 100 | #[serde(deserialize_with = "hex_to_vec")] 101 | pub value: Vec, 102 | 103 | pub index: u64, 104 | 105 | pub key_crc32c: u32, 106 | pub value_crc32c: u32, 107 | } 108 | 109 | fn vec_to_hex(vec: &[u8], serializer: S) -> Result 110 | where 111 | S: Serializer, 112 | { 113 | serializer.serialize_str(&data_encoding::HEXUPPER.encode(vec)) 114 | } 115 | 116 | fn hex_to_vec<'de, D>(deserializer: D) -> Result, D::Error> 117 | where 118 | D: Deserializer<'de>, 119 | { 120 | let s = <&str>::deserialize(deserializer)?; 121 | match data_encoding::HEXUPPER.decode(s.as_bytes()) { 122 | Ok(value) => Ok(value), 123 | Err(error) => Err(serde::de::Error::custom(format!("{:?}", error))), 124 | } 125 | } 126 | 127 | struct ImportReader<'a, R: BufRead> { 128 | database: &'a mut Database, 129 | input_file: &'a mut R, 130 | header_found: bool, 131 | footer_found: bool, 132 | } 133 | 134 | impl<'a, R: BufRead> ImportReader<'a, R> { 135 | fn new(input_file: &'a mut R, database: &'a mut Database) -> Self { 136 | Self { 137 | database, 138 | input_file, 139 | header_found: false, 140 | footer_found: false, 141 | } 142 | } 143 | 144 | fn import(&mut self, mut progress: C) -> Result<(), Error> 145 | where 146 | C: FnMut(u64), 147 | { 148 | let mut buffer = Vec::new(); 149 | let mut counter = 0u64; 150 | 151 | while self.read_record_separator()? { 152 | buffer.clear(); 153 | self.input_file.read_until(NEWLINE, &mut buffer)?; 154 | 155 | if buffer.last().cloned().unwrap_or(0) != NEWLINE { 156 | return Err(ExportError::UnexpectedEof.into()); 157 | } 158 | 159 | let row: Row = serde_json::from_slice(&buffer)?; 160 | 161 | match row { 162 | Row::Metadata(row) => { 163 | self.process_metadata(&row)?; 164 | } 165 | Row::KeyValue(row) => { 166 | self.process_key_value_row(row)?; 167 | counter += 1; 168 | progress(counter); 169 | } 170 | Row::Eof => { 171 | self.process_eof_row()?; 172 | } 173 | } 174 | } 175 | 176 | self.database.flush()?; 177 | self.validate_footer()?; 178 | 179 | Ok(()) 180 | } 181 | 182 | fn read_record_separator(&mut self) -> Result { 183 | let mut record_flag = [0u8; 1]; 184 | 185 | if let Err(error) = self.input_file.read_exact(&mut record_flag) { 186 | if let std::io::ErrorKind::UnexpectedEof = error.kind() { 187 | return Ok(false); 188 | } else { 189 | return Err(error.into()); 190 | } 191 | } 192 | 193 | if record_flag[0] != RECORD_SEPARATOR { 194 | Err(ExportError::MissingRecordSeparator.into()) 195 | } else { 196 | Ok(true) 197 | } 198 | } 199 | 200 | fn process_metadata(&mut self, _row: &MetadataRow) -> Result<(), Error> { 201 | if self.header_found { 202 | return Err(ExportError::DuplicateHeader.into()); 203 | } 204 | 205 | self.header_found = true; 206 | 207 | Ok(()) 208 | } 209 | 210 | fn process_key_value_row(&mut self, row: KeyValueRow) -> Result<(), Error> { 211 | if !self.header_found { 212 | return Err(ExportError::HeaderNotFound.into()); 213 | } 214 | 215 | let key_crc = crc32c::crc32c(&row.key); 216 | 217 | if key_crc != row.key_crc32c { 218 | return Err(ExportError::BadChecksum { 219 | column: "key", 220 | row: row.index, 221 | } 222 | .into()); 223 | } 224 | 225 | let value_crc = crc32c::crc32c(&row.value); 226 | 227 | if value_crc != row.value_crc32c { 228 | return Err(ExportError::BadChecksum { 229 | column: "value", 230 | row: row.index, 231 | } 232 | .into()); 233 | } 234 | 235 | self.database.put(row.key, row.value)?; 236 | 237 | Ok(()) 238 | } 239 | 240 | fn process_eof_row(&mut self) -> Result<(), Error> { 241 | if self.footer_found { 242 | return Err(ExportError::DuplicateFooter.into()); 243 | } 244 | 245 | self.footer_found = true; 246 | 247 | Ok(()) 248 | } 249 | 250 | fn validate_footer(&self) -> Result<(), Error> { 251 | if !self.footer_found { 252 | Err(ExportError::FooterNotFound.into()) 253 | } else { 254 | Ok(()) 255 | } 256 | } 257 | } 258 | 259 | struct ExportWriter<'a, W: Write> { 260 | database: Option<&'a mut Database>, 261 | counter: u64, 262 | output_file: &'a mut W, 263 | } 264 | 265 | impl<'a, W: Write> ExportWriter<'a, W> { 266 | fn new(output_file: &'a mut W, database: &'a mut Database) -> Self { 267 | Self { 268 | database: Some(database), 269 | counter: 0, 270 | output_file, 271 | } 272 | } 273 | 274 | fn export(&mut self, mut progress: C) -> Result<(), Error> 275 | where 276 | C: FnMut(u64), 277 | { 278 | self.write_header()?; 279 | self.write_key_values(&mut progress)?; 280 | self.write_footer()?; 281 | 282 | Ok(()) 283 | } 284 | 285 | fn write_row(&mut self, row: T) -> Result<(), Error> 286 | where 287 | T: Serialize, 288 | { 289 | self.output_file.write_all(&[RECORD_SEPARATOR])?; 290 | 291 | let mut serializer = serde_json::Serializer::new(&mut self.output_file); 292 | row.serialize(&mut serializer)?; 293 | 294 | self.output_file.write_all(&[NEWLINE])?; 295 | 296 | Ok(()) 297 | } 298 | 299 | fn write_header(&mut self) -> Result<(), Error> { 300 | let database = self.database.take().unwrap(); 301 | 302 | let header_row = MetadataRow { 303 | key_value_count: database.metadata().key_value_count(), 304 | }; 305 | 306 | self.write_row(Row::Metadata(header_row))?; 307 | 308 | self.database = Some(database); 309 | 310 | Ok(()) 311 | } 312 | 313 | fn write_footer(&mut self) -> Result<(), Error> { 314 | self.write_row(Row::Eof) 315 | } 316 | 317 | fn write_key_values(&mut self, progress: &mut dyn FnMut(u64)) -> Result<(), Error> { 318 | let database = self.database.take().unwrap(); 319 | let mut cursor = database.cursor()?; 320 | 321 | loop { 322 | let mut row = KeyValueRow::default(); 323 | let has_item = cursor.next_buf(&mut row.key, &mut row.value)?; 324 | 325 | if !has_item { 326 | break; 327 | } 328 | 329 | row.index = self.counter; 330 | row.key_crc32c = crc32c::crc32c(&row.key); 331 | row.value_crc32c = crc32c::crc32c(&row.value); 332 | self.counter += 1; 333 | 334 | self.write_row(Row::KeyValue(row))?; 335 | 336 | progress(self.counter); 337 | } 338 | 339 | self.database = Some(database); 340 | 341 | Ok(()) 342 | } 343 | } 344 | 345 | /// Import key-value pairs from the given source file into the database. 346 | /// 347 | /// The provided progress callback will be called with the number of pairs 348 | /// processed. 349 | /// 350 | /// It is the caller's responsibility to call [`Database::flush()`] after 351 | /// the function completes. 352 | pub fn import(database: &mut Database, input_file: &mut R, progress: C) -> Result<(), Error> 353 | where 354 | C: FnMut(u64), 355 | R: BufRead, 356 | { 357 | let mut reader = ImportReader::new(input_file, database); 358 | reader.import(progress)?; 359 | 360 | Ok(()) 361 | } 362 | 363 | /// Export key-value pairs from the database to the destination file. 364 | /// 365 | /// The provided progress callback will be called with the number of pairs 366 | /// processed. 367 | /// 368 | /// It is the caller's responsibility to ensure data has been persisted using 369 | /// functions such as `flush()` or `sync_data()`. 370 | pub fn export(database: &mut Database, output_file: &mut W, progress: C) -> Result<(), Error> 371 | where 372 | W: Write, 373 | C: FnMut(u64), 374 | { 375 | let mut writer = ExportWriter::new(output_file, database); 376 | writer.export(progress)?; 377 | 378 | Ok(()) 379 | } 380 | -------------------------------------------------------------------------------- /src/library/src/format.rs: -------------------------------------------------------------------------------- 1 | use std::io::{Cursor, Read, Write}; 2 | 3 | use relative_path::RelativePath; 4 | use rmp_serde::{Deserializer, Serializer}; 5 | use serde::{Deserialize, Serialize}; 6 | 7 | use crate::{ 8 | error::Error, 9 | lru::LruVec, 10 | vfs::{Vfs, VfsSyncOption}, 11 | }; 12 | 13 | const MAGIC_BYTES: [u8; 8] = [0xFE, b'G', b'r', b'e', b'b', b'e', 0x00, 0x00]; 14 | 15 | pub struct Format { 16 | file_buffer: Vec, 17 | page_buffer: Vec, 18 | payload_buffer: Vec, 19 | compression_level: Option, 20 | dir_create_cache: LruVec, 21 | } 22 | 23 | impl Default for Format { 24 | fn default() -> Self { 25 | Self { 26 | file_buffer: Vec::new(), 27 | page_buffer: Vec::new(), 28 | payload_buffer: Vec::new(), 29 | compression_level: if cfg!(feature = "zstd") { 30 | Some(0) 31 | } else { 32 | None 33 | }, 34 | dir_create_cache: LruVec::new(8), 35 | } 36 | } 37 | } 38 | 39 | impl Format { 40 | pub fn set_compression_level(&mut self, value: Option) { 41 | self.compression_level = value; 42 | } 43 | 44 | pub fn read_file<'de, T>(&mut self, vfs: &mut dyn Vfs, path: &str) -> Result 45 | where 46 | T: Deserialize<'de>, 47 | { 48 | let mut file = Cursor::new(vfs.read(path)?); 49 | 50 | let mut magic_bytes: [u8; 8] = [0u8; 8]; 51 | file.read_exact(&mut magic_bytes)?; 52 | 53 | if MAGIC_BYTES != magic_bytes { 54 | return Err(Error::InvalidFileFormat { 55 | path: path.to_string(), 56 | message: "not a database", 57 | }); 58 | } 59 | 60 | let mut compression_flag: [u8; 1] = [0u8; 1]; 61 | file.read_exact(&mut compression_flag)?; 62 | 63 | if compression_flag[0] == 0x01 { 64 | self.decompress_to_page_buffer(&mut file)?; 65 | } else { 66 | self.page_buffer.clear(); 67 | file.read_to_end(&mut self.page_buffer)?; 68 | } 69 | 70 | self.deserialize_page(path) 71 | } 72 | 73 | pub fn write_file( 74 | &mut self, 75 | vfs: &mut dyn Vfs, 76 | path: &str, 77 | payload: T, 78 | sync_option: VfsSyncOption, 79 | ) -> Result<(), Error> 80 | where 81 | T: Serialize, 82 | { 83 | self.file_buffer.clear(); 84 | self.page_buffer.clear(); 85 | self.payload_buffer.clear(); 86 | 87 | self.file_buffer.write_all(&MAGIC_BYTES)?; 88 | 89 | if self.compression_level.is_some() { 90 | self.file_buffer.write_all(&[0x01])?; 91 | self.serialize_page(payload)?; 92 | self.write_compressed_page_to_file_buffer()?; 93 | } else { 94 | self.file_buffer.write_all(&[0x00])?; 95 | self.serialize_page(payload)?; 96 | self.file_buffer.write_all(&self.page_buffer)?; 97 | } 98 | 99 | let rel_path = RelativePath::new(path); 100 | let dir_path = rel_path.parent().unwrap(); 101 | 102 | if !self.is_in_dir_cache(dir_path) { 103 | vfs.create_dir_all(dir_path.as_str())?; 104 | } 105 | 106 | vfs.write(path, &self.file_buffer, sync_option)?; 107 | 108 | Ok(()) 109 | } 110 | 111 | fn serialize_page(&mut self, object: T) -> Result<(), Error> 112 | where 113 | T: Serialize, 114 | { 115 | serialize_payload(object, &mut self.payload_buffer)?; 116 | 117 | let size_bytes = self.payload_buffer.len().to_be_bytes(); 118 | 119 | self.page_buffer.write_all(&size_bytes)?; 120 | self.page_buffer.write_all(&self.payload_buffer)?; 121 | 122 | let crc = crc32c::crc32c(&self.payload_buffer); 123 | let crc_bytes = crc.to_be_bytes(); 124 | 125 | self.page_buffer.write_all(&crc_bytes)?; 126 | 127 | Ok(()) 128 | } 129 | 130 | fn write_compressed_page_to_file_buffer(&mut self) -> Result<(), Error> { 131 | #[cfg(feature = "zstd")] 132 | { 133 | let mut temp_buffer = Vec::with_capacity(0); 134 | std::mem::swap(&mut self.file_buffer, &mut temp_buffer); 135 | 136 | let compression_level = self.compression_level.unwrap(); 137 | let mut compressor = zstd::Encoder::new(temp_buffer, compression_level)?; 138 | compressor.write_all(&self.page_buffer)?; 139 | let mut old_writer = compressor.finish()?; 140 | 141 | std::mem::swap(&mut self.file_buffer, &mut old_writer); 142 | 143 | Ok(()) 144 | } 145 | #[cfg(not(feature = "zstd"))] 146 | { 147 | Err(Error::CompressionUnavailable) 148 | } 149 | } 150 | 151 | fn is_in_dir_cache(&mut self, dir_path: &RelativePath) -> bool { 152 | let dir_path = dir_path.to_string(); 153 | 154 | if !self.dir_create_cache.touch(&dir_path) { 155 | self.dir_create_cache.insert(dir_path); 156 | false 157 | } else { 158 | true 159 | } 160 | } 161 | 162 | fn decompress_to_page_buffer(&mut self, source: &mut dyn Read) -> Result<(), Error> { 163 | self.page_buffer.clear(); 164 | 165 | #[cfg(feature = "zstd")] 166 | { 167 | let mut decompressor = zstd::Decoder::new(source)?; 168 | decompressor.read_to_end(&mut self.page_buffer)?; 169 | Ok(()) 170 | } 171 | #[cfg(not(feature = "zstd"))] 172 | { 173 | let _ = source; 174 | Err(Error::CompressionUnavailable) 175 | } 176 | } 177 | 178 | fn deserialize_page<'de, T>(&mut self, path: &str) -> Result 179 | where 180 | T: Deserialize<'de>, 181 | { 182 | let mut size_bytes: [u8; 8] = [0u8; 8]; 183 | let mut data = Cursor::new(&mut self.page_buffer); 184 | 185 | data.read_exact(&mut size_bytes)?; 186 | let size = u64::from_be_bytes(size_bytes) as usize; 187 | 188 | let payload = deserialize_payload(&mut data)?; 189 | 190 | let mut crc_bytes: [u8; 4] = [0; 4]; 191 | data.read_exact(&mut crc_bytes)?; 192 | let crc = u32::from_be_bytes(crc_bytes); 193 | 194 | let test_crc = crc32c::crc32c(&self.page_buffer[8..8 + size]); 195 | 196 | if crc != test_crc { 197 | Err(Error::BadChecksum { 198 | path: path.to_string(), 199 | }) 200 | } else { 201 | Ok(payload) 202 | } 203 | } 204 | } 205 | 206 | fn serialize_payload(object: T, destination: W) -> Result<(), Error> 207 | where 208 | T: Serialize, 209 | W: Write, 210 | { 211 | let mut serializer = Serializer::new(destination) 212 | .with_binary() 213 | .with_string_variants() 214 | .with_struct_map(); 215 | 216 | match object.serialize(&mut serializer) { 217 | Ok(_) => Ok(()), 218 | Err(error) => Err(Error::Other(Box::new(error))), 219 | } 220 | } 221 | 222 | fn deserialize_payload<'de, T, R>(source: R) -> Result 223 | where 224 | T: Deserialize<'de>, 225 | R: Read, 226 | { 227 | let mut deserializer = Deserializer::new(source).with_binary(); 228 | 229 | match Deserialize::deserialize(&mut deserializer) { 230 | Ok(value) => Ok(value), 231 | Err(error) => Err(Error::Other(Box::new(error))), 232 | } 233 | } 234 | 235 | #[cfg(test)] 236 | mod tests { 237 | use super::*; 238 | use crate::vfs::MemoryVfs; 239 | 240 | #[test] 241 | fn test_format() -> Result<(), Error> { 242 | let mut format = Format::default(); 243 | let mut vfs = MemoryVfs::new(); 244 | 245 | format.write_file(&mut vfs, "my_file", "hello world", VfsSyncOption::None)?; 246 | 247 | let payload: String = format.read_file(&mut vfs, "my_file")?; 248 | 249 | assert_eq!(&payload, "hello world"); 250 | 251 | Ok(()) 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /src/library/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Lightweight embedded key-value store/database backed by files 2 | //! in a virtual file system interface. 3 | //! 4 | //! To open a database, use [`Database`]: 5 | //! 6 | //! ``` 7 | //! use grebedb::{Database, Options}; 8 | //! 9 | //! # fn main() -> Result<(), grebedb::Error> { 10 | //! let options = Options::default(); 11 | //! // let mut db = Database::open_memory("path/to/empty/directory/", options)?; 12 | //! let mut db = Database::open_memory(options)?; 13 | //! 14 | //! db.put("my_key", "hello world!")?; 15 | //! db.flush()?; 16 | //! 17 | //! # Ok(()) 18 | //! # } 19 | //! ``` 20 | //! 21 | //! For important details, such as limitations and guarantees, see the 22 | //! README.md file in the project's source code repository. 23 | 24 | #![forbid(unsafe_code)] 25 | #![warn(missing_docs)] 26 | 27 | pub mod error; 28 | pub mod export; 29 | mod format; 30 | mod lru; 31 | mod page; 32 | mod system; 33 | mod tree; 34 | pub mod vfs; 35 | 36 | use std::{ 37 | fmt::Debug, 38 | ops::{Bound, RangeBounds}, 39 | path::{Path, PathBuf}, 40 | time::{Duration, Instant}, 41 | }; 42 | 43 | pub use crate::error::Error; 44 | use crate::format::Format; 45 | use crate::page::{Metadata as PageMetadata, Page, PageOpenMode, PageTableOptions}; 46 | use crate::tree::{Node, Tree, TreeCursor, TreeMetadata}; 47 | use crate::vfs::{MemoryVfs, OsVfs, ReadOnlyVfs, Vfs, VfsSyncOption}; 48 | 49 | /// Type alias for an owned key-value pair. 50 | pub type KeyValuePair = (Vec, Vec); 51 | 52 | /// Database configuration options. 53 | #[derive(Debug, Clone)] 54 | pub struct Options { 55 | /// Option when opening a database. Default: LoadOrCreate. 56 | pub open_mode: OpenMode, 57 | 58 | /// Maximum number of keys-value pairs per node. Default: 1024. 59 | /// 60 | /// This value specifies the threshold when a tree node is considered full. 61 | /// When a node is full, it is split into two and the tree is rebalanced. 62 | /// 63 | /// A page contains a single node and a page is stored on disk as one file. 64 | /// 65 | /// This option shouldn't be changed without making performance and resource usage 66 | /// benchmarks. 67 | pub keys_per_node: usize, 68 | 69 | /// Maximum number of pages held in memory cache. Default: 64. 70 | /// 71 | /// The cache is used to store frequently accessed pages for reducing disk operations. 72 | /// 73 | /// If memory usage is too high, consider decreasing this value first. 74 | pub page_cache_size: usize, 75 | 76 | /// Whether to use file locking to prevent corruption by multiple processes. 77 | /// Default: true. 78 | pub file_locking: bool, 79 | 80 | /// Level of file synchronization to increase durability on disk file systems. 81 | /// Default: Data 82 | pub file_sync: SyncOption, 83 | 84 | /// Whether to flush the data to the file system periodically when a 85 | /// database operation is performed. 86 | /// Default: true. 87 | /// 88 | /// When true, data is flushed when the database is dropped or when enough 89 | /// modifications accumulate. Setting this option to false allows you to 90 | /// manually persist changes at more optimal points. 91 | /// 92 | /// There is no background maintenance thread that does automatic flushing; 93 | /// automatic flushing occurs when a database modifying function, 94 | /// such as put() or remove(), is called. 95 | pub automatic_flush: bool, 96 | 97 | /// Number of modifications required for automatic flush to be considered. 98 | /// Default: 2048 99 | /// 100 | /// When the threshold is reached after 300 seconds, 101 | /// or the threshold × 2 is reached after 60 seconds, 102 | /// a flush is scheduled to be performed on the next modification. 103 | pub automatic_flush_threshold: usize, 104 | 105 | /// Compression level for each page. Default: Low. 106 | pub compression_level: CompressionLevel, 107 | } 108 | 109 | impl Default for Options { 110 | fn default() -> Self { 111 | Self { 112 | open_mode: OpenMode::default(), 113 | keys_per_node: 1024, 114 | page_cache_size: 64, 115 | file_locking: true, 116 | file_sync: SyncOption::default(), 117 | automatic_flush: true, 118 | automatic_flush_threshold: 2048, 119 | compression_level: CompressionLevel::default(), 120 | } 121 | } 122 | } 123 | 124 | impl Options { 125 | fn validate(&self) -> Result<(), Error> { 126 | if self.keys_per_node < 2 { 127 | return Err(Error::InvalidConfig { 128 | message: "required keys_per_node >= 2", 129 | }); 130 | } 131 | if self.page_cache_size < 1 { 132 | return Err(Error::InvalidConfig { 133 | message: "required page_cache_size >= 1", 134 | }); 135 | } 136 | 137 | Ok(()) 138 | } 139 | } 140 | 141 | impl From for PageTableOptions { 142 | fn from(options: Options) -> Self { 143 | Self { 144 | open_mode: options.open_mode.into(), 145 | page_cache_size: options.page_cache_size, 146 | file_locking: options.file_locking, 147 | file_sync: options.file_sync.into(), 148 | keys_per_node: options.keys_per_node, 149 | compression_level: options.compression_level.to_zstd(), 150 | } 151 | } 152 | } 153 | 154 | /// Database open modes. 155 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 156 | pub enum OpenMode { 157 | /// Open an existing database only if it exists. 158 | LoadOnly, 159 | /// Create a database only if it does not already exist. 160 | CreateOnly, 161 | /// Open a database, creating it if it does not exist. 162 | LoadOrCreate, 163 | /// Open an existing database and avoid modifying it. 164 | ReadOnly, 165 | } 166 | 167 | impl Default for OpenMode { 168 | fn default() -> Self { 169 | Self::LoadOrCreate 170 | } 171 | } 172 | 173 | impl From for PageOpenMode { 174 | fn from(option: OpenMode) -> Self { 175 | match option { 176 | OpenMode::LoadOnly => PageOpenMode::LoadOnly, 177 | OpenMode::CreateOnly => PageOpenMode::CreateOnly, 178 | OpenMode::LoadOrCreate => PageOpenMode::LoadOrCreate, 179 | OpenMode::ReadOnly => PageOpenMode::ReadOnly, 180 | } 181 | } 182 | } 183 | 184 | /// Database data compression level. 185 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 186 | pub enum CompressionLevel { 187 | /// Disable compression. 188 | None, 189 | 190 | /// Very fast compression speeds at the expense of low compression ratios. 191 | /// 192 | /// Currently, this corresponds to Zstandard level 1. 193 | VeryLow, 194 | 195 | /// Fast compression speeds at the expense of somewhat low compression ratios. 196 | /// 197 | /// Currently, this corresponds to Zstandard level 3, the default value. 198 | Low, 199 | 200 | /// Higher compression ratios at the expense of slower compression speeds. 201 | /// 202 | /// Currently, this corresponds to Zstandard level 9. 203 | Medium, 204 | 205 | /// Best compression ratios at the expense of very slow compression speeds. 206 | /// 207 | /// Currently, this corresponds to Zstandard level 19. 208 | High, 209 | } 210 | 211 | impl Default for CompressionLevel { 212 | fn default() -> Self { 213 | Self::Low 214 | } 215 | } 216 | 217 | impl CompressionLevel { 218 | fn to_zstd(self) -> Option { 219 | match self { 220 | Self::None => None, 221 | Self::VeryLow => Some(1), 222 | Self::Low => Some(3), 223 | Self::Medium => Some(9), 224 | Self::High => Some(19), 225 | } 226 | } 227 | } 228 | 229 | /// Level of file synchronization for files created by the database. 230 | /// 231 | /// These options are equivalent to [`vfs::VfsSyncOption`]. 232 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 233 | pub enum SyncOption { 234 | /// Don't require any flushing and simply overwrite files. 235 | None, 236 | 237 | /// Flush file content only and use file rename technique. 238 | /// 239 | /// Flush command is equivalent to `File::sync_data()` or Unix `fdatasync()`. 240 | Data, 241 | 242 | /// Flush file content including metadata and use file rename technique. 243 | /// 244 | /// Flush command is equivalent to `File::sync_all()` or Unix `fsync()`. 245 | All, 246 | } 247 | 248 | impl Default for SyncOption { 249 | fn default() -> Self { 250 | Self::Data 251 | } 252 | } 253 | 254 | impl From for VfsSyncOption { 255 | fn from(option: SyncOption) -> Self { 256 | match option { 257 | SyncOption::None => Self::None, 258 | SyncOption::Data => Self::Data, 259 | SyncOption::All => Self::All, 260 | } 261 | } 262 | } 263 | 264 | /// GrebeDB database interface. 265 | pub struct Database { 266 | options: Options, 267 | tree: Tree, 268 | flush_tracker: Option, 269 | } 270 | 271 | impl Database { 272 | /// Open a database using the given virtual file system and options. 273 | pub fn open(vfs: Box, options: Options) -> Result { 274 | options.validate()?; 275 | 276 | let vfs: Box = if options.open_mode == OpenMode::ReadOnly { 277 | Box::new(ReadOnlyVfs::new(vfs)) 278 | } else { 279 | vfs 280 | }; 281 | 282 | let mut tree = Tree::open(vfs, options.clone().into())?; 283 | 284 | match options.open_mode { 285 | OpenMode::CreateOnly | OpenMode::LoadOrCreate => { 286 | tree.init_if_empty()?; 287 | tree.upgrade()?; 288 | } 289 | OpenMode::LoadOnly => { 290 | tree.upgrade()?; 291 | } 292 | _ => {} 293 | } 294 | 295 | let flush_tracker = if options.automatic_flush && options.open_mode != OpenMode::ReadOnly { 296 | Some(FlushTracker::new(options.automatic_flush_threshold)) 297 | } else { 298 | None 299 | }; 300 | 301 | Ok(Self { 302 | options, 303 | tree, 304 | flush_tracker, 305 | }) 306 | } 307 | 308 | /// Open a database in temporary memory. 309 | pub fn open_memory(options: Options) -> Result { 310 | Self::open(Box::new(MemoryVfs::default()), options) 311 | } 312 | 313 | /// Open a database to a path on the disk. 314 | /// 315 | /// The path must be a directory. 316 | pub fn open_path

(root_path: P, options: Options) -> Result 317 | where 318 | P: Into, 319 | { 320 | Self::open(Box::new(OsVfs::new(root_path)), options) 321 | } 322 | 323 | /// Return database metadata information. 324 | pub fn metadata(&self) -> Metadata { 325 | Metadata { 326 | tree_metadata: self.tree.metadata(), 327 | } 328 | } 329 | 330 | /// Return whether the key exists. 331 | pub fn contains_key(&mut self, key: K) -> Result 332 | where 333 | K: AsRef<[u8]>, 334 | { 335 | self.tree.contains_key(key.as_ref()) 336 | } 337 | 338 | /// Retrieve a stored value, by its key, as a vector. 339 | pub fn get(&mut self, key: K) -> Result>, Error> 340 | where 341 | K: AsRef<[u8]>, 342 | { 343 | let mut value = Vec::new(); 344 | if self.tree.get(key.as_ref(), &mut value)? { 345 | Ok(Some(value)) 346 | } else { 347 | Ok(None) 348 | } 349 | } 350 | 351 | /// Retrieve a stored value, by its key, into the given buffer. 352 | /// 353 | /// Returns true if the key-value pair was found. The vector will be 354 | /// cleared and resized. 355 | pub fn get_buf(&mut self, key: K, value_destination: &mut Vec) -> Result 356 | where 357 | K: AsRef<[u8]>, 358 | { 359 | self.tree.get(key.as_ref(), value_destination) 360 | } 361 | 362 | /// Store a key-value pair. 363 | pub fn put(&mut self, key: K, value: V) -> Result<(), Error> 364 | where 365 | K: Into>, 366 | V: Into>, 367 | { 368 | self.maybe_flush(true)?; 369 | self.tree.put(key.into(), value.into()) 370 | } 371 | 372 | /// Remove a key-value pair by its key. 373 | /// 374 | /// No error occurs if the key does not exist. 375 | pub fn remove(&mut self, key: K) -> Result<(), Error> 376 | where 377 | K: AsRef<[u8]>, 378 | { 379 | self.maybe_flush(true)?; 380 | self.tree.remove(key.as_ref()) 381 | } 382 | 383 | /// Return a cursor for iterating all the key-value pairs. 384 | pub fn cursor(&mut self) -> Result, Error> { 385 | Ok(Cursor::new(&mut self.tree)) 386 | } 387 | 388 | /// Return a cursor for iterating all the key-value pairs within the given 389 | /// range. 390 | /// 391 | /// This method is equivalent of obtaining a cursor and calling 392 | /// [`Cursor::seek()`] and [`Cursor::set_range()`] 393 | pub fn cursor_range(&mut self, range: R) -> Result, Error> 394 | where 395 | K: AsRef<[u8]>, 396 | R: RangeBounds, 397 | { 398 | let mut cursor = Cursor::new(&mut self.tree); 399 | 400 | match range.start_bound() { 401 | Bound::Included(key) => { 402 | cursor.seek(key)?; 403 | } 404 | Bound::Excluded(key) => { 405 | let mut key = key.as_ref().to_vec(); 406 | key.push(0); 407 | cursor.seek(key)?; 408 | } 409 | Bound::Unbounded => {} 410 | } 411 | 412 | cursor.set_range(range); 413 | 414 | Ok(cursor) 415 | } 416 | 417 | /// Persist all modifications to the file system. 418 | /// 419 | /// Calling this function ensures that all changes pending, whether cached 420 | /// in memory or in files, are atomically saved on the file system 421 | /// before this function returns. If the database is not flushed when 422 | /// dropped or the program exits, changes since the last successful flush 423 | /// will be discarded. This function effectively emulates a transaction. 424 | /// 425 | /// For details about automatic flushing, see [`Options`]. 426 | pub fn flush(&mut self) -> Result<(), Error> { 427 | self.tree.flush() 428 | } 429 | 430 | /// Check the database for internal consistency and data integrity. 431 | /// 432 | /// The provided callback function is called with the number of items 433 | /// processed and the estimated number of items. 434 | /// 435 | /// The function returns an error on the first verification failure or 436 | /// other error. 437 | pub fn verify

(&mut self, progress_callback: P) -> Result<(), Error> 438 | where 439 | P: FnMut(usize, usize), 440 | { 441 | self.tree.verify_tree(progress_callback) 442 | } 443 | 444 | /// Print the tree for debugging purposes. 445 | pub fn debug_print_tree(&mut self) -> Result<(), Error> { 446 | self.tree.dump_tree() 447 | } 448 | 449 | fn maybe_flush(&mut self, increment: bool) -> Result<(), Error> { 450 | if let Some(flush_tracker) = &mut self.flush_tracker { 451 | if increment { 452 | flush_tracker.increment_modification(); 453 | } 454 | 455 | if flush_tracker.check_should_flush() { 456 | self.flush()?; 457 | } 458 | } 459 | 460 | Ok(()) 461 | } 462 | } 463 | 464 | impl Drop for Database { 465 | fn drop(&mut self) { 466 | if self.options.automatic_flush && self.options.open_mode != OpenMode::ReadOnly { 467 | let _ = self.flush(); 468 | } 469 | } 470 | } 471 | 472 | impl Debug for Database { 473 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 474 | write!(f, "Database {{ open_mode: {:?} }}", self.options.open_mode) 475 | } 476 | } 477 | 478 | /// Cursor for navigating key-value pairs in sorted order. 479 | pub struct Cursor<'a> { 480 | tree: &'a mut Tree, 481 | tree_cursor: TreeCursor, 482 | error: Option, 483 | has_seeked: bool, 484 | range: (Bound>, Bound>), 485 | } 486 | 487 | impl<'a> Cursor<'a> { 488 | fn new(tree: &'a mut Tree) -> Self { 489 | Self { 490 | tree, 491 | tree_cursor: TreeCursor::default(), 492 | error: None, 493 | has_seeked: false, 494 | range: (Bound::Unbounded, Bound::Unbounded), 495 | } 496 | } 497 | 498 | /// Return the most recent error. 499 | pub fn error(&self) -> Option<&Error> { 500 | self.error.as_ref() 501 | } 502 | 503 | /// Reposition the cursor at or after the given key. 504 | /// 505 | /// In other words, the cursor will be positioned to return key-value pairs 506 | /// that are equal or greater than the given key. 507 | /// 508 | /// If a range has been set and the cursor is positioned outside the range, 509 | /// the iteration is considered terminated and no key-value pairs will returned. 510 | pub fn seek(&mut self, key: K) -> Result<(), Error> 511 | where 512 | K: AsRef<[u8]>, 513 | { 514 | self.has_seeked = true; 515 | self.tree.cursor_start(&mut self.tree_cursor, key.as_ref()) 516 | } 517 | 518 | /// Limit the key-value pairs within a range of keys. 519 | /// 520 | /// The cursor will return key-value pairs where the keys are contained 521 | /// within the given range. 522 | /// 523 | /// This function will not reposition the cursor to a position within the 524 | /// range. You must call [`Self::seek()`] manually since the cursor will not 525 | /// automatically seek forward to a range's starting bound. 526 | pub fn set_range(&mut self, range: R) 527 | where 528 | K: AsRef<[u8]>, 529 | R: RangeBounds, 530 | { 531 | self.range = concrete_range(range); 532 | } 533 | 534 | /// Advance the cursor forward and write the key-value pair to the given buffers. 535 | /// 536 | /// Returns true if the key-value pair was written. 537 | /// Returns false if there are no more key-value pairs 538 | /// or the cursor is positioned outside the range if set. 539 | /// 540 | /// The vectors will be cleared and resized. 541 | pub fn next_buf(&mut self, key: &mut Vec, value: &mut Vec) -> Result { 542 | if !self.has_seeked { 543 | self.has_seeked = true; 544 | self.tree.cursor_start(&mut self.tree_cursor, b"")?; 545 | } 546 | 547 | if self 548 | .tree 549 | .cursor_next(&mut self.tree_cursor, key, value, &slice_range(&self.range))? 550 | { 551 | Ok(true) 552 | } else { 553 | Ok(false) 554 | } 555 | } 556 | } 557 | 558 | impl<'a> Iterator for Cursor<'a> { 559 | type Item = KeyValuePair; 560 | 561 | fn next(&mut self) -> Option { 562 | let mut key_buffer = Vec::new(); 563 | let mut value_buffer = Vec::new(); 564 | 565 | match self.next_buf(&mut key_buffer, &mut value_buffer) { 566 | Ok(success) => { 567 | if success { 568 | Some((key_buffer, value_buffer)) 569 | } else { 570 | None 571 | } 572 | } 573 | Err(error) => { 574 | self.error = Some(error); 575 | None 576 | } 577 | } 578 | } 579 | } 580 | 581 | impl<'a> Debug for Cursor<'a> { 582 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 583 | write!(f, "DatabaseCursor") 584 | } 585 | } 586 | 587 | #[derive(Debug)] 588 | /// Additional non-critical information associated with the database. 589 | pub struct Metadata<'a> { 590 | tree_metadata: Option<&'a TreeMetadata>, 591 | } 592 | 593 | impl<'a> Metadata<'a> { 594 | /// Return the approximate number of key-value pairs in the database. 595 | pub fn key_value_count(&self) -> u64 { 596 | if let Some(meta) = self.tree_metadata { 597 | meta.key_value_count 598 | } else { 599 | 0 600 | } 601 | } 602 | } 603 | 604 | struct FlushTracker { 605 | base_threshold: usize, 606 | modification_count: usize, 607 | last_flush_time: Instant, 608 | } 609 | 610 | impl FlushTracker { 611 | pub fn new(base_threshold: usize) -> Self { 612 | Self { 613 | base_threshold, 614 | modification_count: 0, 615 | last_flush_time: Instant::now(), 616 | } 617 | } 618 | 619 | pub fn increment_modification(&mut self) { 620 | self.modification_count += 1; 621 | } 622 | 623 | pub fn check_should_flush(&mut self) -> bool { 624 | let level_long = self.modification_count >= self.base_threshold 625 | && self.last_flush_time.elapsed() >= Duration::from_secs(300); 626 | let level_short = self.modification_count >= self.base_threshold * 2 627 | && self.last_flush_time.elapsed() >= Duration::from_secs(60); 628 | 629 | if level_long || level_short { 630 | self.modification_count = 0; 631 | self.last_flush_time = Instant::now(); 632 | true 633 | } else { 634 | false 635 | } 636 | } 637 | } 638 | 639 | /// Print the page contents for debugging purposes. 640 | pub fn debug_print_page(path: &Path) -> Result<(), Error> { 641 | let mut format = Format::default(); 642 | let mut vfs = ReadOnlyVfs::new(Box::new(OsVfs::new(path.parent().unwrap()))); 643 | 644 | let filename = path.file_name().unwrap().to_str().unwrap(); 645 | 646 | if filename.contains("meta") { 647 | let payload: PageMetadata = format.read_file(&mut vfs, filename)?; 648 | 649 | eprintln!("{:?}", payload); 650 | } else { 651 | let payload: Page = format.read_file(&mut vfs, filename)?; 652 | 653 | eprintln!("{:?}", payload); 654 | } 655 | 656 | Ok(()) 657 | } 658 | 659 | fn concrete_range(range: R) -> (Bound>, Bound>) 660 | where 661 | K: AsRef<[u8]>, 662 | R: RangeBounds, 663 | { 664 | let start_bound: Bound> = match range.start_bound() { 665 | Bound::Included(bound) => Bound::Included(bound.as_ref().to_vec()), 666 | Bound::Excluded(bound) => Bound::Excluded(bound.as_ref().to_vec()), 667 | Bound::Unbounded => Bound::Unbounded, 668 | }; 669 | let end_bound: Bound> = match range.end_bound() { 670 | Bound::Included(bound) => Bound::Included(bound.as_ref().to_vec()), 671 | Bound::Excluded(bound) => Bound::Excluded(bound.as_ref().to_vec()), 672 | Bound::Unbounded => Bound::Unbounded, 673 | }; 674 | (start_bound, end_bound) 675 | } 676 | 677 | fn slice_range<'a>( 678 | range: &'a (Bound>, Bound>), 679 | ) -> (Bound<&'a [u8]>, Bound<&'a [u8]>) { 680 | let start_bound: Bound<&'a [u8]> = match range.start_bound() { 681 | Bound::Included(bound) => Bound::Included(bound), 682 | Bound::Excluded(bound) => Bound::Excluded(bound), 683 | Bound::Unbounded => Bound::Unbounded, 684 | }; 685 | let end_bound: Bound<&'a [u8]> = match range.end_bound() { 686 | Bound::Included(bound) => Bound::Included(bound), 687 | Bound::Excluded(bound) => Bound::Excluded(bound), 688 | Bound::Unbounded => Bound::Unbounded, 689 | }; 690 | (start_bound, end_bound) 691 | } 692 | -------------------------------------------------------------------------------- /src/library/src/lru.rs: -------------------------------------------------------------------------------- 1 | /// Tracks least recently used items. 2 | pub struct LruVec { 3 | capacity: usize, 4 | entries: Vec<(u64, T)>, 5 | counter: u64, 6 | } 7 | 8 | impl LruVec 9 | where 10 | T: PartialEq, 11 | { 12 | pub fn new(capacity: usize) -> Self { 13 | Self { 14 | capacity, 15 | entries: Vec::with_capacity(capacity), 16 | counter: u64::MAX, 17 | } 18 | } 19 | 20 | #[cfg(test)] 21 | pub fn len(&self) -> usize { 22 | self.entries.len() 23 | } 24 | 25 | #[cfg(test)] 26 | pub fn is_empty(&self) -> bool { 27 | self.entries.is_empty() 28 | } 29 | 30 | /// Add an item or update an existing item to the front. 31 | /// 32 | /// Returns an evicted item if any. 33 | pub fn insert(&mut self, item: T) -> Option { 34 | if self.find_and_update(&item) { 35 | self.sort_items(); 36 | None 37 | } else if self.entries.len() == self.capacity { 38 | let old_entry = self.entries.pop(); 39 | self.counter -= 1; 40 | self.entries.insert(0, (self.counter, item)); 41 | debug_assert!(self.entries.len() <= self.capacity); 42 | 43 | match old_entry { 44 | Some(item) => Some(item.1), 45 | None => None, 46 | } 47 | } else { 48 | self.counter -= 1; 49 | self.entries.insert(0, (self.counter, item)); 50 | debug_assert!(self.entries.len() <= self.capacity); 51 | None 52 | } 53 | } 54 | 55 | /// Move an item to the front. 56 | /// 57 | /// Returns whether the item exists. 58 | pub fn touch(&mut self, item: &T) -> bool { 59 | if self.find_and_update(item) { 60 | self.sort_items(); 61 | true 62 | } else { 63 | false 64 | } 65 | } 66 | 67 | /// Remove all items and returns them. 68 | #[allow(dead_code)] 69 | pub fn clear(&mut self) -> Vec { 70 | let mut new_vec = Vec::with_capacity(self.entries.len()); 71 | 72 | while let Some(entry) = self.entries.pop() { 73 | new_vec.push(entry.1); 74 | } 75 | 76 | new_vec.reverse(); 77 | 78 | new_vec 79 | } 80 | 81 | fn find_and_update(&mut self, item: &T) -> bool { 82 | for current_item in self.entries.iter_mut() { 83 | if ¤t_item.1 == item { 84 | self.counter -= 1; 85 | current_item.0 = self.counter; 86 | return true; 87 | } 88 | } 89 | 90 | false 91 | } 92 | 93 | fn sort_items(&mut self) { 94 | self.entries.sort_unstable_by_key(|item| item.0); 95 | } 96 | 97 | #[cfg(test)] 98 | pub(in crate::lru) fn item_at(&self, index: usize) -> Option<&T> { 99 | let entry = self.entries.get(index)?; 100 | Some(&entry.1) 101 | } 102 | } 103 | 104 | #[cfg(test)] 105 | mod tests { 106 | use super::*; 107 | 108 | #[test] 109 | fn test_lru_vec() { 110 | let mut lru = LruVec::::new(3); 111 | 112 | assert!(!lru.touch(&1)); 113 | 114 | assert!(lru.insert(1).is_none()); // [1] 115 | assert!(lru.insert(2).is_none()); // [2, 1] 116 | 117 | assert_eq!(lru.len(), 2); 118 | assert!(!lru.is_empty()); 119 | 120 | assert!(lru.insert(3).is_none()); // [3, 2, 1] 121 | assert_eq!(lru.insert(4), Some(1)); // [4, 3, 2] 122 | 123 | assert_eq!(lru.item_at(0), Some(&4)); 124 | assert_eq!(lru.item_at(1), Some(&3)); 125 | assert_eq!(lru.item_at(2), Some(&2)); 126 | 127 | assert!(lru.touch(&3)); // [3, 4, 2] 128 | 129 | assert_eq!(lru.item_at(0), Some(&3)); 130 | assert_eq!(lru.item_at(1), Some(&4)); 131 | assert_eq!(lru.item_at(2), Some(&2)); 132 | 133 | let items = lru.clear(); 134 | assert_eq!(&items, &[3, 4, 2]); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/library/src/page.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::{HashMap, HashSet, VecDeque}, 3 | ops::{Deref, DerefMut}, 4 | }; 5 | 6 | use serde::{de::DeserializeOwned, Deserialize, Serialize}; 7 | use uuid::Uuid; 8 | 9 | use crate::{ 10 | error::Error, 11 | format::Format, 12 | lru::LruVec, 13 | system::UuidGenerator, 14 | vfs::{Vfs, VfsSyncOption}, 15 | }; 16 | 17 | const LOCK_FILENAME: &str = "grebedb_lock.lock"; 18 | const METADATA_FILENAME: &str = "grebedb_meta.grebedb"; 19 | const METADATA_NEW_FILENAME: &str = "grebedb_meta.grebedb.tmp"; 20 | const METADATA_OLD_FILENAME: &str = "grebedb_meta_prev.grebedb"; 21 | const METADATA_COPY_FILENAME: &str = "grebedb_meta_copy.grebedb"; 22 | 23 | pub type PageId = u64; 24 | pub type RevisionId = u64; 25 | 26 | #[derive(Debug, Serialize, Deserialize)] 27 | pub struct Page { 28 | pub uuid: Uuid, // should match metadata 29 | pub id: PageId, 30 | pub revision: RevisionId, 31 | pub deleted: bool, 32 | pub content: Option, 33 | } 34 | 35 | #[derive(Debug, Clone, Serialize, Deserialize)] 36 | pub struct Metadata { 37 | pub uuid: Uuid, // uuid for the entire database 38 | pub revision: RevisionId, 39 | pub id_counter: PageId, // current allocated ID 40 | pub free_id_list: Vec, 41 | pub root_id: Option, 42 | pub auxiliary: Option, 43 | } 44 | 45 | struct PageCache { 46 | lru: LruVec, 47 | cached_pages: HashMap>, 48 | modified_pages: HashSet, // pages in cache not yet written to disk 49 | } 50 | 51 | impl PageCache { 52 | pub fn new(capacity: usize) -> Self { 53 | assert!(capacity >= 1); 54 | 55 | Self { 56 | lru: LruVec::new(capacity), 57 | cached_pages: HashMap::with_capacity(capacity + 1), // +1 due to statement order 58 | modified_pages: HashSet::with_capacity(capacity + 1), 59 | } 60 | } 61 | 62 | pub fn modified_pages(&self) -> &HashSet { 63 | &self.modified_pages 64 | } 65 | 66 | pub fn clear_modified_pages(&mut self) { 67 | self.modified_pages.clear(); 68 | } 69 | 70 | pub fn contains_page(&mut self, page_id: PageId) -> bool { 71 | self.cached_pages.contains_key(&page_id) 72 | } 73 | 74 | pub fn get_touched(&mut self, page_id: PageId) -> Option<&Page> { 75 | self.lru.touch(&page_id); 76 | self.cached_pages.get(&page_id) 77 | } 78 | 79 | pub fn _peek(&self, page_id: PageId) -> Option<&Page> { 80 | self.cached_pages.get(&page_id) 81 | } 82 | 83 | pub fn get_touched_mut(&mut self, page_id: PageId) -> Option<&mut Page> { 84 | self.lru.touch(&page_id); 85 | self.modified_pages.insert(page_id); 86 | self.cached_pages.get_mut(&page_id) 87 | } 88 | 89 | pub fn set_page_revision(&mut self, page_id: PageId, revision: RevisionId) { 90 | let mut page = self.cached_pages.get_mut(&page_id).unwrap(); 91 | page.revision = revision; 92 | } 93 | 94 | #[must_use] 95 | pub fn put_touched(&mut self, page_id: PageId, page: Page) -> Option> { 96 | self.cached_pages.insert(page_id, page); 97 | self.modified_pages.insert(page_id); 98 | 99 | if let Some(evicted_page_id) = self.lru.insert(page_id) { 100 | let modified = self.modified_pages.remove(&evicted_page_id); 101 | let page = self.cached_pages.remove(&evicted_page_id).unwrap(); 102 | 103 | Some(EvictedPage { 104 | id: evicted_page_id, 105 | page, 106 | modified, 107 | }) 108 | } else { 109 | None 110 | } 111 | } 112 | 113 | // Reserved for when borrow checker doesn't agree 114 | pub fn take(&mut self, page_id: PageId) -> Option> { 115 | self.cached_pages.remove(&page_id) 116 | } 117 | 118 | // Reserved for when borrow checker doesn't agree 119 | pub fn untake(&mut self, page_id: PageId, page: Page) { 120 | self.cached_pages.insert(page_id, page); 121 | } 122 | } 123 | 124 | struct EvictedPage { 125 | id: PageId, 126 | page: Page, 127 | modified: bool, 128 | } 129 | 130 | #[derive(Default)] 131 | struct FileTracker { 132 | pub pending_sync: HashSet, // files written but not fsync()-ed 133 | pub pending_promotion: HashSet, // files not renamed to the main filename 134 | } 135 | 136 | #[derive(Default)] 137 | struct CounterTracker { 138 | dirty: bool, 139 | revision: RevisionId, // revision counter in memory 140 | revision_on_persistence: RevisionId, // revision counter that is saved on disk 141 | root_id: Option, 142 | id_counter: PageId, // current allocated page ID counter 143 | free_id_list: VecDeque, 144 | } 145 | 146 | impl CounterTracker { 147 | pub fn is_dirty(&self) -> bool { 148 | self.dirty 149 | } 150 | 151 | pub fn unset_dirty(&mut self) { 152 | self.dirty = false; 153 | } 154 | 155 | pub fn revision(&self) -> RevisionId { 156 | self.revision 157 | } 158 | 159 | pub fn revision_on_persistence(&self) -> RevisionId { 160 | self.revision_on_persistence 161 | } 162 | 163 | pub fn root_id(&self) -> Option { 164 | self.root_id 165 | } 166 | 167 | pub fn set_root_id(&mut self, value: Option) { 168 | self.dirty = true; 169 | self.root_id = value; 170 | } 171 | 172 | pub fn id_counter(&self) -> PageId { 173 | self.id_counter 174 | } 175 | 176 | pub fn free_id_list(&self) -> &VecDeque { 177 | &self.free_id_list 178 | } 179 | 180 | pub fn restore( 181 | &mut self, 182 | revision: RevisionId, 183 | root_id: Option, 184 | id_counter: PageId, 185 | free_id_list: &[PageId], 186 | ) { 187 | assert!(self.revision == 0); 188 | assert!(self.revision_on_persistence == 0); 189 | assert!(self.root_id == None); 190 | assert!(self.id_counter == 0); 191 | assert!(self.free_id_list.is_empty()); 192 | 193 | self.revision = revision; 194 | self.revision_on_persistence = revision; 195 | self.root_id = root_id; 196 | self.id_counter = id_counter; 197 | self.free_id_list.extend(free_id_list); 198 | } 199 | 200 | pub fn new_page_id(&mut self) -> PageId { 201 | self.dirty = true; 202 | 203 | if let Some(id) = self.free_id_list.pop_front() { 204 | id 205 | } else { 206 | self.id_counter += 1; 207 | self.id_counter 208 | } 209 | } 210 | 211 | pub fn free_page_id(&mut self, page_id: PageId) { 212 | self.dirty = true; 213 | 214 | self.free_id_list.push_back(page_id); 215 | } 216 | 217 | pub fn increment_revision(&mut self) { 218 | self.dirty = true; 219 | self.revision += 1; 220 | } 221 | 222 | pub fn set_revision_persisted(&mut self) { 223 | self.revision_on_persistence = self.revision; 224 | } 225 | } 226 | 227 | enum RevisionFlag { 228 | Current, 229 | New, 230 | NewUnsync, 231 | } 232 | 233 | #[derive(Debug, Clone)] 234 | pub struct PageTableOptions { 235 | pub open_mode: PageOpenMode, 236 | pub page_cache_size: usize, 237 | pub keys_per_node: usize, 238 | pub file_locking: bool, 239 | pub file_sync: VfsSyncOption, 240 | pub compression_level: Option, 241 | } 242 | 243 | impl Default for PageTableOptions { 244 | fn default() -> Self { 245 | Self { 246 | open_mode: PageOpenMode::default(), 247 | page_cache_size: 64, 248 | keys_per_node: 1024, 249 | file_locking: true, 250 | file_sync: VfsSyncOption::Data, 251 | compression_level: Some(3), 252 | } 253 | } 254 | } 255 | 256 | #[derive(Debug, Clone, PartialEq, Eq)] 257 | pub enum PageOpenMode { 258 | LoadOnly, 259 | CreateOnly, 260 | LoadOrCreate, 261 | ReadOnly, 262 | } 263 | 264 | impl Default for PageOpenMode { 265 | fn default() -> Self { 266 | Self::LoadOrCreate 267 | } 268 | } 269 | 270 | pub struct PageTable 271 | where 272 | T: Serialize + DeserializeOwned, 273 | M: Serialize + DeserializeOwned + Clone, 274 | { 275 | options: PageTableOptions, 276 | vfs: Box, 277 | format: Format, 278 | page_cache: PageCache, 279 | file_tracker: FileTracker, 280 | counter_tracker: CounterTracker, 281 | uuid_generator: UuidGenerator, 282 | uuid: Uuid, 283 | closed: bool, 284 | auxiliary_metadata: Option, 285 | } 286 | 287 | impl PageTable 288 | where 289 | T: Serialize + DeserializeOwned, 290 | M: Serialize + DeserializeOwned + Clone, 291 | { 292 | pub fn open( 293 | mut vfs: Box, 294 | options: PageTableOptions, 295 | ) -> Result { 296 | if matches!( 297 | options.open_mode, 298 | PageOpenMode::LoadOnly | PageOpenMode::ReadOnly 299 | ) && !Self::metadata_file_exists(vfs.as_ref())? 300 | { 301 | return Err(Error::InvalidFileFormat { 302 | path: "(directory contents)".to_string(), 303 | message: "not a database", 304 | }); 305 | } 306 | 307 | if options.file_locking { 308 | vfs.lock(LOCK_FILENAME)?; 309 | } 310 | 311 | let metadata_file_exists = Self::metadata_file_exists(vfs.as_ref())?; 312 | 313 | let mut format = Format::default(); 314 | format.set_compression_level(options.compression_level); 315 | 316 | let mut table = Self { 317 | options: options.clone(), 318 | vfs, 319 | format, 320 | page_cache: PageCache::new(options.page_cache_size), 321 | uuid: Uuid::nil(), 322 | file_tracker: FileTracker::default(), 323 | counter_tracker: CounterTracker::default(), 324 | uuid_generator: UuidGenerator::new(), 325 | closed: false, 326 | auxiliary_metadata: None, 327 | }; 328 | 329 | match options.open_mode { 330 | PageOpenMode::LoadOnly | PageOpenMode::ReadOnly => { 331 | table.load_and_restore_metadata()?; 332 | } 333 | PageOpenMode::CreateOnly => { 334 | table.save_new_metadata()?; 335 | } 336 | PageOpenMode::LoadOrCreate => { 337 | if metadata_file_exists { 338 | table.load_and_restore_metadata()?; 339 | } else { 340 | table.save_new_metadata()?; 341 | } 342 | } 343 | } 344 | 345 | Ok(table) 346 | } 347 | 348 | fn metadata_file_exists(vfs: &dyn Vfs) -> Result { 349 | Ok(vfs.exists(METADATA_FILENAME)? 350 | || vfs.exists(METADATA_COPY_FILENAME)? 351 | || vfs.exists(METADATA_OLD_FILENAME)?) 352 | } 353 | 354 | pub fn root_id(&self) -> Option { 355 | self.counter_tracker.root_id() 356 | } 357 | 358 | pub fn set_root_id(&mut self, value: Option) { 359 | self.counter_tracker.set_root_id(value); 360 | } 361 | 362 | pub fn new_page_id(&mut self) -> PageId { 363 | self.counter_tracker.new_page_id() 364 | } 365 | 366 | pub fn auxiliary_metadata(&self) -> Option<&M> { 367 | self.auxiliary_metadata.as_ref() 368 | } 369 | 370 | pub fn auxiliary_metadata_mut(&mut self) -> Option<&mut M> { 371 | self.auxiliary_metadata.as_mut() 372 | } 373 | 374 | pub fn set_auxiliary_metadata(&mut self, value: Option) { 375 | self.auxiliary_metadata = value; 376 | } 377 | 378 | pub fn get(&mut self, page_id: PageId) -> Result, Error> { 379 | self.check_if_closed()?; 380 | 381 | self.get_(page_id) 382 | } 383 | 384 | fn get_(&mut self, page_id: PageId) -> Result, Error> { 385 | self.check_page_id_counter_consistency(page_id)?; 386 | 387 | if !self.page_cache.contains_page(page_id) { 388 | self.load_page_into_cache(page_id)?; 389 | } 390 | 391 | if let Some(page) = self.page_cache.get_touched(page_id) { 392 | if let Some(content) = &page.content { 393 | Ok(Some(content)) 394 | } else { 395 | Ok(None) 396 | } 397 | } else { 398 | Ok(None) 399 | } 400 | } 401 | 402 | pub fn put(&mut self, page_id: PageId, content: T) -> Result<(), Error> { 403 | self.check_if_closed()?; 404 | self.check_if_read_only()?; 405 | 406 | let result = self.put_(page_id, content); 407 | 408 | if result.is_err() { 409 | self.closed = true; 410 | } 411 | 412 | result 413 | } 414 | 415 | fn put_(&mut self, page_id: PageId, content: T) -> Result<(), Error> { 416 | self.check_page_id_counter_consistency(page_id)?; 417 | 418 | let page = Page { 419 | uuid: self.uuid, 420 | id: page_id, 421 | revision: self.counter_tracker.revision(), 422 | deleted: false, 423 | content: Some(content), 424 | }; 425 | 426 | if let Some(evicted_page_info) = self.page_cache.put_touched(page_id, page) { 427 | self.maybe_save_evicted_page(evicted_page_info)?; 428 | } 429 | 430 | Ok(()) 431 | } 432 | 433 | pub fn update(&mut self, page_id: PageId) -> Result>, Error> { 434 | self.check_if_closed()?; 435 | self.check_if_read_only()?; 436 | 437 | self.update_(page_id) 438 | } 439 | 440 | fn update_(&mut self, page_id: PageId) -> Result>, Error> { 441 | self.check_page_id_counter_consistency(page_id)?; 442 | 443 | if !self.page_cache.contains_page(page_id) { 444 | self.load_page_into_cache(page_id)?; 445 | } 446 | 447 | if let Some(page) = self.page_cache.get_touched_mut(page_id) { 448 | if page.content.is_some() { 449 | Ok(Some(PageUpdateGuard::new(page))) 450 | } else { 451 | Ok(None) 452 | } 453 | } else { 454 | Ok(None) 455 | } 456 | } 457 | 458 | pub fn remove(&mut self, page_id: PageId) -> Result<(), Error> { 459 | self.check_if_closed()?; 460 | self.check_if_read_only()?; 461 | 462 | let result = self.remove_(page_id); 463 | 464 | if result.is_err() { 465 | self.closed = true; 466 | } 467 | 468 | result 469 | } 470 | 471 | fn remove_(&mut self, page_id: PageId) -> Result<(), Error> { 472 | self.check_page_id_counter_consistency(page_id)?; 473 | 474 | let page = Page { 475 | uuid: self.uuid, 476 | id: page_id, 477 | revision: self.counter_tracker.revision(), 478 | deleted: true, 479 | content: None, 480 | }; 481 | 482 | if let Some(evicted_page_info) = self.page_cache.put_touched(page_id, page) { 483 | self.maybe_save_evicted_page(evicted_page_info)?; 484 | } 485 | 486 | self.counter_tracker.free_page_id(page_id); 487 | 488 | Ok(()) 489 | } 490 | 491 | pub fn commit(&mut self) -> Result<(), Error> { 492 | self.check_if_closed()?; 493 | self.check_if_read_only()?; 494 | 495 | let result = self.commit_(); 496 | 497 | if result.is_err() { 498 | self.closed = true; 499 | } 500 | 501 | result 502 | } 503 | 504 | fn commit_(&mut self) -> Result<(), Error> { 505 | if !self.is_anything_modified() { 506 | return Ok(()); 507 | } 508 | 509 | self.counter_tracker.increment_revision(); 510 | 511 | self.save_all_modified_pages()?; 512 | self.sync_and_rename_pending_page_files()?; 513 | self.file_tracker.pending_sync.clear(); 514 | self.save_metadata()?; 515 | self.commit_counters(); 516 | self.promote_page_filenames()?; 517 | self.file_tracker.pending_promotion.clear(); 518 | self.page_cache.clear_modified_pages(); 519 | 520 | Ok(()) 521 | } 522 | 523 | fn is_anything_modified(&self) -> bool { 524 | self.counter_tracker.is_dirty() || !self.page_cache.modified_pages().is_empty() 525 | } 526 | 527 | fn load_and_restore_metadata(&mut self) -> Result<(), Error> { 528 | let metadata: Metadata = self 529 | .format 530 | .read_file(self.vfs.as_mut(), METADATA_FILENAME)?; 531 | 532 | self.uuid = metadata.uuid; 533 | 534 | self.counter_tracker.restore( 535 | metadata.revision, 536 | metadata.root_id, 537 | metadata.id_counter, 538 | &metadata.free_id_list, 539 | ); 540 | 541 | self.auxiliary_metadata = metadata.auxiliary; 542 | 543 | // TODO: the copy backup file could be read if the main metadata file 544 | // is unreadable 545 | 546 | Ok(()) 547 | } 548 | 549 | fn save_new_metadata(&mut self) -> Result<(), Error> { 550 | self.uuid = self.uuid_generator.new_uuid(); 551 | 552 | // We check for the backup file too in case the main file disappears 553 | if self.vfs.exists(METADATA_FILENAME)? 554 | || self.vfs.exists(METADATA_COPY_FILENAME)? 555 | || self.vfs.exists(METADATA_OLD_FILENAME)? 556 | { 557 | return Err(Error::InvalidMetadata { 558 | message: "database already exists", 559 | }); 560 | } 561 | 562 | self.save_metadata()?; 563 | 564 | Ok(()) 565 | } 566 | 567 | fn load_page( 568 | &mut self, 569 | page_id: PageId, 570 | revision_flag: RevisionFlag, 571 | ) -> Result>, Error> { 572 | let path = make_path(page_id, revision_flag); 573 | 574 | if !self.vfs.exists(&path)? { 575 | return Ok(None); 576 | } 577 | 578 | let page: Page = self.format.read_file(self.vfs.as_mut(), &path)?; 579 | 580 | if !self.uuid.is_nil() && page.uuid != self.uuid { 581 | return Err(Error::InvalidPageData { 582 | page: page_id, 583 | message: "wrong UUID", 584 | }); 585 | } 586 | 587 | if page.id != page_id { 588 | return Err(Error::InvalidPageData { 589 | page: page_id, 590 | message: "wrong page ID", 591 | }); 592 | } 593 | 594 | Ok(Some(page)) 595 | } 596 | 597 | fn load_latest_known_page(&mut self, page_id: PageId) -> Result>, Error> { 598 | if self.file_tracker.pending_sync.contains(&page_id) { 599 | let page_2 = self.load_page(page_id, RevisionFlag::NewUnsync)?; 600 | 601 | if let Some(page) = page_2 { 602 | if page.revision <= self.counter_tracker.revision() { 603 | return Ok(Some(page)); 604 | } 605 | } 606 | } 607 | 608 | let page_1 = self.load_page(page_id, RevisionFlag::New)?; 609 | 610 | if let Some(page) = page_1 { 611 | if page.revision <= self.counter_tracker.revision() { 612 | self.maybe_queue_page_for_filename_promotion(&page); 613 | 614 | return Ok(Some(page)); 615 | } 616 | } 617 | 618 | let page_0 = self.load_page(page_id, RevisionFlag::Current)?; 619 | 620 | if let Some(page) = page_0 { 621 | if page.revision <= self.counter_tracker.revision() { 622 | return Ok(Some(page)); 623 | } else { 624 | return Err(Error::InvalidPageData { 625 | page: page_id, 626 | message: "missing page", 627 | }); 628 | } 629 | } 630 | 631 | Ok(None) 632 | } 633 | 634 | fn load_page_into_cache(&mut self, page_id: PageId) -> Result { 635 | let page = self.load_latest_known_page(page_id)?; 636 | 637 | if let Some(page) = page { 638 | if page.deleted || page.content.is_none() { 639 | return Ok(false); 640 | } 641 | 642 | if let Some(evicted_page_info) = self.page_cache.put_touched(page_id, page) { 643 | self.maybe_save_evicted_page(evicted_page_info)?; 644 | } 645 | 646 | Ok(true) 647 | } else { 648 | Ok(false) 649 | } 650 | } 651 | 652 | fn save_page(&mut self, page_id: PageId, page: &Page) -> Result<(), Error> { 653 | self.check_if_read_only()?; 654 | 655 | if self.options.file_sync == VfsSyncOption::None { 656 | self.save_page_by_overwrite(page_id, page)?; 657 | } else { 658 | self.save_page_with_delayed_sync(page_id, page)?; 659 | } 660 | // TODO: provide an option for the user to decide, or stop queueing once 661 | // the queues are getting relatively full, 662 | // but it will introduce more code path test complexity. 663 | // } else { 664 | // self.save_page_by_atomic(page_id, page)?; 665 | // } 666 | 667 | Ok(()) 668 | } 669 | 670 | fn save_page_by_overwrite(&mut self, page_id: PageId, page: &Page) -> Result<(), Error> { 671 | let path_1 = make_path(page_id, RevisionFlag::New); 672 | self.format 673 | .write_file(self.vfs.as_mut(), &path_1, page, VfsSyncOption::None)?; 674 | Ok(()) 675 | } 676 | 677 | fn save_page_with_delayed_sync( 678 | &mut self, 679 | page_id: PageId, 680 | page: &Page, 681 | ) -> Result<(), Error> { 682 | let path_2 = make_path(page_id, RevisionFlag::NewUnsync); 683 | 684 | self.format 685 | .write_file(self.vfs.as_mut(), &path_2, page, VfsSyncOption::None)?; 686 | 687 | self.file_tracker.pending_sync.insert(page_id); 688 | 689 | Ok(()) 690 | } 691 | 692 | fn _save_page_by_atomic(&mut self, page_id: PageId, page: &Page) -> Result<(), Error> { 693 | let path_1 = make_path(page_id, RevisionFlag::New); 694 | let path_1_temp = format!("{}.tmp", &path_1); 695 | 696 | self.format.write_file( 697 | self.vfs.as_mut(), 698 | &path_1_temp, 699 | page, 700 | self.options.file_sync, 701 | )?; 702 | 703 | self.vfs.rename_file(&path_1_temp, &path_1)?; 704 | self.file_tracker.pending_promotion.insert(page_id); 705 | 706 | Ok(()) 707 | } 708 | 709 | fn save_page_from_cache(&mut self, page_id: PageId) -> Result<(), Error> { 710 | self.check_if_read_only()?; 711 | 712 | let page = self.page_cache.take(page_id).unwrap(); 713 | let result = self.save_page(page_id, &page); 714 | self.page_cache.untake(page_id, page); 715 | 716 | result?; 717 | 718 | Ok(()) 719 | } 720 | 721 | fn commit_counters(&mut self) { 722 | self.counter_tracker.unset_dirty(); 723 | self.counter_tracker.set_revision_persisted(); 724 | } 725 | 726 | fn save_metadata(&mut self) -> Result<(), Error> { 727 | self.check_if_read_only()?; 728 | 729 | let metadata = Metadata { 730 | uuid: self.uuid, 731 | revision: self.counter_tracker.revision(), 732 | id_counter: self.counter_tracker.id_counter(), 733 | root_id: self.counter_tracker.root_id(), 734 | free_id_list: self 735 | .counter_tracker 736 | .free_id_list() 737 | .iter() 738 | .cloned() 739 | .collect(), 740 | auxiliary: self.auxiliary_metadata.clone(), 741 | }; 742 | 743 | if self.vfs.exists(METADATA_FILENAME)? { 744 | let data = self.vfs.read(METADATA_FILENAME)?; 745 | self.vfs 746 | .write(METADATA_OLD_FILENAME, &data, self.options.file_sync)?; 747 | } 748 | 749 | if self.options.file_sync == VfsSyncOption::None { 750 | self.format.write_file( 751 | self.vfs.as_mut(), 752 | METADATA_FILENAME, 753 | metadata.clone(), 754 | self.options.file_sync, 755 | )?; 756 | } else { 757 | self.format.write_file( 758 | self.vfs.as_mut(), 759 | METADATA_NEW_FILENAME, 760 | metadata.clone(), 761 | self.options.file_sync, 762 | )?; 763 | 764 | self.vfs 765 | .rename_file(METADATA_NEW_FILENAME, METADATA_FILENAME)?; 766 | } 767 | 768 | self.format.write_file( 769 | self.vfs.as_mut(), 770 | METADATA_COPY_FILENAME, 771 | metadata, 772 | self.options.file_sync, 773 | )?; 774 | 775 | Ok(()) 776 | } 777 | 778 | fn maybe_save_evicted_page(&mut self, evicted_page_info: EvictedPage) -> Result<(), Error> { 779 | if self.options.open_mode != PageOpenMode::ReadOnly && evicted_page_info.modified { 780 | self.save_evicted_page(evicted_page_info.id, evicted_page_info.page)?; 781 | } 782 | 783 | Ok(()) 784 | } 785 | 786 | fn save_evicted_page(&mut self, page_id: PageId, mut page: Page) -> Result<(), Error> { 787 | self.counter_tracker.increment_revision(); 788 | page.revision = self.counter_tracker.revision(); 789 | 790 | self.save_page(page_id, &page)?; 791 | 792 | Ok(()) 793 | } 794 | 795 | fn save_all_modified_pages(&mut self) -> Result<(), Error> { 796 | let page_ids: Vec = self.page_cache.modified_pages().iter().cloned().collect(); 797 | 798 | for page_id in page_ids { 799 | self.page_cache 800 | .set_page_revision(page_id, self.counter_tracker.revision()); 801 | 802 | self.save_page_from_cache(page_id)?; 803 | } 804 | 805 | Ok(()) 806 | } 807 | 808 | fn sync_and_rename_pending_page_files(&mut self) -> Result<(), Error> { 809 | let page_ids: Vec = self.file_tracker.pending_sync.iter().cloned().collect(); 810 | 811 | for page_id in &page_ids { 812 | self.sync_pending_page_file(*page_id)?; 813 | } 814 | for page_id in &page_ids { 815 | self.rename_pending_page_file(*page_id)?; 816 | } 817 | 818 | Ok(()) 819 | } 820 | 821 | fn sync_pending_page_file(&mut self, page_id: PageId) -> Result<(), Error> { 822 | let path_2 = make_path(page_id, RevisionFlag::NewUnsync); 823 | 824 | self.vfs.sync_file(&path_2, self.options.file_sync)?; 825 | 826 | Ok(()) 827 | } 828 | 829 | fn rename_pending_page_file(&mut self, page_id: PageId) -> Result<(), Error> { 830 | let path_1 = make_path(page_id, RevisionFlag::New); 831 | let path_2 = make_path(page_id, RevisionFlag::NewUnsync); 832 | 833 | self.vfs.rename_file(&path_2, &path_1)?; 834 | self.file_tracker.pending_promotion.insert(page_id); 835 | 836 | Ok(()) 837 | } 838 | 839 | fn promote_page_filename(&mut self, page_id: PageId) -> Result<(), Error> { 840 | self.check_if_read_only()?; 841 | 842 | assert!(self.file_tracker.pending_sync.is_empty()); 843 | 844 | let path_0 = make_path(page_id, RevisionFlag::Current); 845 | let path_1 = make_path(page_id, RevisionFlag::New); 846 | 847 | self.vfs.rename_file(&path_1, &path_0)?; 848 | 849 | Ok(()) 850 | } 851 | 852 | fn maybe_queue_page_for_filename_promotion(&mut self, page: &Page) { 853 | if self.options.open_mode != PageOpenMode::ReadOnly 854 | && page.revision <= self.counter_tracker.revision_on_persistence() 855 | { 856 | // This case is possible when the process crashed after 857 | // writing metadata, but before all filenames were promoted. 858 | // Possibly in the future, the queue is too large and not all pages 859 | // were promoted to reduce memory usage. 860 | 861 | self.file_tracker.pending_promotion.insert(page.id); 862 | } 863 | } 864 | 865 | fn promote_page_filenames(&mut self) -> Result<(), Error> { 866 | assert!(self.counter_tracker.revision_on_persistence() == self.counter_tracker.revision()); 867 | assert!(self.file_tracker.pending_sync.is_empty()); 868 | 869 | let page_ids: Vec = self 870 | .file_tracker 871 | .pending_promotion 872 | .iter() 873 | .cloned() 874 | .collect(); 875 | 876 | for page_id in page_ids { 877 | self.promote_page_filename(page_id)?; 878 | } 879 | 880 | Ok(()) 881 | } 882 | 883 | fn check_if_closed(&self) -> Result<(), Error> { 884 | if self.closed { 885 | Err(Error::Closed) 886 | } else { 887 | Ok(()) 888 | } 889 | } 890 | 891 | fn check_if_read_only(&self) -> Result<(), Error> { 892 | if let PageOpenMode::ReadOnly = &self.options.open_mode { 893 | Err(Error::ReadOnly) 894 | } else { 895 | Ok(()) 896 | } 897 | } 898 | 899 | fn check_page_id_counter_consistency(&self, page_id: PageId) -> Result<(), Error> { 900 | if page_id <= self.counter_tracker.id_counter() { 901 | Ok(()) 902 | } else { 903 | Err(Error::InvalidPageData { 904 | page: page_id, 905 | message: "requested page beyond id counter", 906 | }) 907 | } 908 | } 909 | } 910 | 911 | impl Drop for PageTable 912 | where 913 | T: Serialize + DeserializeOwned, 914 | M: Serialize + DeserializeOwned + Clone, 915 | { 916 | fn drop(&mut self) { 917 | if self.options.file_locking { 918 | let _ = self.vfs.unlock(LOCK_FILENAME); 919 | } 920 | } 921 | } 922 | 923 | pub struct PageUpdateGuard<'a, T> { 924 | page: &'a mut Page, 925 | content: Option, 926 | } 927 | 928 | impl<'a, T> PageUpdateGuard<'a, T> { 929 | pub fn new(page: &'a mut Page) -> Self { 930 | let content = page.content.take().unwrap(); 931 | 932 | Self { 933 | page, 934 | content: Some(content), 935 | } 936 | } 937 | } 938 | 939 | impl<'a, T> Deref for PageUpdateGuard<'a, T> { 940 | type Target = T; 941 | 942 | fn deref(&self) -> &Self::Target { 943 | self.content.as_ref().unwrap() 944 | } 945 | } 946 | 947 | impl<'a, T> DerefMut for PageUpdateGuard<'a, T> { 948 | fn deref_mut(&mut self) -> &mut Self::Target { 949 | self.content.as_mut().unwrap() 950 | } 951 | } 952 | 953 | impl<'a, T> Drop for PageUpdateGuard<'a, T> { 954 | fn drop(&mut self) { 955 | let content = self.content.take().unwrap(); 956 | self.page.content.replace(content); 957 | } 958 | } 959 | 960 | fn make_path(page_id: PageId, revision_flag: RevisionFlag) -> String { 961 | format!( 962 | "{}/{}", 963 | split_number(page_id), 964 | make_filename(page_id, revision_flag) 965 | ) 966 | } 967 | 968 | fn make_filename(page_id: PageId, revision_flag: RevisionFlag) -> String { 969 | format!( 970 | "grebedb_{:016x}_{}.grebedb", 971 | page_id, 972 | match revision_flag { 973 | RevisionFlag::Current => 0, 974 | RevisionFlag::New => 1, 975 | RevisionFlag::NewUnsync => 2, 976 | } 977 | ) 978 | } 979 | 980 | fn split_number(mut id: u64) -> String { 981 | let mut parts = [0u64; 8]; 982 | let bits = 8; 983 | let mask = 0xff; 984 | 985 | for index in (0..bits).rev() { 986 | parts[index] = id & mask; 987 | id >>= bits; 988 | } 989 | 990 | format!( 991 | "{:02x}/{:02x}/{:02x}/{:02x}/{:02x}/{:02x}/{:02x}", 992 | parts[0], parts[1], parts[2], parts[3], parts[4], parts[5], parts[6] 993 | ) 994 | } 995 | 996 | #[cfg(test)] 997 | mod tests { 998 | use crate::vfs::MemoryVfs; 999 | 1000 | use super::*; 1001 | 1002 | #[test] 1003 | fn test_split_number() { 1004 | assert_eq!(&split_number(0), "00/00/00/00/00/00/00"); 1005 | assert_eq!(&split_number(1), "00/00/00/00/00/00/00"); 1006 | assert_eq!(&split_number(0xaabb_ccdd), "00/00/00/00/aa/bb/cc"); 1007 | assert_eq!(&split_number(0xaabb_ccdd_1122_3344), "aa/bb/cc/dd/11/22/33"); 1008 | } 1009 | 1010 | #[test] 1011 | fn test_make_filename() { 1012 | assert_eq!( 1013 | &make_filename(0, RevisionFlag::Current), 1014 | "grebedb_0000000000000000_0.grebedb" 1015 | ); 1016 | assert_eq!( 1017 | &make_filename(0, RevisionFlag::New), 1018 | "grebedb_0000000000000000_1.grebedb" 1019 | ); 1020 | assert_eq!( 1021 | &make_filename(12345678, RevisionFlag::Current), 1022 | "grebedb_0000000000bc614e_0.grebedb" 1023 | ); 1024 | assert_eq!( 1025 | &make_filename(0xaabb_ccdd, RevisionFlag::New), 1026 | "grebedb_00000000aabbccdd_1.grebedb" 1027 | ); 1028 | } 1029 | 1030 | #[test] 1031 | fn test_page_table_create_load() { 1032 | let vfs = MemoryVfs::new(); 1033 | 1034 | let options = PageTableOptions { 1035 | open_mode: PageOpenMode::CreateOnly, 1036 | ..Default::default() 1037 | }; 1038 | 1039 | let mut page_table = PageTable::::open(Box::new(vfs.clone()), options).unwrap(); 1040 | 1041 | let page_id = page_table.new_page_id(); 1042 | page_table.put(page_id, 789).unwrap(); 1043 | 1044 | page_table.commit().unwrap(); 1045 | 1046 | drop(page_table); 1047 | 1048 | let options = PageTableOptions { 1049 | open_mode: PageOpenMode::LoadOnly, 1050 | ..Default::default() 1051 | }; 1052 | 1053 | let mut page_table = PageTable::::open(Box::new(vfs), options).unwrap(); 1054 | 1055 | let content = page_table.get(page_id).unwrap(); 1056 | assert_eq!(content.cloned(), Some(789)); 1057 | } 1058 | 1059 | #[test] 1060 | fn test_page_table_create_load_exists() { 1061 | let vfs = MemoryVfs::new(); 1062 | 1063 | let options = PageTableOptions { 1064 | open_mode: PageOpenMode::LoadOnly, 1065 | ..Default::default() 1066 | }; 1067 | 1068 | assert!(PageTable::<()>::open(Box::new(vfs.clone()), options).is_err()); 1069 | 1070 | let options = PageTableOptions { 1071 | open_mode: PageOpenMode::CreateOnly, 1072 | ..Default::default() 1073 | }; 1074 | 1075 | let _page_table = PageTable::<()>::open(Box::new(vfs.clone()), options).unwrap(); 1076 | 1077 | let _page_table = 1078 | PageTable::<()>::open(Box::new(vfs), PageTableOptions::default()).unwrap(); 1079 | } 1080 | 1081 | #[test] 1082 | fn test_page_table_get_put() { 1083 | let vfs = MemoryVfs::new(); 1084 | let mut page_table = 1085 | PageTable::::open(Box::new(vfs), PageTableOptions::default()).unwrap(); 1086 | 1087 | let page_id = page_table.new_page_id(); 1088 | 1089 | assert_eq!(page_table.get(page_id).unwrap(), None); 1090 | 1091 | page_table.put(page_id, 789).unwrap(); 1092 | 1093 | let content = page_table.get(page_id).unwrap(); 1094 | 1095 | assert_eq!(content.cloned(), Some(789)); 1096 | 1097 | page_table.set_root_id(Some(page_id)); 1098 | assert_eq!(Some(page_id), page_table.root_id()); 1099 | } 1100 | 1101 | #[test] 1102 | fn test_page_table_update() { 1103 | let vfs = MemoryVfs::new(); 1104 | let mut page_table = 1105 | PageTable::::open(Box::new(vfs.clone()), PageTableOptions::default()).unwrap(); 1106 | 1107 | let page_id = page_table.new_page_id(); 1108 | 1109 | page_table.put(page_id, 789).unwrap(); 1110 | 1111 | { 1112 | let mut guard = page_table.update(page_id).unwrap().unwrap(); 1113 | *guard = 123; 1114 | } 1115 | 1116 | let content = page_table.get(page_id).unwrap(); 1117 | assert_eq!(content.cloned(), Some(123)); 1118 | 1119 | page_table.commit().unwrap(); 1120 | 1121 | drop(page_table); 1122 | 1123 | let mut page_table = 1124 | PageTable::::open(Box::new(vfs), PageTableOptions::default()).unwrap(); 1125 | 1126 | let content = page_table.get(page_id).unwrap(); 1127 | assert_eq!(content.cloned(), Some(123)); 1128 | } 1129 | 1130 | #[test] 1131 | fn test_page_table_many_on_single_page() { 1132 | let vfs = MemoryVfs::new(); 1133 | let mut page_table = 1134 | PageTable::::open(Box::new(vfs), PageTableOptions::default()).unwrap(); 1135 | 1136 | let page_id = page_table.new_page_id(); 1137 | 1138 | for num in 0..10 { 1139 | page_table.put(page_id, 1000 + num).unwrap(); 1140 | } 1141 | 1142 | let content = page_table.get(page_id).unwrap(); 1143 | 1144 | assert_eq!(content.cloned(), Some(1000 + 9)); 1145 | } 1146 | 1147 | #[test] 1148 | fn test_page_table_many_pages() { 1149 | let vfs = MemoryVfs::new(); 1150 | let mut page_table = 1151 | PageTable::::open(Box::new(vfs), PageTableOptions::default()).unwrap(); 1152 | 1153 | let mut first_page_id = None; 1154 | 1155 | // `num` must be bigger than page cache size 1156 | for num in 0..100 { 1157 | let page_id = page_table.new_page_id(); 1158 | 1159 | if first_page_id.is_none() { 1160 | first_page_id = Some(page_id); 1161 | } 1162 | 1163 | page_table.put(page_id, 1000 + num).unwrap(); 1164 | } 1165 | 1166 | for num in 0..100 { 1167 | let content = page_table.get(first_page_id.unwrap() + num).unwrap(); 1168 | 1169 | assert_eq!(content.cloned(), Some(1000 + num)); 1170 | } 1171 | } 1172 | 1173 | #[test] 1174 | fn test_page_table_remove() { 1175 | let vfs = MemoryVfs::new(); 1176 | let mut page_table = 1177 | PageTable::::open(Box::new(vfs), PageTableOptions::default()).unwrap(); 1178 | 1179 | let page_id = page_table.new_page_id(); 1180 | let page_id_2 = page_table.new_page_id(); 1181 | 1182 | page_table.put(page_id, 123).unwrap(); 1183 | page_table.put(page_id_2, 456).unwrap(); 1184 | 1185 | page_table.remove(page_id).unwrap(); 1186 | 1187 | assert!(page_table.get(page_id).unwrap().is_none()); 1188 | 1189 | // removing already removed should not error 1190 | page_table.remove(page_id).unwrap(); 1191 | assert!(page_table.get(page_id).unwrap().is_none()); 1192 | 1193 | let page_id_3 = page_table.new_page_id(); 1194 | assert_eq!(page_id_3, page_id); // check that id is recycled from free list 1195 | assert_eq!(page_table.get(page_id_3).unwrap(), None); 1196 | assert_eq!(page_table.get(page_id_2).unwrap().cloned(), Some(456)); 1197 | } 1198 | } 1199 | -------------------------------------------------------------------------------- /src/library/src/system.rs: -------------------------------------------------------------------------------- 1 | use uuid::Uuid; 2 | 3 | pub struct UuidGenerator {} 4 | 5 | impl UuidGenerator { 6 | pub fn new() -> Self { 7 | Self {} 8 | } 9 | 10 | #[cfg(feature = "system")] 11 | pub fn new_uuid(&self) -> Uuid { 12 | Uuid::new_v4() 13 | } 14 | 15 | #[cfg(not(feature = "system"))] 16 | pub fn new_uuid(&self) -> Uuid { 17 | Uuid::nil() 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/library/src/vfs.rs: -------------------------------------------------------------------------------- 1 | //! Virtual file system interface for database storage. 2 | 3 | use std::{fmt::Debug, io::Write, path::PathBuf}; 4 | 5 | #[cfg(feature = "fslock")] 6 | use std::collections::HashMap; 7 | 8 | use relative_path::{RelativePath, RelativePathBuf}; 9 | use vfs::{MemoryFS, VfsFileType, VfsPath}; 10 | 11 | use crate::error::Error; 12 | 13 | /// Represents a virtual file system. 14 | /// 15 | /// File paths are characters within pattern `[a-z0-9._]` in Unix style 16 | /// where directory separators as slashes (`/`). Paths are specified in 17 | /// relative notation such as `example/my_file.ext`. 18 | /// 19 | /// Implementations are not expected to support directory traversal notations 20 | /// or handling redundant slashes. Implementations can return an error in 21 | /// those cases. 22 | pub trait Vfs { 23 | /// Lock the file preventing other processes from accessing it. 24 | /// 25 | /// If the file is already locked, an error is returned. 26 | fn lock(&mut self, path: &str) -> Result<(), Error>; 27 | 28 | /// Unlock the file. 29 | /// 30 | /// If the file is not locked, an error is returned. 31 | fn unlock(&mut self, path: &str) -> Result<(), Error>; 32 | 33 | /// Read the contents of a file to a vector. 34 | fn read(&self, path: &str) -> Result, Error>; 35 | 36 | /// Write the contents to a file. 37 | /// 38 | /// The file will be created if it does not exist and existing data is 39 | /// overwritten. 40 | /// 41 | /// If `sync_option` is a flushing operation, it will flush data from 42 | /// buffers to persistent storage before returning. 43 | fn write(&mut self, path: &str, data: &[u8], sync_option: VfsSyncOption) -> Result<(), Error>; 44 | 45 | /// Flush buffered data of a file to persistent storage. 46 | /// 47 | /// If supported by the file system, the method calls the appropriate 48 | /// sync operation on an existing, writable file without modifying the file 49 | /// contents. Flush operations complete before returning. 50 | fn sync_file(&mut self, path: &str, sync_option: VfsSyncOption) -> Result<(), Error>; 51 | 52 | /// Delete a file. 53 | /// 54 | /// If the file does not exist, an error is returned. 55 | fn remove_file(&mut self, path: &str) -> Result<(), Error>; 56 | 57 | /// Return a vector of filenames in a directory. 58 | fn read_dir(&self, path: &str) -> Result, Error>; 59 | 60 | /// Create a directory at the given path. 61 | /// 62 | /// The parent directory must exist. 63 | fn create_dir(&mut self, path: &str) -> Result<(), Error>; 64 | 65 | /// Create directories for all components of the path if they do not exist. 66 | fn create_dir_all(&mut self, path: &str) -> Result<(), Error> { 67 | let mut current_path = RelativePathBuf::default(); 68 | for part in RelativePath::new(path).components() { 69 | current_path.push(part.as_str()); 70 | 71 | if !self.exists(current_path.as_str())? { 72 | self.create_dir(current_path.as_str())?; 73 | } 74 | } 75 | 76 | Ok(()) 77 | } 78 | 79 | /// Remove an empty directory. 80 | /// 81 | /// If the path is not an empty directory, an error is returned. 82 | fn remove_dir(&mut self, path: &str) -> Result<(), Error>; 83 | 84 | /// Remove empty directories in the path if they exist. 85 | fn remove_empty_dir_all(&mut self, path: &str) -> Result<(), Error> { 86 | let mut current_path = RelativePathBuf::from(path); 87 | 88 | loop { 89 | if current_path.as_str() != "" && self.read_dir(current_path.as_str())?.is_empty() { 90 | self.remove_dir(current_path.as_str())?; 91 | } else { 92 | break; 93 | } 94 | 95 | if let Some(parent) = current_path.parent() { 96 | current_path = parent.to_owned(); 97 | } else { 98 | break; 99 | } 100 | } 101 | 102 | Ok(()) 103 | } 104 | 105 | /// Rename a file. 106 | /// 107 | /// If the destination file path already exists, the file is overwritten. 108 | fn rename_file(&mut self, old_path: &str, new_path: &str) -> Result<(), Error>; 109 | 110 | /// Return whether the path is a directory. 111 | /// 112 | /// Returns an error if the path does not exist. 113 | fn is_dir(&self, path: &str) -> Result; 114 | 115 | /// Return whether the path exists. 116 | fn exists(&self, path: &str) -> Result; 117 | } 118 | 119 | /// File system synchronization options for synchronizing data to disk. 120 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 121 | pub enum VfsSyncOption { 122 | /// Don't require any flushing. 123 | None, 124 | 125 | /// Flush file content only. Equivalent to `File::sync_data()` or Unix `fdatasync()`. 126 | Data, 127 | 128 | /// Flush file content and metadata. Equivalent to `File::sync_all()` or Unix `fsync()`. 129 | All, 130 | } 131 | 132 | impl Default for VfsSyncOption { 133 | fn default() -> Self { 134 | Self::None 135 | } 136 | } 137 | 138 | /// A file system that is stored temporarily to memory. 139 | #[derive(Clone)] 140 | pub struct MemoryVfs { 141 | vfs: VfsPath, 142 | } 143 | 144 | impl MemoryVfs { 145 | /// Create a in-memory file system. 146 | pub fn new() -> Self { 147 | Self { 148 | vfs: VfsPath::new(MemoryFS::default()), 149 | } 150 | } 151 | } 152 | 153 | impl Default for MemoryVfs { 154 | fn default() -> Self { 155 | Self::new() 156 | } 157 | } 158 | 159 | impl Debug for MemoryVfs { 160 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 161 | write!(f, "MemoryVfs") 162 | } 163 | } 164 | 165 | impl Vfs for MemoryVfs { 166 | fn lock(&mut self, _path: &str) -> Result<(), Error> { 167 | Ok(()) 168 | } 169 | 170 | fn unlock(&mut self, _path: &str) -> Result<(), Error> { 171 | Ok(()) 172 | } 173 | 174 | fn read(&self, path: &str) -> Result, Error> { 175 | let mut file = self.vfs.join(path)?.open_file()?; 176 | let mut buffer = Vec::new(); 177 | file.read_to_end(&mut buffer)?; 178 | Ok(buffer) 179 | } 180 | 181 | fn write(&mut self, path: &str, data: &[u8], _sync_option: VfsSyncOption) -> Result<(), Error> { 182 | let mut file = self.vfs.join(path)?.create_file()?; 183 | file.write_all(data)?; 184 | Ok(()) 185 | } 186 | 187 | fn sync_file(&mut self, _path: &str, _sync_option: VfsSyncOption) -> Result<(), Error> { 188 | Ok(()) 189 | } 190 | 191 | fn remove_file(&mut self, path: &str) -> Result<(), Error> { 192 | self.vfs.join(path)?.remove_file()?; 193 | Ok(()) 194 | } 195 | 196 | fn read_dir(&self, path: &str) -> Result, Error> { 197 | let mut filenames = Vec::new(); 198 | 199 | for sub_path in self.vfs.join(path)?.read_dir()? { 200 | filenames.push(sub_path.filename()); 201 | } 202 | 203 | Ok(filenames) 204 | } 205 | 206 | fn create_dir(&mut self, path: &str) -> Result<(), Error> { 207 | self.vfs.join(path)?.create_dir()?; 208 | Ok(()) 209 | } 210 | 211 | fn remove_dir(&mut self, path: &str) -> Result<(), Error> { 212 | self.vfs.join(path)?.remove_dir()?; 213 | Ok(()) 214 | } 215 | 216 | fn rename_file(&mut self, old_path: &str, new_path: &str) -> Result<(), Error> { 217 | if self.exists(new_path)? { 218 | self.remove_file(new_path)?; 219 | } 220 | 221 | self.vfs 222 | .join(old_path)? 223 | .move_file(&self.vfs.join(new_path)?)?; 224 | 225 | Ok(()) 226 | } 227 | 228 | fn is_dir(&self, path: &str) -> Result { 229 | let metadata = self.vfs.join(path)?.metadata()?; 230 | Ok(matches!(metadata.file_type, VfsFileType::Directory)) 231 | } 232 | 233 | fn exists(&self, path: &str) -> Result { 234 | Ok(self.vfs.join(path)?.exists()?) 235 | } 236 | } 237 | 238 | #[cfg(feature = "fslock")] 239 | type LockFileType = fslock::LockFile; 240 | 241 | /// Interface to a real file system on disk. 242 | pub struct OsVfs { 243 | root: PathBuf, 244 | 245 | #[cfg(feature = "fslock")] 246 | locks: HashMap, 247 | } 248 | 249 | impl OsVfs { 250 | /// Create a file system interface to the given path. 251 | /// 252 | /// The given path is treated as the root for subsequent relative path 253 | /// operations. 254 | pub fn new

(root: P) -> Self 255 | where 256 | P: Into, 257 | { 258 | Self { 259 | root: root.into(), 260 | #[cfg(feature = "fslock")] 261 | locks: HashMap::new(), 262 | } 263 | } 264 | } 265 | 266 | impl Debug for OsVfs { 267 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 268 | write!(f, "OsVfs {{ path: {:?} }}", &self.root) 269 | } 270 | } 271 | 272 | impl Vfs for OsVfs { 273 | #[cfg(feature = "fslock")] 274 | fn lock(&mut self, path: &str) -> Result<(), Error> { 275 | let mut lock = fslock::LockFile::open(self.root.join(path).as_path())?; 276 | if !lock.try_lock()? { 277 | return Err(Error::Locked); 278 | } 279 | self.locks.insert(self.root.join(path), lock); 280 | 281 | Ok(()) 282 | } 283 | #[cfg(not(feature = "fslock"))] 284 | fn lock(&mut self, _path: &str) -> Result<(), Error> { 285 | Err(Error::FileLockingUnavailable) 286 | } 287 | 288 | #[cfg(feature = "fslock")] 289 | fn unlock(&mut self, path: &str) -> Result<(), Error> { 290 | if let Some(mut lock) = self.locks.remove(&self.root.join(path)) { 291 | lock.unlock()?; 292 | } else { 293 | return Err(Error::Io(std::io::Error::new( 294 | std::io::ErrorKind::InvalidInput, 295 | "file not locked", 296 | ))); 297 | } 298 | 299 | Ok(()) 300 | } 301 | 302 | #[cfg(not(feature = "fslock"))] 303 | fn unlock(&mut self, _path: &str) -> Result<(), Error> { 304 | Err(Error::FileLockingUnavailable) 305 | } 306 | 307 | fn read(&self, path: &str) -> Result, Error> { 308 | Ok(std::fs::read(self.root.join(path))?) 309 | } 310 | 311 | fn write(&mut self, path: &str, data: &[u8], sync_option: VfsSyncOption) -> Result<(), Error> { 312 | match sync_option { 313 | VfsSyncOption::None => Ok(std::fs::write(self.root.join(path), data)?), 314 | VfsSyncOption::Data => { 315 | let mut file = std::fs::File::create(self.root.join(path))?; 316 | file.write_all(data)?; 317 | file.sync_data()?; 318 | 319 | Ok(()) 320 | } 321 | VfsSyncOption::All => { 322 | let mut file = std::fs::File::create(self.root.join(path))?; 323 | file.write_all(data)?; 324 | file.sync_all()?; 325 | 326 | Ok(()) 327 | } 328 | } 329 | } 330 | 331 | fn sync_file(&mut self, path: &str, sync_option: VfsSyncOption) -> Result<(), Error> { 332 | let file = std::fs::OpenOptions::new() 333 | .append(true) 334 | .open(self.root.join(path))?; 335 | 336 | match sync_option { 337 | VfsSyncOption::None => {} 338 | VfsSyncOption::Data => { 339 | file.sync_data()?; 340 | } 341 | VfsSyncOption::All => { 342 | file.sync_all()?; 343 | } 344 | } 345 | 346 | Ok(()) 347 | } 348 | 349 | fn remove_file(&mut self, path: &str) -> Result<(), Error> { 350 | Ok(std::fs::remove_file(self.root.join(path))?) 351 | } 352 | 353 | fn read_dir(&self, path: &str) -> Result, Error> { 354 | let dir = std::fs::read_dir(self.root.join(path))?; 355 | let mut filenames = Vec::new(); 356 | 357 | for entry in dir { 358 | let entry = entry?; 359 | 360 | if let Ok(filename) = entry.file_name().into_string() { 361 | filenames.push(filename); 362 | } 363 | } 364 | 365 | Ok(filenames) 366 | } 367 | 368 | fn create_dir(&mut self, path: &str) -> Result<(), Error> { 369 | std::fs::create_dir(self.root.join(path))?; 370 | Ok(()) 371 | } 372 | 373 | fn remove_dir(&mut self, path: &str) -> Result<(), Error> { 374 | std::fs::remove_dir(self.root.join(path))?; 375 | Ok(()) 376 | } 377 | 378 | fn rename_file(&mut self, old_path: &str, new_path: &str) -> Result<(), Error> { 379 | std::fs::rename(self.root.join(old_path), self.root.join(new_path))?; 380 | Ok(()) 381 | } 382 | 383 | fn is_dir(&self, path: &str) -> Result { 384 | let metadata = std::fs::metadata(self.root.join(path))?; 385 | 386 | Ok(metadata.is_dir()) 387 | } 388 | 389 | fn exists(&self, path: &str) -> Result { 390 | Ok(self.root.join(path).exists()) 391 | } 392 | } 393 | 394 | /// Wrapper that allows only read operations. 395 | pub struct ReadOnlyVfs { 396 | inner: Box, 397 | } 398 | 399 | impl ReadOnlyVfs { 400 | /// Wrap a VFS. 401 | pub fn new(inner: Box) -> Self { 402 | Self { inner } 403 | } 404 | 405 | /// Return the wrapped VFS. 406 | pub fn into_inner(self) -> Box { 407 | self.inner 408 | } 409 | } 410 | 411 | impl Vfs for ReadOnlyVfs { 412 | fn lock(&mut self, path: &str) -> Result<(), Error> { 413 | self.inner.lock(path) 414 | } 415 | 416 | fn unlock(&mut self, path: &str) -> Result<(), Error> { 417 | self.inner.unlock(path) 418 | } 419 | 420 | fn read(&self, path: &str) -> Result, Error> { 421 | self.inner.read(path) 422 | } 423 | 424 | fn write( 425 | &mut self, 426 | _path: &str, 427 | _data: &[u8], 428 | _sync_option: VfsSyncOption, 429 | ) -> Result<(), Error> { 430 | Err(Error::ReadOnly) 431 | } 432 | 433 | fn sync_file(&mut self, _path: &str, _sync_option: VfsSyncOption) -> Result<(), Error> { 434 | Err(Error::ReadOnly) 435 | } 436 | 437 | fn remove_file(&mut self, _path: &str) -> Result<(), Error> { 438 | Err(Error::ReadOnly) 439 | } 440 | 441 | fn read_dir(&self, path: &str) -> Result, Error> { 442 | self.inner.read_dir(path) 443 | } 444 | 445 | fn create_dir(&mut self, _path: &str) -> Result<(), Error> { 446 | Err(Error::ReadOnly) 447 | } 448 | 449 | fn remove_dir(&mut self, _path: &str) -> Result<(), Error> { 450 | Err(Error::ReadOnly) 451 | } 452 | 453 | fn rename_file(&mut self, _old_path: &str, _new_path: &str) -> Result<(), Error> { 454 | Err(Error::ReadOnly) 455 | } 456 | 457 | fn is_dir(&self, path: &str) -> Result { 458 | self.inner.is_dir(path) 459 | } 460 | 461 | fn exists(&self, path: &str) -> Result { 462 | self.inner.exists(path) 463 | } 464 | } 465 | 466 | impl Debug for ReadOnlyVfs { 467 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 468 | write!(f, "ReadOnlyVfs") 469 | } 470 | } 471 | 472 | #[cfg(test)] 473 | mod tests { 474 | use super::*; 475 | 476 | #[test] 477 | fn test_recursive_helpers() { 478 | let mut vfs = MemoryVfs::new(); 479 | 480 | vfs.create_dir_all("a/b/c").unwrap(); 481 | vfs.write( 482 | "a/b/c/my_file", 483 | "hello world!".as_bytes(), 484 | VfsSyncOption::None, 485 | ) 486 | .unwrap(); 487 | vfs.remove_empty_dir_all("a/b/c").unwrap(); 488 | assert!(vfs.exists("a/b/c").unwrap()); 489 | vfs.remove_file("a/b/c/my_file").unwrap(); 490 | vfs.remove_empty_dir_all("a/b/c").unwrap(); 491 | assert!(!vfs.exists("a/b/c").unwrap()); 492 | } 493 | } 494 | -------------------------------------------------------------------------------- /src/library/tests/common.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{ 2 | atomic::{AtomicBool, Ordering}, 3 | Arc, 4 | }; 5 | 6 | use grebedb::vfs::{MemoryVfs, Vfs, VfsSyncOption}; 7 | use tempfile::TempDir; 8 | 9 | #[allow(dead_code)] 10 | pub fn make_tempdir() -> TempDir { 11 | tempfile::Builder::new() 12 | .prefix("grebetest_") 13 | .tempdir() 14 | .unwrap() 15 | } 16 | 17 | #[macro_export] 18 | macro_rules! matrix_test { 19 | ($fn_name:ident) => { 20 | multiple_config_test!($fn_name, cfg(test)); 21 | }; 22 | } 23 | 24 | #[macro_export] 25 | macro_rules! matrix_test_ignore { 26 | ($fn_name:ident) => { 27 | multiple_config_test!($fn_name, ignore); 28 | }; 29 | } 30 | 31 | #[macro_export] 32 | macro_rules! multiple_vfs_test { 33 | ($fn_name:ident, $option_name:expr, $options:expr, $ignore:meta) => { 34 | paste::paste! { 35 | #[test] 36 | #[$ignore] 37 | fn []() { 38 | let db = grebedb::Database::open_memory($options).unwrap(); 39 | $fn_name(db).unwrap(); 40 | } 41 | } 42 | 43 | paste::paste! { 44 | #[test] 45 | #[$ignore] 46 | fn []() { 47 | let temp_dir = $crate::common::make_tempdir(); 48 | let db = grebedb::Database::open_path(temp_dir.path(), $options).unwrap(); 49 | $fn_name(db).unwrap(); 50 | } 51 | } 52 | }; 53 | } 54 | 55 | #[macro_export] 56 | macro_rules! multiple_config_test { 57 | ($fn_name:ident, $ignore:meta) => { 58 | multiple_vfs_test!($fn_name, "default", grebedb::Options::default(), $ignore); 59 | 60 | multiple_vfs_test!( 61 | $fn_name, 62 | "small_options", 63 | grebedb::Options { 64 | keys_per_node: 128, 65 | page_cache_size: 8, 66 | ..Default::default() 67 | }, 68 | $ignore 69 | ); 70 | 71 | multiple_vfs_test!( 72 | $fn_name, 73 | "tiny_options", 74 | grebedb::Options { 75 | keys_per_node: 16, 76 | page_cache_size: 8, 77 | ..Default::default() 78 | }, 79 | $ignore 80 | ); 81 | }; 82 | } 83 | 84 | #[derive(Clone)] 85 | pub struct CrashingVfs { 86 | inner: MemoryVfs, 87 | pub metadata_rename_crash: Arc, 88 | pub after_metadata_rename_crash: Arc, 89 | metadata_found: Arc, 90 | } 91 | 92 | impl CrashingVfs { 93 | #[allow(dead_code)] 94 | #[allow(clippy::new_without_default)] 95 | pub fn new() -> Self { 96 | Self { 97 | inner: MemoryVfs::default(), 98 | metadata_rename_crash: Arc::new(AtomicBool::new(false)), 99 | after_metadata_rename_crash: Arc::new(AtomicBool::new(false)), 100 | metadata_found: Arc::new(AtomicBool::new(false)), 101 | } 102 | } 103 | 104 | fn make_crash_error() -> grebedb::Error { 105 | grebedb::Error::Io(std::io::Error::new(std::io::ErrorKind::Other, "crash")) 106 | } 107 | } 108 | 109 | impl Vfs for CrashingVfs { 110 | fn lock(&mut self, path: &str) -> Result<(), grebedb::Error> { 111 | self.inner.lock(path) 112 | } 113 | 114 | fn unlock(&mut self, path: &str) -> Result<(), grebedb::Error> { 115 | self.inner.unlock(path) 116 | } 117 | 118 | fn read(&self, path: &str) -> Result, grebedb::Error> { 119 | eprintln!("read {}", path); 120 | self.inner.read(path) 121 | } 122 | 123 | fn write( 124 | &mut self, 125 | path: &str, 126 | data: &[u8], 127 | sync_option: VfsSyncOption, 128 | ) -> Result<(), grebedb::Error> { 129 | eprintln!("write {}", path); 130 | self.inner.write(path, data, sync_option) 131 | } 132 | 133 | fn sync_file(&mut self, path: &str, sync_option: VfsSyncOption) -> Result<(), grebedb::Error> { 134 | eprintln!("sync_file {}", path); 135 | self.inner.sync_file(path, sync_option) 136 | } 137 | 138 | fn remove_file(&mut self, path: &str) -> Result<(), grebedb::Error> { 139 | eprintln!("remove_file {}", path); 140 | self.inner.remove_file(path) 141 | } 142 | 143 | fn read_dir(&self, path: &str) -> Result, grebedb::Error> { 144 | eprintln!("read_dir {}", path); 145 | self.inner.read_dir(path) 146 | } 147 | 148 | fn create_dir(&mut self, path: &str) -> Result<(), grebedb::Error> { 149 | eprintln!("create_dir {}", path); 150 | self.inner.create_dir(path) 151 | } 152 | 153 | fn remove_dir(&mut self, path: &str) -> Result<(), grebedb::Error> { 154 | eprintln!("remove_dir {}", path); 155 | self.inner.remove_dir(path) 156 | } 157 | 158 | fn rename_file(&mut self, old_path: &str, new_path: &str) -> Result<(), grebedb::Error> { 159 | eprintln!("rename_file {} {}", old_path, new_path); 160 | 161 | if new_path == "grebedb_meta.grebedb" { 162 | if self.after_metadata_rename_crash.load(Ordering::Relaxed) { 163 | self.metadata_found.store(true, Ordering::Relaxed); 164 | } 165 | 166 | if self.metadata_rename_crash.load(Ordering::Relaxed) { 167 | eprintln!("crash on metadata rename"); 168 | return Err(Self::make_crash_error()); 169 | } 170 | } else if self.after_metadata_rename_crash.load(Ordering::Relaxed) 171 | && self.metadata_found.load(Ordering::Relaxed) 172 | { 173 | eprintln!("crash on after metadata rename"); 174 | return Err(Self::make_crash_error()); 175 | } 176 | 177 | self.inner.rename_file(old_path, new_path) 178 | } 179 | 180 | fn is_dir(&self, path: &str) -> Result { 181 | eprintln!("is_dir {}", path); 182 | self.inner.is_dir(path) 183 | } 184 | 185 | fn exists(&self, path: &str) -> Result { 186 | // eprintln!("exists {}", path); 187 | self.inner.exists(path) 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /src/library/tests/common_traits.rs: -------------------------------------------------------------------------------- 1 | use grebedb::{Database, Options}; 2 | 3 | #[test] 4 | fn test_debug() { 5 | let mut database = Database::open_memory(Options::default()).unwrap(); 6 | 7 | println!("{:?}", &database); 8 | println!("{:?}", database.cursor()); 9 | } 10 | -------------------------------------------------------------------------------- /src/library/tests/cursor.rs: -------------------------------------------------------------------------------- 1 | mod common; 2 | 3 | use grebedb::{Database, Error}; 4 | 5 | fn cursor_sequential(mut database: Database) -> Result<(), Error> { 6 | for num in 0..10000 { 7 | let key = format!("{:08x}", num); 8 | let value = format!("hello world {}", num); 9 | 10 | database.put(key, value)?; 11 | } 12 | 13 | let cursor = database.cursor()?; 14 | let values: Vec<(Vec, Vec)> = cursor.collect(); 15 | 16 | assert_eq!(values.len(), 10000); 17 | 18 | for (num, (key, value)) in values.iter().enumerate() { 19 | let expected_key = format!("{:08x}", num); 20 | let expected_value = format!("hello world {}", num); 21 | 22 | assert_eq!(key, expected_key.as_bytes()); 23 | assert_eq!(value, expected_value.as_bytes()); 24 | } 25 | 26 | Ok(()) 27 | } 28 | 29 | fn cursor_iter_manual(mut database: Database) -> Result<(), Error> { 30 | for num in 0..500 { 31 | let key = format!("{:08x}", num); 32 | let value = format!("hello world {}", num); 33 | 34 | database.put(key, value)?; 35 | } 36 | 37 | let mut cursor = database.cursor()?; 38 | let mut count = 0; 39 | 40 | while let Some((key, value)) = cursor.next() { 41 | assert!(!key.is_empty()); 42 | assert!(!value.is_empty()); 43 | assert!(cursor.error().is_none()); 44 | count += 1; 45 | } 46 | 47 | assert_eq!(count, 500); 48 | 49 | Ok(()) 50 | } 51 | 52 | fn cursor_next_buf(mut database: Database) -> Result<(), Error> { 53 | for num in 0..500 { 54 | let key = format!("{:08x}", num); 55 | let value = format!("hello world {}", num); 56 | 57 | database.put(key, value)?; 58 | } 59 | 60 | let mut cursor = database.cursor()?; 61 | let mut count = 0; 62 | let mut key = Vec::new(); 63 | let mut value = Vec::new(); 64 | 65 | while cursor.next_buf(&mut key, &mut value)? { 66 | let expected_key = format!("{:08x}", count); 67 | let expected_value = format!("hello world {}", count); 68 | 69 | assert_eq!(key, expected_key.as_bytes()); 70 | assert_eq!(value, expected_value.as_bytes()); 71 | 72 | count += 1; 73 | } 74 | 75 | assert_eq!(count, 500); 76 | 77 | Ok(()) 78 | } 79 | 80 | fn cursor_removed_items(mut database: Database) -> Result<(), Error> { 81 | for num in 0..10000 { 82 | let key = format!("{:08x}", num); 83 | let value = format!("hello world {}", num); 84 | 85 | database.put(key, value)?; 86 | } 87 | 88 | for num in 0..10000 { 89 | let key = format!("{:08x}", num); 90 | database.remove(key)?; 91 | } 92 | 93 | let cursor = database.cursor()?; 94 | 95 | assert_eq!(cursor.count(), 0); 96 | 97 | Ok(()) 98 | } 99 | 100 | fn cursor_range(mut database: Database) -> Result<(), Error> { 101 | database.put("key:100", "hello world 100")?; 102 | database.put("key:200", "hello world 200")?; 103 | database.put("key:300", "hello world 300")?; 104 | database.put("key:400", "hello world 400")?; 105 | database.put("key:500", "hello world 500")?; 106 | database.put("key:600", "hello world 600")?; 107 | database.put("key:700", "hello world 700")?; 108 | database.put("key:800", "hello world 800")?; 109 | 110 | let cursor = database.cursor_range("key:250".."key:650")?; 111 | let keys: Vec = cursor 112 | .map(|(key, _value)| String::from_utf8(key).unwrap()) 113 | .collect(); 114 | 115 | assert_eq!( 116 | keys, 117 | vec![ 118 | "key:300".to_string(), 119 | "key:400".to_string(), 120 | "key:500".to_string(), 121 | "key:600".to_string() 122 | ] 123 | ); 124 | 125 | let cursor = database.cursor_range("key:100"..="key:200")?; 126 | let keys: Vec = cursor 127 | .map(|(key, _value)| String::from_utf8(key).unwrap()) 128 | .collect(); 129 | assert_eq!(keys, vec!["key:100".to_string(), "key:200".to_string(),]); 130 | 131 | let cursor = database.cursor_range(.."key:200")?; 132 | let keys: Vec = cursor 133 | .map(|(key, _value)| String::from_utf8(key).unwrap()) 134 | .collect(); 135 | assert_eq!(keys, vec!["key:100".to_string(),]); 136 | 137 | let cursor = database.cursor_range("key:750"..)?; 138 | let keys: Vec = cursor 139 | .map(|(key, _value)| String::from_utf8(key).unwrap()) 140 | .collect(); 141 | assert_eq!(keys, vec!["key:800".to_string(),]); 142 | 143 | Ok(()) 144 | } 145 | 146 | matrix_test!(cursor_sequential); 147 | matrix_test!(cursor_iter_manual); 148 | matrix_test!(cursor_next_buf); 149 | matrix_test!(cursor_range); 150 | matrix_test!(cursor_removed_items); 151 | -------------------------------------------------------------------------------- /src/library/tests/export.rs: -------------------------------------------------------------------------------- 1 | use std::io::BufReader; 2 | 3 | use grebedb::{Database, Options}; 4 | 5 | #[test] 6 | fn test_export() { 7 | let mut database = Database::open_memory(Options::default()).unwrap(); 8 | 9 | database.put("key1", "value1").unwrap(); 10 | database.put("key2", "value2").unwrap(); 11 | database.put("key3", "value3").unwrap(); 12 | 13 | let mut file = Vec::new(); 14 | 15 | grebedb::export::export(&mut database, &mut file, |_| {}).unwrap(); 16 | 17 | let mut database = Database::open_memory(Options::default()).unwrap(); 18 | 19 | grebedb::export::import( 20 | &mut database, 21 | &mut BufReader::new(std::io::Cursor::new(file)), 22 | |_| {}, 23 | ) 24 | .unwrap(); 25 | 26 | assert_eq!(database.get("key1").unwrap(), Some(b"value1".to_vec())); 27 | assert_eq!(database.get("key2").unwrap(), Some(b"value2".to_vec())); 28 | assert_eq!(database.get("key3").unwrap(), Some(b"value3".to_vec())); 29 | } 30 | -------------------------------------------------------------------------------- /src/library/tests/flush.rs: -------------------------------------------------------------------------------- 1 | mod common; 2 | 3 | use grebedb::Database; 4 | 5 | fn put_one(mut db: Database) -> anyhow::Result<()> { 6 | db.put("my key", "hello world")?; 7 | db.flush()?; 8 | 9 | Ok(()) 10 | } 11 | 12 | fn put_many(mut db: Database) -> anyhow::Result<()> { 13 | for num in 0..10000 { 14 | let key = format!("{:08x}", num); 15 | let value = format!("hello world {}", num); 16 | 17 | db.put(key, value)?; 18 | } 19 | 20 | db.flush()?; 21 | 22 | Ok(()) 23 | } 24 | 25 | matrix_test!(put_one); 26 | matrix_test!(put_many); 27 | -------------------------------------------------------------------------------- /src/library/tests/get_put_remove.rs: -------------------------------------------------------------------------------- 1 | mod common; 2 | 3 | use grebedb::{Database, Error}; 4 | 5 | fn simple_get_put_remove(mut database: Database) -> Result<(), Error> { 6 | database.put("key1", "hello")?; 7 | database.put("key2", "world")?; 8 | 9 | assert!(database.contains_key("key1")?); 10 | assert!(database.contains_key("key2")?); 11 | assert!(!database.contains_key("key3")?); 12 | 13 | assert_eq!(database.get("key1")?, Some("hello".into())); 14 | assert_eq!(database.get("key2")?, Some("world".into())); 15 | assert_eq!(database.get("key3")?, None); 16 | 17 | database.remove("key1")?; 18 | database.remove("key2")?; 19 | database.remove("key3")?; 20 | 21 | assert!(!database.contains_key("key1")?); 22 | assert!(!database.contains_key("key2")?); 23 | assert!(!database.contains_key("key3")?); 24 | 25 | Ok(()) 26 | } 27 | 28 | fn sequential_numbers(mut database: Database) -> Result<(), Error> { 29 | let mut buffer = Vec::new(); 30 | 31 | for num in 0..10000 { 32 | let key = format!("{:08x}", num); 33 | let value = format!("hello world {}", num); 34 | 35 | assert!(!database.contains_key(&key)?); 36 | database.put(key.clone(), value.clone())?; 37 | assert!(database.contains_key(&key)?); 38 | database.get_buf(&key, &mut buffer)?; 39 | assert_eq!(&buffer, value.as_bytes()); 40 | } 41 | 42 | for num in 0..10000 { 43 | let key = format!("{:08x}", num); 44 | 45 | database.remove(&key)?; 46 | assert!(!database.contains_key(&key)?); 47 | } 48 | 49 | Ok(()) 50 | } 51 | 52 | matrix_test!(simple_get_put_remove); 53 | matrix_test!(sequential_numbers); 54 | -------------------------------------------------------------------------------- /src/library/tests/metadata.rs: -------------------------------------------------------------------------------- 1 | use grebedb::{Database, Options}; 2 | use indexmap::IndexSet; 3 | 4 | #[test] 5 | fn test_metadata() { 6 | let options = Options::default(); 7 | let mut db = Database::open_memory(options).unwrap(); 8 | 9 | let mut keys = IndexSet::new(); 10 | 11 | for num in 0..500 { 12 | keys.insert(num); 13 | let key = format!("{:08x}", num); 14 | db.put(key, "hello world!").unwrap(); 15 | } 16 | 17 | assert_eq!(db.metadata().key_value_count(), 500); 18 | } 19 | -------------------------------------------------------------------------------- /src/library/tests/options.rs: -------------------------------------------------------------------------------- 1 | mod common; 2 | 3 | use grebedb::{ 4 | vfs::{MemoryVfs, ReadOnlyVfs}, 5 | CompressionLevel, Database, OpenMode, Options, SyncOption, 6 | }; 7 | 8 | #[test] 9 | fn test_read_only() -> anyhow::Result<()> { 10 | let memory_vfs = MemoryVfs::default(); 11 | let options = Options { 12 | keys_per_node: 128, 13 | page_cache_size: 4, 14 | ..Default::default() 15 | }; 16 | let mut db = Database::open(Box::new(memory_vfs.clone()), options)?; 17 | 18 | for num in 0..1000 { 19 | let key = format!("key:{:016x}", num); 20 | let value = format!("hello world {}", num); 21 | db.put(key, value)?; 22 | } 23 | 24 | db.flush()?; 25 | drop(db); 26 | 27 | let options = Options { 28 | open_mode: OpenMode::ReadOnly, 29 | keys_per_node: 128, 30 | page_cache_size: 4, 31 | ..Default::default() 32 | }; 33 | let mut db = Database::open(Box::new(memory_vfs.clone()), options.clone())?; 34 | 35 | for num in 0..1000 { 36 | let key = format!("key:{:016x}", num); 37 | db.get(key)?; 38 | } 39 | 40 | let mut db = Database::open(Box::new(ReadOnlyVfs::new(Box::new(memory_vfs))), options)?; 41 | 42 | for num in 0..1000 { 43 | let key = format!("key:{:016x}", num); 44 | db.get(key)?; 45 | } 46 | 47 | Ok(()) 48 | } 49 | 50 | #[test] 51 | fn test_read_only_no_clobber_not_a_db() -> anyhow::Result<()> { 52 | let dir = common::make_tempdir(); 53 | let options = Options { 54 | open_mode: OpenMode::ReadOnly, 55 | ..Default::default() 56 | }; 57 | 58 | let result = Database::open_path(dir.path(), options); 59 | assert!(result.is_err()); 60 | 61 | let contents = std::fs::read_dir(dir.path())?.count(); 62 | assert_eq!(contents, 0); 63 | 64 | Ok(()) 65 | } 66 | 67 | #[test] 68 | fn test_create_only() -> anyhow::Result<()> { 69 | let memory_vfs = MemoryVfs::default(); 70 | let options = Options::default(); 71 | let mut db = Database::open(Box::new(memory_vfs.clone()), options)?; 72 | 73 | for num in 0..1000 { 74 | let key = format!("key:{:016x}", num); 75 | let value = format!("hello world {}", num); 76 | db.put(key, value)?; 77 | } 78 | 79 | db.flush()?; 80 | drop(db); 81 | 82 | let options = Options { 83 | open_mode: OpenMode::CreateOnly, 84 | ..Default::default() 85 | }; 86 | 87 | let result = Database::open(Box::new(memory_vfs), options); 88 | 89 | assert!(result.is_err()); 90 | 91 | Ok(()) 92 | } 93 | 94 | #[test] 95 | fn test_load_only() { 96 | let options = Options { 97 | open_mode: OpenMode::LoadOnly, 98 | ..Default::default() 99 | }; 100 | 101 | let result = Database::open_memory(options); 102 | 103 | assert!(result.is_err()); 104 | } 105 | 106 | #[test] 107 | fn test_load_only_no_clobber_not_a_db() -> anyhow::Result<()> { 108 | let dir = common::make_tempdir(); 109 | let options = Options { 110 | open_mode: OpenMode::LoadOnly, 111 | ..Default::default() 112 | }; 113 | 114 | let result = Database::open_path(dir.path(), options); 115 | assert!(result.is_err()); 116 | 117 | let contents = std::fs::read_dir(dir.path())?.count(); 118 | assert_eq!(contents, 0); 119 | 120 | Ok(()) 121 | } 122 | 123 | #[test] 124 | fn test_no_compression() -> anyhow::Result<()> { 125 | let vfs = MemoryVfs::default(); 126 | let options = Options { 127 | compression_level: CompressionLevel::None, 128 | ..Default::default() 129 | }; 130 | let mut db = Database::open(Box::new(vfs.clone()), options.clone())?; 131 | 132 | db.put("my key", "hello world")?; 133 | db.flush()?; 134 | 135 | let mut db = Database::open(Box::new(vfs), options)?; 136 | 137 | assert!(db.get("my key")?.is_some()); 138 | 139 | Ok(()) 140 | } 141 | 142 | #[test] 143 | fn test_no_file_locking() -> anyhow::Result<()> { 144 | let dir = common::make_tempdir(); 145 | let options = Options { 146 | file_locking: false, 147 | ..Default::default() 148 | }; 149 | let mut db = Database::open_path(dir.path(), options)?; 150 | 151 | db.put("my key", "hello world")?; 152 | db.flush()?; 153 | 154 | Ok(()) 155 | } 156 | 157 | #[test] 158 | fn test_no_file_sync() -> anyhow::Result<()> { 159 | let dir = common::make_tempdir(); 160 | let options = Options { 161 | file_sync: SyncOption::None, 162 | keys_per_node: 128, 163 | page_cache_size: 4, 164 | ..Default::default() 165 | }; 166 | let mut db = Database::open_path(dir.path(), options)?; 167 | 168 | for num in 0..1000 { 169 | db.put(format!("my key {}", num), "hello world")?; 170 | } 171 | db.flush()?; 172 | 173 | for num in 0..1000 { 174 | db.get(format!("my key {}", num))?; 175 | } 176 | 177 | Ok(()) 178 | } 179 | -------------------------------------------------------------------------------- /src/library/tests/random_operations.rs: -------------------------------------------------------------------------------- 1 | mod common; 2 | 3 | use std::collections::BTreeMap; 4 | 5 | use grebedb::{Database, Error}; 6 | use indexmap::IndexSet; 7 | use rand::{distributions::Uniform, prelude::*, SeedableRng}; 8 | use rand_xorshift::XorShiftRng; 9 | 10 | enum OperationChoice { 11 | Get, 12 | Put, 13 | Remove, 14 | Flush, 15 | } 16 | 17 | enum Operation { 18 | Get(Vec), 19 | Put(Vec, Vec), 20 | Remove(Vec), 21 | Flush, 22 | } 23 | 24 | struct OperationGenerator { 25 | rng: XorShiftRng, 26 | key_range: Uniform, 27 | existing_keys: IndexSet, 28 | choices: Vec, 29 | } 30 | 31 | impl OperationGenerator { 32 | fn new(max_keys: usize) -> Self { 33 | let mut choices = Vec::new(); 34 | 35 | for _ in 0..200 { 36 | choices.push(OperationChoice::Get); 37 | } 38 | for _ in 0..700 { 39 | choices.push(OperationChoice::Put); 40 | } 41 | for _ in 0..100 { 42 | choices.push(OperationChoice::Remove); 43 | } 44 | 45 | choices.push(OperationChoice::Flush); 46 | 47 | Self { 48 | rng: XorShiftRng::seed_from_u64(1), 49 | key_range: Uniform::from(0..max_keys as u64), 50 | existing_keys: IndexSet::new(), 51 | choices, 52 | } 53 | } 54 | 55 | pub fn get(&mut self) -> Operation { 56 | let operation_seed: u64 = self.rng.gen(); 57 | let mut operation_rng = XorShiftRng::seed_from_u64(operation_seed); 58 | let choice = self.choices.choose(&mut operation_rng).unwrap(); 59 | 60 | match choice { 61 | OperationChoice::Get => { 62 | if !self.existing_keys.is_empty() && self.rng.gen_bool(0.6) { 63 | let key = self.get_existing_key(&mut operation_rng); 64 | let key_vec = Self::derive_key_bytes(key); 65 | Operation::Get(key_vec) 66 | } else { 67 | let key = self.get_nonexisting_key(&mut operation_rng); 68 | let key_vec = Self::derive_key_bytes(key); 69 | Operation::Get(key_vec) 70 | } 71 | } 72 | OperationChoice::Put => { 73 | if !self.existing_keys.is_empty() && self.rng.gen_bool(0.6) { 74 | let key = self.get_existing_key(&mut operation_rng); 75 | let key_vec = Self::derive_key_bytes(key); 76 | let value_vec = Self::derive_value_bytes(key, operation_seed); 77 | Operation::Put(key_vec, value_vec) 78 | } else { 79 | let key = self.generate_new_key(&mut operation_rng); 80 | let key_vec = Self::derive_key_bytes(key); 81 | let value_vec = Self::derive_value_bytes(key, operation_seed); 82 | Operation::Put(key_vec, value_vec) 83 | } 84 | } 85 | OperationChoice::Remove => { 86 | if !self.existing_keys.is_empty() && self.rng.gen_bool(0.8) { 87 | let index = self.get_existing_index(&mut operation_rng); 88 | let key = self.existing_keys.swap_remove_index(index).unwrap(); 89 | let key_vec = Self::derive_key_bytes(key); 90 | Operation::Remove(key_vec) 91 | } else { 92 | let key = self.get_nonexisting_key(&mut operation_rng); 93 | let key_vec = Self::derive_key_bytes(key); 94 | Operation::Remove(key_vec) 95 | } 96 | } 97 | OperationChoice::Flush => Operation::Flush, 98 | } 99 | } 100 | 101 | fn get_existing_index(&mut self, rng: &mut impl Rng) -> usize { 102 | rng.gen_range(0..self.existing_keys.len()) 103 | } 104 | 105 | fn get_existing_key(&mut self, rng: &mut impl Rng) -> u64 { 106 | let index = self.get_existing_index(rng); 107 | *self.existing_keys.get_index(index).unwrap() 108 | } 109 | 110 | fn get_nonexisting_key(&mut self, rng: &mut impl Rng) -> u64 { 111 | loop { 112 | let key = self.key_range.sample(rng); 113 | 114 | if !self.existing_keys.contains(&key) { 115 | return key; 116 | } 117 | } 118 | } 119 | 120 | fn generate_new_key(&mut self, rng: &mut impl Rng) -> u64 { 121 | let key = self.key_range.sample(rng); 122 | self.existing_keys.insert(key); 123 | key 124 | } 125 | 126 | fn derive_key_bytes(key: u64) -> Vec { 127 | format!("my key {:016x}", key).into_bytes() 128 | } 129 | 130 | fn derive_value_bytes(key: u64, seed: u64) -> Vec { 131 | let mut value_rng = XorShiftRng::seed_from_u64(key ^ seed); 132 | 133 | let size = value_rng.gen_range(1..=4096); 134 | let mut value = vec![0; size]; 135 | value_rng.fill_bytes(&mut value); 136 | 137 | value 138 | } 139 | } 140 | 141 | fn rand_operation(mut database: Database, rounds: usize) -> Result<(), Error> { 142 | let mut generator = OperationGenerator::new((rounds / 4).max(10)); 143 | let mut std_map = BTreeMap::, Vec>::new(); 144 | let mut value_buffer = Vec::new(); 145 | 146 | for num in 0..rounds { 147 | match generator.get() { 148 | Operation::Get(key) => { 149 | let has_key = database.get_buf(&key, &mut value_buffer)?; 150 | 151 | match std_map.get(&key) { 152 | Some(value) => { 153 | assert!(has_key); 154 | assert_eq!(&value_buffer, value); 155 | } 156 | None => { 157 | assert!(!has_key); 158 | } 159 | } 160 | } 161 | Operation::Put(key, value) => { 162 | database.put(key.clone(), value.clone())?; 163 | assert!(database.contains_key(&key)?); 164 | std_map.insert(key, value); 165 | } 166 | Operation::Remove(key) => { 167 | database.remove(key.clone())?; 168 | assert!(!database.contains_key(&key)?); 169 | std_map.remove(&key); 170 | } 171 | Operation::Flush => database.flush()?, 172 | } 173 | 174 | if num % 1000 == 0 { 175 | database.verify(|_, _| {})?; 176 | } 177 | } 178 | 179 | database.verify(|_, _| {})?; 180 | 181 | println!( 182 | "current len={}, expected len={}", 183 | database.cursor()?.count(), 184 | std_map.len() 185 | ); 186 | 187 | let mut cursor = database.cursor()?; 188 | let mut std_iter = std_map.iter(); 189 | 190 | loop { 191 | let current = cursor.next(); 192 | let expected = std_iter.next(); 193 | 194 | if current.is_none() && expected.is_none() { 195 | break; 196 | } else { 197 | let (key, value) = current.unwrap(); 198 | let (expected_key, expected_value) = expected.unwrap(); 199 | 200 | assert_eq!(&key, expected_key); 201 | assert_eq!(&value, expected_value); 202 | } 203 | } 204 | 205 | Ok(()) 206 | } 207 | 208 | fn rand_operation_10000(database: Database) -> Result<(), Error> { 209 | rand_operation(database, 10000) 210 | } 211 | 212 | fn rand_operation_100000(database: Database) -> Result<(), Error> { 213 | rand_operation(database, 100000) 214 | } 215 | 216 | #[cfg(debug_assertions)] 217 | mod a { 218 | use grebedb::Options; 219 | 220 | use super::*; 221 | 222 | #[test] 223 | fn rand_operation_10000_fast() { 224 | let database = Database::open_memory(Options::default()).unwrap(); 225 | rand_operation(database, 10000).unwrap(); 226 | } 227 | 228 | matrix_test_ignore!(rand_operation_10000); 229 | matrix_test_ignore!(rand_operation_100000); 230 | } 231 | #[cfg(not(debug_assertions))] 232 | mod a { 233 | use super::*; 234 | 235 | matrix_test!(rand_operation_10000); 236 | matrix_test_ignore!(rand_operation_100000); 237 | } 238 | -------------------------------------------------------------------------------- /src/library/tests/remove.rs: -------------------------------------------------------------------------------- 1 | mod common; 2 | 3 | use grebedb::{Database, Error}; 4 | use indexmap::IndexSet; 5 | use rand::{Rng, SeedableRng}; 6 | use rand_xorshift::XorShiftRng; 7 | 8 | fn fill_and_random_remove(mut db: Database) -> Result<(), Error> { 9 | let mut keys = IndexSet::new(); 10 | 11 | for num in 0..2000 { 12 | keys.insert(num); 13 | let key = format!("{:08x}", num); 14 | db.put(key, "hello world!")?; 15 | } 16 | 17 | let mut rng = XorShiftRng::seed_from_u64(1); 18 | let mut count = 0; 19 | 20 | while !keys.is_empty() { 21 | let index = rng.gen_range(0..keys.len()); 22 | let key = keys.swap_remove_index(index).unwrap(); 23 | let key = format!("{:08x}", key); 24 | 25 | assert!(db.contains_key(&key)?); 26 | 27 | db.remove(&key)?; 28 | 29 | if count % 100 == 0 { 30 | db.verify(|_, _| {})?; 31 | } 32 | 33 | assert!(!db.contains_key(&key)?); 34 | 35 | count += 1; 36 | } 37 | 38 | Ok(()) 39 | } 40 | 41 | matrix_test!(fill_and_random_remove); 42 | -------------------------------------------------------------------------------- /src/library/tests/sync.rs: -------------------------------------------------------------------------------- 1 | use grebedb::{Database, Options}; 2 | 3 | #[test] 4 | fn test_send() { 5 | fn assert_send() {} 6 | assert_send::(); 7 | } 8 | 9 | #[test] 10 | fn test_sync() { 11 | fn assert_sync() {} 12 | assert_sync::(); 13 | } 14 | 15 | #[test] 16 | fn test_send_thread() -> anyhow::Result<()> { 17 | let mut database = Database::open_memory(Options::default())?; 18 | 19 | database.put("k", "v")?; 20 | 21 | let handle = std::thread::spawn(move || { 22 | database.get("k").unwrap(); 23 | }); 24 | 25 | handle.join().unwrap(); 26 | 27 | Ok(()) 28 | } 29 | -------------------------------------------------------------------------------- /src/library/tests/vfs_crash.rs: -------------------------------------------------------------------------------- 1 | mod common; 2 | 3 | use std::sync::atomic::Ordering; 4 | 5 | use common::CrashingVfs; 6 | use grebedb::{Database, Options}; 7 | 8 | #[test] 9 | fn test_crash_before_metadata_commit() { 10 | let vfs = CrashingVfs::new(); 11 | let options = Options { 12 | keys_per_node: 128, 13 | page_cache_size: 4, 14 | automatic_flush: false, 15 | ..Default::default() 16 | }; 17 | let mut database = Database::open(Box::new(vfs.clone()), options).unwrap(); 18 | 19 | for num in 0..2000 { 20 | database 21 | .put(format!("key:{:04x}", num), "hello world") 22 | .unwrap(); 23 | 24 | if num == 1000 { 25 | database.flush().unwrap(); 26 | } 27 | } 28 | 29 | for num in 0..2000 { 30 | database.get(format!("key:{:04x}", num)).unwrap(); 31 | } 32 | 33 | database.put("key:0000", "new value").unwrap(); // a key near start 34 | database.put("key:07A0", "new value").unwrap(); // a key near end 35 | 36 | // New copy-on-write pages should be written successfully, 37 | // the metadata should fail to be renamed 38 | vfs.metadata_rename_crash.store(true, Ordering::Relaxed); 39 | database.flush().unwrap_err(); 40 | 41 | // Expect old pages with revision flag 0 to be read, and flag 1 to be ignored: 42 | let mut database = Database::open(Box::new(vfs), Options::default()).unwrap(); 43 | 44 | assert_eq!( 45 | database 46 | .get("key:0000") 47 | .unwrap() 48 | .map(|item| String::from_utf8(item).unwrap()), 49 | Some("hello world".to_string()) 50 | ); 51 | assert_eq!(database.get("key:07A0").unwrap(), None); 52 | } 53 | 54 | #[test] 55 | fn test_crash_after_metadata_commit() { 56 | let vfs = CrashingVfs::new(); 57 | let options = Options { 58 | keys_per_node: 128, 59 | page_cache_size: 4, 60 | automatic_flush: false, 61 | ..Default::default() 62 | }; 63 | let mut database = Database::open(Box::new(vfs.clone()), options).unwrap(); 64 | 65 | for num in 0..2000 { 66 | database 67 | .put(format!("key:{:04x}", num), "hello world") 68 | .unwrap(); 69 | 70 | if num == 1000 { 71 | database.flush().unwrap(); 72 | } 73 | } 74 | 75 | for num in 0..2000 { 76 | database.get(format!("key:{:04x}", num)).unwrap(); 77 | } 78 | 79 | database.put("key:0000", "new value").unwrap(); // a key near start 80 | database.put("key:07A0", "new value").unwrap(); // a key near end 81 | 82 | // New copy-on-write pages should be written successfully, 83 | // the metadata should to be renamed successfully, 84 | // but subsequent copy-on-write pages should fail to rename from revision flag 1 to 0 85 | vfs.after_metadata_rename_crash 86 | .store(true, Ordering::Relaxed); 87 | 88 | database.flush().unwrap_err(); 89 | 90 | vfs.after_metadata_rename_crash 91 | .store(false, Ordering::Relaxed); 92 | 93 | // Expect read new pages in either copy-on-write revision flag 0 or 1: 94 | let mut database = Database::open(Box::new(vfs), Options::default()).unwrap(); 95 | 96 | assert_eq!( 97 | database 98 | .get("key:0000") 99 | .unwrap() 100 | .map(|item| String::from_utf8(item).unwrap()), 101 | Some("new value".to_string()) 102 | ); 103 | assert_eq!( 104 | database 105 | .get("key:07A0") 106 | .unwrap() 107 | .map(|item| String::from_utf8(item).unwrap()), 108 | Some("new value".to_string()) 109 | ); 110 | } 111 | -------------------------------------------------------------------------------- /src/tool/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # grebedb-tool change log 2 | 3 | (This log only contains changes for grebedb-tool. Changes for the library are located in its own file.) 4 | 5 | ## 1.0.0 (2021-06-04) 6 | 7 | * Inspect command's interactive session now supports Unix shell quoting and escapes. 8 | * Updated dependency to grebedb 1.0 9 | 10 | ## 0.3.0 (2021-04-20) 11 | 12 | * Added `--zstd` flag to `import` and `export` commands. 13 | * Added `inspect` command. 14 | * Updated dependency to grebedb 0.3 15 | 16 | ## 0.2.0 (2021-04-07) 17 | 18 | * Added `verify` command. 19 | * Updated dependency to grebedb 0.2 20 | 21 | ## 0.1.0 (2021-03-28) 22 | 23 | Initial version. 24 | -------------------------------------------------------------------------------- /src/tool/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "grebedb-tool" 3 | version = "1.0.0" 4 | authors = ["Christopher Foo "] 5 | edition = "2021" 6 | description = "Command line tool for operating on GrebeDB databases" 7 | repository = "https://github.com/chfoo/grebedb/" 8 | license = "MPL-2.0" 9 | keywords = ["database", "key-value-store"] 10 | categories = ["command-line-utilities"] 11 | 12 | [features] 13 | default = ["compression", "file_locking", "system"] 14 | compression = ["grebedb/compression", "zstd"] 15 | file_locking = ["grebedb/file_locking"] 16 | system = ["grebedb/system"] 17 | 18 | [dependencies.grebedb] 19 | version = "1.0" 20 | path = "../library" 21 | default-features = false 22 | 23 | [dependencies] 24 | anyhow = "1.0" 25 | bson = "2.0" 26 | clap = "2.33" 27 | crc32c = "0.6" 28 | data-encoding = "2.3" 29 | percent-encoding = "2.1" 30 | rmpv = "1.0" 31 | rustyline = "9.0" 32 | serde = { version = "1.0", features = ["derive"] } 33 | serde_json = "1.0" 34 | shell-words = "1.0" 35 | zstd = { version = "0.9", optional = true } 36 | -------------------------------------------------------------------------------- /src/tool/LICENSE: -------------------------------------------------------------------------------- 1 | ../../LICENSE -------------------------------------------------------------------------------- /src/tool/README.md: -------------------------------------------------------------------------------- 1 | # grebedb-tool 2 | 3 | A command-line program to provide basic manipulation (such as import, export) and debugging on [GrebeDB](https://github.com/chfoo/grebedb) databases. 4 | 5 | The tool is currently offered only in source code form, so you will need to install Rust first. Once Rust is installed, run `cargo install grebedb-tool` to install the tool to `$HOME/.cargo/bin`, then run `grebedb-tool --help` to show program options. To update the program, run the install command again. To remove the program, run `cargo uninstall grebedb-tool`. 6 | 7 | ## Quick start 8 | 9 | Example commands for running the tool assuming a Unix-like shell. 10 | 11 | ### Import and export 12 | 13 | The import and export commands can be used to create backups of the database contents to a [JSON sequence file](https://tools.ietf.org/html/rfc7464). 14 | 15 | The commands can import/export directly to a given filename or through standard input/output. 16 | 17 | Export a database to a (non-existing) file: 18 | 19 | grebedb-tool export path/to/database/ database.json-seq 20 | 21 | or with compression: 22 | 23 | grebedb-tool export path/to/database/ database.json-seq.zst --zstd 24 | 25 | To import a database (into an empty directory) from a file: 26 | 27 | grebedb-tool import path/to/database/ database.json-seq 28 | 29 | or with compression: 30 | 31 | grebedb-tool import path/to/database/ database.json-seq.zst --zstd 32 | 33 | ### Verify 34 | 35 | The verify command checks that the database has not been corrupted. 36 | 37 | grebedb-tool verify path/to/database/ --verbose 38 | 39 | ### Inspect 40 | 41 | The inspect command launches an interactive session for browsing and editing the database contents. 42 | 43 | grebedb-tool inspect path/to/database/ 44 | 45 | Inputting `help` will show all available commands. Inputting `help` and then the name of the command will show all options for a given command. 46 | 47 | Note that because the format of the contents depends on the application, the inspect command is not intended as a user-friendly way of directly editing application data. 48 | -------------------------------------------------------------------------------- /src/tool/src/export.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fs::{File, OpenOptions}, 3 | io::{BufReader, BufWriter, Read, Write}, 4 | path::Path, 5 | }; 6 | 7 | use grebedb::{Database, OpenMode, Options}; 8 | 9 | pub fn dump( 10 | database_path: &Path, 11 | output_path: &Path, 12 | compression: Option, 13 | ) -> anyhow::Result<()> { 14 | let options = Options { 15 | open_mode: OpenMode::ReadOnly, 16 | ..Default::default() 17 | }; 18 | let mut database = Database::open_path(database_path, options)?; 19 | 20 | // TODO: this needs refactoring 21 | if output_path.as_os_str() != "-" { 22 | let mut file = OpenOptions::new() 23 | .write(true) 24 | .create_new(true) 25 | .open(output_path)?; 26 | 27 | if let Some(compression) = compression { 28 | #[cfg(feature = "zstd")] 29 | { 30 | let mut file = zstd::Encoder::new(&mut file, compression)?; 31 | grebedb::export::export(&mut database, &mut file, |_| {})?; 32 | file.finish()?; 33 | } 34 | #[cfg(not(feature = "zstd"))] 35 | { 36 | let _ = compression; 37 | return Err(anyhow::anyhow!("Compression feature not enabled")); 38 | } 39 | } else { 40 | grebedb::export::export(&mut database, &mut file, |_| {})?; 41 | } 42 | 43 | file.flush()?; 44 | file.sync_all()?; 45 | } else { 46 | let mut file = BufWriter::new(std::io::stdout()); 47 | 48 | if let Some(compression) = compression { 49 | #[cfg(feature = "zstd")] 50 | { 51 | let mut file = zstd::Encoder::new(&mut file, compression)?; 52 | grebedb::export::export(&mut database, &mut file, |_| {})?; 53 | file.finish()?; 54 | } 55 | #[cfg(not(feature = "zstd"))] 56 | { 57 | let _ = compression; 58 | return Err(anyhow::anyhow!("Compression feature not enabled")); 59 | } 60 | } else { 61 | grebedb::export::export(&mut database, &mut file, |_| {})?; 62 | } 63 | file.flush()?; 64 | } 65 | 66 | Ok(()) 67 | } 68 | 69 | pub fn load(database_path: &Path, input_path: &Path, compression: bool) -> anyhow::Result<()> { 70 | let options = Options { 71 | open_mode: OpenMode::CreateOnly, 72 | ..Default::default() 73 | }; 74 | let mut database = Database::open_path(database_path, options)?; 75 | 76 | let mut file: BufReader> = if input_path.as_os_str() != "-" { 77 | BufReader::new(Box::new(File::open(input_path)?)) 78 | } else { 79 | BufReader::new(Box::new(std::io::stdin())) 80 | }; 81 | 82 | if compression { 83 | #[cfg(feature = "zstd")] 84 | { 85 | let mut file = BufReader::new(zstd::Decoder::new(file)?); 86 | grebedb::export::import(&mut database, &mut file, |_| {})? 87 | } 88 | #[cfg(not(feature = "zstd"))] 89 | { 90 | return Err(anyhow::anyhow!("Compression feature not enabled")); 91 | } 92 | } else { 93 | grebedb::export::import(&mut database, &mut file, |_| {})? 94 | } 95 | 96 | database.flush()?; 97 | 98 | Ok(()) 99 | } 100 | -------------------------------------------------------------------------------- /src/tool/src/main.rs: -------------------------------------------------------------------------------- 1 | mod export; 2 | mod repl; 3 | mod verify; 4 | 5 | use std::path::Path; 6 | 7 | use clap::{crate_version, App, AppSettings, Arg, ArgMatches, SubCommand}; 8 | use grebedb::{Database, OpenMode, Options}; 9 | 10 | fn main() -> anyhow::Result<()> { 11 | let db_path_arg = Arg::with_name("database_path") 12 | .value_name("DATABASE") 13 | .help("Path to the directory containing the database.") 14 | .required(true); 15 | 16 | let zstd_arg = Arg::with_name("zstd").long("zstd"); 17 | let compression_level_arg = Arg::with_name("compression_level") 18 | .long("compression-level") 19 | .short("l") 20 | .help("Compression level where 1 is worst (fastest) and 3 is best (slowest).") 21 | .default_value("2") 22 | .possible_values(&["1", "2", "3", "4", "5"]); 23 | 24 | let app = App::new("GrebeDB database manipulation tool") 25 | .version(crate_version!()) 26 | .setting(AppSettings::SubcommandRequiredElseHelp) 27 | .subcommand( 28 | SubCommand::with_name("export") 29 | .about("Export the contents of the database to a JSON text sequence (RFC 7464) file.") 30 | .arg(db_path_arg.clone()) 31 | .arg( 32 | Arg::with_name("json_path") 33 | .value_name("DESTINATION") 34 | .default_value("-") 35 | .help("Filename of the exported file."), 36 | ) 37 | .arg(zstd_arg.clone().help("Use Zstandard compression when writing to DESTINATION.")) 38 | .arg(compression_level_arg) 39 | ) 40 | .subcommand( 41 | SubCommand::with_name("import") 42 | .about("Import the contents from a JSON text sequence (RFC 7464) file into the database.") 43 | .arg(db_path_arg.clone()) 44 | .arg( 45 | Arg::with_name("json_path") 46 | .value_name("SOURCE") 47 | .default_value("-") 48 | .help("Filename of the source file."), 49 | ) 50 | .arg(zstd_arg.clone().help("Use Zstandard decompression when reading from SOURCE.")) 51 | ) 52 | .subcommand( 53 | SubCommand::with_name("verify") 54 | .about("Check the database for internal consistency and data integrity.") 55 | .arg(db_path_arg.clone()) 56 | .arg( 57 | Arg::with_name("write") 58 | .long("write") 59 | .short("w") 60 | .help("Open in read & write mode to allow cleanup operations."), 61 | ) 62 | .arg( 63 | Arg::with_name("verbose") 64 | .long("verbose") 65 | .short("v") 66 | .help("Print rough progress."), 67 | ) 68 | ) 69 | .subcommand( 70 | SubCommand::with_name("inspect") 71 | .about("Start a interactive session for browsing and editing the database contents.") 72 | .arg(db_path_arg.clone()) 73 | .arg( 74 | Arg::with_name("write") 75 | .long("write") 76 | .short("w") 77 | .help("Open in read & write mode."), 78 | ) 79 | .arg( 80 | Arg::with_name("batch") 81 | .long("batch") 82 | .short("b") 83 | .help("Enable batch mode for scripting.") 84 | .long_help("Enable batch mode for scripting.\n\n\ 85 | When enabled, any errors are assumed to be undesired and causes the \ 86 | program to exit. This can be useful for scripts to send commands \ 87 | using standard input.") 88 | ) 89 | ) 90 | .subcommand( 91 | SubCommand::with_name("debug_print_tree") 92 | .about("Print the database tree for debugging purposes.") 93 | .arg(db_path_arg.clone()) 94 | ) 95 | .subcommand( 96 | SubCommand::with_name("debug_print_page") 97 | .about("Print a database page for debugging purposes.") 98 | .arg( 99 | Arg::with_name("page_path") 100 | .value_name("PATH") 101 | .help("Path to the database page.") 102 | .required(true) 103 | ) 104 | ); 105 | 106 | let matches = app.get_matches(); 107 | 108 | match matches.subcommand() { 109 | ("export", Some(sub_m)) => crate::export::dump( 110 | sub_m.value_of_os("database_path").unwrap().as_ref(), 111 | sub_m.value_of_os("json_path").unwrap().as_ref(), 112 | parse_zstd_compression_args(sub_m), 113 | ), 114 | ("import", Some(sub_m)) => crate::export::load( 115 | sub_m.value_of_os("database_path").unwrap().as_ref(), 116 | sub_m.value_of_os("json_path").unwrap().as_ref(), 117 | sub_m.is_present("zstd"), 118 | ), 119 | ("verify", Some(sub_m)) => crate::verify::verify( 120 | sub_m.value_of_os("database_path").unwrap().as_ref(), 121 | sub_m.is_present("write"), 122 | sub_m.is_present("verbose"), 123 | ), 124 | ("inspect", Some(sub_m)) => crate::repl::inspect( 125 | sub_m.value_of_os("database_path").unwrap().as_ref(), 126 | sub_m.is_present("write"), 127 | sub_m.is_present("batch"), 128 | ), 129 | ("debug_print_tree", Some(sub_m)) => { 130 | debug_print_tree_command(sub_m.value_of_os("database_path").unwrap().as_ref()) 131 | } 132 | ("debug_print_page", Some(sub_m)) => { 133 | debug_print_page_command(sub_m.value_of_os("page_path").unwrap().as_ref()) 134 | } 135 | _ => { 136 | unreachable!(); 137 | } 138 | } 139 | } 140 | 141 | fn parse_zstd_compression_args(args: &ArgMatches) -> Option { 142 | if args.is_present("zstd") { 143 | let level = args.value_of("compression_level").unwrap(); 144 | let level: u8 = level.parse().unwrap(); 145 | 146 | match level { 147 | 1 => Some(1), 148 | 2 => Some(3), 149 | 3 => Some(9), 150 | 4 => Some(15), 151 | 5 => Some(19), 152 | _ => unreachable!(), 153 | } 154 | } else { 155 | None 156 | } 157 | } 158 | 159 | fn debug_print_tree_command(database_path: &Path) -> anyhow::Result<()> { 160 | let mut database = Database::open_path( 161 | database_path, 162 | Options { 163 | open_mode: OpenMode::ReadOnly, 164 | ..Default::default() 165 | }, 166 | )?; 167 | database.debug_print_tree()?; 168 | 169 | Ok(()) 170 | } 171 | 172 | fn debug_print_page_command(path: &Path) -> anyhow::Result<()> { 173 | grebedb::debug_print_page(path)?; 174 | 175 | Ok(()) 176 | } 177 | -------------------------------------------------------------------------------- /src/tool/src/repl/encoding.rs: -------------------------------------------------------------------------------- 1 | use std::convert::TryFrom; 2 | 3 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 4 | pub enum Encoding { 5 | Utf8, 6 | Percent, 7 | Hex, 8 | Base64, 9 | } 10 | 11 | impl From for &str { 12 | fn from(value: Encoding) -> Self { 13 | match value { 14 | Encoding::Utf8 => "utf8", 15 | Encoding::Percent => "percent", 16 | Encoding::Hex => "hex", 17 | Encoding::Base64 => "base64", 18 | } 19 | } 20 | } 21 | 22 | impl TryFrom<&str> for Encoding { 23 | type Error = anyhow::Error; 24 | 25 | fn try_from(value: &str) -> Result { 26 | match value { 27 | "utf8" => Ok(Encoding::Utf8), 28 | "percent" => Ok(Encoding::Percent), 29 | "hex" => Ok(Encoding::Hex), 30 | "base64" => Ok(Encoding::Base64), 31 | _ => Err(anyhow::anyhow!("Unknown encoding")), 32 | } 33 | } 34 | } 35 | 36 | impl Encoding { 37 | pub fn list() -> [&'static str; 4] { 38 | [ 39 | Encoding::Utf8.into(), 40 | Encoding::Percent.into(), 41 | Encoding::Hex.into(), 42 | Encoding::Base64.into(), 43 | ] 44 | } 45 | } 46 | 47 | pub fn text_to_binary(value: &str, encoding: Encoding) -> anyhow::Result> { 48 | match encoding { 49 | Encoding::Utf8 => Ok(value.as_bytes().to_vec()), 50 | Encoding::Percent => { 51 | Ok(percent_encoding::percent_decode(value.as_bytes()).collect::>()) 52 | } 53 | Encoding::Hex => Ok(data_encoding::HEXUPPER_PERMISSIVE.decode(value.as_bytes())?), 54 | Encoding::Base64 => Ok(data_encoding::BASE64.decode(value.as_bytes())?), 55 | } 56 | } 57 | 58 | pub fn binary_to_text(value: &[u8], encoding: Encoding) -> String { 59 | match encoding { 60 | Encoding::Utf8 => String::from_utf8_lossy(value).to_string(), 61 | Encoding::Percent => { 62 | const SET: &percent_encoding::AsciiSet = &percent_encoding::CONTROLS.add(b' '); 63 | let result = percent_encoding::percent_encode(value, SET); 64 | result.to_string() 65 | } 66 | Encoding::Hex => data_encoding::HEXUPPER_PERMISSIVE.encode(value), 67 | Encoding::Base64 => data_encoding::BASE64.encode(value), 68 | } 69 | } 70 | 71 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 72 | pub enum DocumentFormat { 73 | Json, 74 | MessagePack, 75 | Bson, 76 | } 77 | 78 | impl From for &str { 79 | fn from(value: DocumentFormat) -> Self { 80 | match value { 81 | DocumentFormat::Json => "json", 82 | DocumentFormat::MessagePack => "msgpack", 83 | DocumentFormat::Bson => "bson", 84 | } 85 | } 86 | } 87 | 88 | impl TryFrom<&str> for DocumentFormat { 89 | type Error = anyhow::Error; 90 | 91 | fn try_from(value: &str) -> Result { 92 | match value { 93 | "json" => Ok(DocumentFormat::Json), 94 | "msgpack" => Ok(DocumentFormat::MessagePack), 95 | "bson" => Ok(DocumentFormat::Bson), 96 | _ => Err(anyhow::anyhow!("Unknown document format")), 97 | } 98 | } 99 | } 100 | 101 | impl DocumentFormat { 102 | pub fn list() -> [&'static str; 3] { 103 | [ 104 | DocumentFormat::Json.into(), 105 | DocumentFormat::MessagePack.into(), 106 | DocumentFormat::Bson.into(), 107 | ] 108 | } 109 | } 110 | 111 | pub fn binary_to_document(value: &[u8], format: DocumentFormat) -> anyhow::Result { 112 | match format { 113 | DocumentFormat::Json => { 114 | let doc: serde_json::Value = serde_json::from_slice(value)?; 115 | Ok(doc.to_string()) 116 | } 117 | DocumentFormat::MessagePack => { 118 | let doc = rmpv::decode::read_value_ref(&mut std::io::Cursor::new(value))?; 119 | Ok(doc.to_string()) 120 | } 121 | DocumentFormat::Bson => { 122 | let doc = bson::Document::from_reader(&mut std::io::Cursor::new(value))?; 123 | Ok(doc.to_string()) 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/tool/src/repl/mod.rs: -------------------------------------------------------------------------------- 1 | mod encoding; 2 | 3 | use std::convert::TryInto; 4 | use std::path::Path; 5 | 6 | use clap::{App, AppSettings, Arg, ArgMatches, SubCommand}; 7 | use grebedb::{Database, OpenMode, Options}; 8 | use rustyline::{error::ReadlineError, Editor}; 9 | 10 | use self::encoding::{DocumentFormat, Encoding}; 11 | 12 | pub fn inspect(database_path: &Path, write: bool, batch_mode: bool) -> anyhow::Result<()> { 13 | let options = Options { 14 | open_mode: if write { 15 | OpenMode::LoadOnly 16 | } else { 17 | OpenMode::ReadOnly 18 | }, 19 | automatic_flush: false, 20 | ..Default::default() 21 | }; 22 | 23 | let mut database = Database::open_path(database_path, options)?; 24 | 25 | let mut readline = Editor::<()>::new(); 26 | 27 | if !batch_mode { 28 | eprintln!("Welcome to the inspector. Type `help` and press enter for list of commands."); 29 | } 30 | 31 | loop { 32 | let line = readline.readline(">> "); 33 | 34 | match line { 35 | Ok(line) => { 36 | readline.add_history_entry(line.as_str()); 37 | 38 | match execute_command(&mut database, &line) { 39 | Ok(command_result) => match command_result { 40 | CommandResult::Continue => {} 41 | CommandResult::Exit => { 42 | break; 43 | } 44 | CommandResult::Error(error) => { 45 | if batch_mode { 46 | return Err(error); 47 | } else { 48 | eprintln!("{}", error); 49 | } 50 | } 51 | }, 52 | Err(error) => { 53 | if batch_mode { 54 | return Err(error); 55 | } else { 56 | eprintln!("Error: {}", error); 57 | } 58 | } 59 | } 60 | } 61 | Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => break, 62 | Err(error) => return Err(anyhow::Error::new(error)), 63 | } 64 | } 65 | 66 | eprintln!("Exiting."); 67 | 68 | Ok(()) 69 | } 70 | 71 | fn build_command_args() -> App<'static, 'static> { 72 | let key_format_arg = Arg::with_name("key_encoding") 73 | .value_name("ENCODING") 74 | .long("key-encoding") 75 | .short("K") 76 | .help("Use the given encoding to show keys in textual form.") 77 | .possible_values(&Encoding::list()) 78 | .default_value(Encoding::Utf8.into()); 79 | 80 | let value_format_arg = Arg::with_name("value_encoding") 81 | .value_name("ENCODING") 82 | .long("value-encoding") 83 | .short("V") 84 | .help("Use the given encoding to show values in textual form.") 85 | .possible_values(&Encoding::list()) 86 | .default_value(Encoding::Utf8.into()); 87 | 88 | App::new("") 89 | .setting(AppSettings::DisableVersion) 90 | .setting(AppSettings::SubcommandRequiredElseHelp) 91 | .setting(AppSettings::NoBinaryName) 92 | .subcommand( 93 | SubCommand::with_name("count").about("Get number of key-value pairs in the database."), 94 | ) 95 | .subcommand( 96 | SubCommand::with_name("get") 97 | .about("Get key-value pair by its key.") 98 | .arg(Arg::with_name("key").required(true)) 99 | .arg(key_format_arg.clone()) 100 | .arg(value_format_arg.clone()), 101 | ) 102 | .subcommand( 103 | SubCommand::with_name("scan") 104 | .about("Get all key-value pairs within a range.") 105 | .arg( 106 | Arg::with_name("key_start") 107 | .value_name("START") 108 | .help("Starting key range (inclusive)."), 109 | ) 110 | .arg( 111 | Arg::with_name("key_end") 112 | .value_name("END") 113 | .help("Ending key range (exclusive)."), 114 | ) 115 | .arg( 116 | Arg::with_name("keys_only") 117 | .long("keys-only") 118 | .short("k") 119 | .help("Show only keys and don't print values."), 120 | ) 121 | .arg(key_format_arg.clone()) 122 | .arg(value_format_arg.clone()), 123 | ) 124 | .subcommand( 125 | SubCommand::with_name("put") 126 | .about("Insert a key-value pair.") 127 | .arg(Arg::with_name("key").required(true)) 128 | .arg(Arg::with_name("value").required(true)) 129 | .arg(key_format_arg.clone()) 130 | .arg(value_format_arg.clone()), 131 | ) 132 | .subcommand( 133 | SubCommand::with_name("remove") 134 | .about("Remove a key-value pair by its key.") 135 | .arg(Arg::with_name("key").required(true)) 136 | .arg(key_format_arg.clone()), 137 | ) 138 | .subcommand( 139 | SubCommand::with_name("preview") 140 | .about("Read a pair's value as a document and show it textually.") 141 | .after_help( 142 | "The textual representation of the document is intended for inspection only. \ 143 | In particular of binary formats, the representation may not be canonical \ 144 | and may not convert back to binary format without data loss.", 145 | ) 146 | .arg(Arg::with_name("key").required(true)) 147 | .arg( 148 | Arg::with_name("format") 149 | .required(true) 150 | .possible_values(&DocumentFormat::list()), 151 | ) 152 | .arg(key_format_arg.clone()), 153 | ) 154 | .subcommand(SubCommand::with_name("flush").about("Persist changes to database.")) 155 | .subcommand(SubCommand::with_name("exit").about("Exit the inspector.")) 156 | } 157 | 158 | enum CommandResult { 159 | Continue, 160 | Exit, 161 | Error(anyhow::Error), 162 | } 163 | 164 | fn execute_command(database: &mut Database, line: &str) -> anyhow::Result { 165 | let args = build_command_args(); 166 | 167 | match args.get_matches_from_safe(shell_words::split(line)?) { 168 | Ok(matches) => match matches.subcommand() { 169 | ("count", _) => { 170 | count_command(database); 171 | Ok(CommandResult::Continue) 172 | } 173 | ("get", sub_args) => { 174 | get_command(database, sub_args.unwrap())?; 175 | Ok(CommandResult::Continue) 176 | } 177 | ("scan", sub_args) => { 178 | scan_command(database, sub_args.unwrap())?; 179 | Ok(CommandResult::Continue) 180 | } 181 | ("put", sub_args) => { 182 | put_command(database, sub_args.unwrap())?; 183 | Ok(CommandResult::Continue) 184 | } 185 | ("remove", sub_args) => { 186 | remove_command(database, sub_args.unwrap())?; 187 | Ok(CommandResult::Continue) 188 | } 189 | ("flush", _) => { 190 | flush_command(database)?; 191 | Ok(CommandResult::Continue) 192 | } 193 | ("preview", sub_args) => { 194 | preview_command(database, sub_args.unwrap())?; 195 | Ok(CommandResult::Continue) 196 | } 197 | ("exit", _) => Ok(CommandResult::Exit), 198 | _ => unreachable!(), 199 | }, 200 | Err(error) => Ok(CommandResult::Error(anyhow::Error::new(error))), 201 | } 202 | } 203 | 204 | fn encoding_from_args<'a>(args: &'a ArgMatches, name: &str) -> Encoding { 205 | args.value_of(name) 206 | .unwrap_or_default() 207 | .try_into() 208 | .unwrap_or(Encoding::Utf8) 209 | } 210 | 211 | fn text_or_error_from_args<'a>(args: &'a ArgMatches, name: &str) -> anyhow::Result<&'a str> { 212 | let text = args 213 | .value_of(name) 214 | .ok_or_else(|| anyhow::anyhow!("Invalid UTF-8 input"))?; 215 | Ok(text) 216 | } 217 | 218 | fn count_command(database: &mut Database) { 219 | let metadata = database.metadata(); 220 | metadata.key_value_count(); 221 | 222 | println!("{}", metadata.key_value_count()); 223 | } 224 | 225 | fn get_command<'a>(database: &mut Database, args: &'a ArgMatches) -> anyhow::Result<()> { 226 | let key_encoding = encoding_from_args(args, "key_encoding"); 227 | let value_encoding = encoding_from_args(args, "value_encoding"); 228 | 229 | let key = text_or_error_from_args(args, "key")?; 230 | let key = self::encoding::text_to_binary(key, key_encoding)?; 231 | 232 | let value = database.get(key)?; 233 | if let Some(value) = value { 234 | let value = self::encoding::binary_to_text(&value, value_encoding); 235 | 236 | println!("{}", value); 237 | } 238 | 239 | Ok(()) 240 | } 241 | 242 | fn scan_command<'a>(database: &mut Database, args: &'a ArgMatches) -> anyhow::Result<()> { 243 | let key_encoding = encoding_from_args(args, "key_encoding"); 244 | let value_encoding = encoding_from_args(args, "value_encoding"); 245 | 246 | let key_start = args.value_of("key_start").unwrap_or_default(); 247 | let key_start = self::encoding::text_to_binary(key_start, key_encoding)?; 248 | 249 | let key_end = args.value_of("key_end").unwrap_or_default(); 250 | let key_end = self::encoding::text_to_binary(key_end, key_encoding)?; 251 | 252 | let cursor = { 253 | if !key_end.is_empty() { 254 | database.cursor_range(key_start..key_end)? 255 | } else { 256 | database.cursor_range(key_start..)? 257 | } 258 | }; 259 | 260 | for (key, value) in cursor { 261 | let key = self::encoding::binary_to_text(&key, key_encoding); 262 | let value = self::encoding::binary_to_text(&value, value_encoding); 263 | 264 | println!("{}", key); 265 | 266 | if !args.is_present("keys_only") { 267 | println!("{}", value); 268 | } 269 | } 270 | 271 | Ok(()) 272 | } 273 | 274 | fn put_command<'a>(database: &mut Database, args: &'a ArgMatches) -> anyhow::Result<()> { 275 | let key_encoding = encoding_from_args(args, "key_encoding"); 276 | let value_encoding = encoding_from_args(args, "value_encoding"); 277 | 278 | let key = text_or_error_from_args(args, "key")?; 279 | let key = self::encoding::text_to_binary(key, key_encoding)?; 280 | 281 | let value = text_or_error_from_args(args, "value")?; 282 | let value = self::encoding::text_to_binary(value, value_encoding)?; 283 | 284 | database.put(key, value)?; 285 | println!("OK"); 286 | 287 | Ok(()) 288 | } 289 | 290 | fn remove_command<'a>(database: &mut Database, args: &'a ArgMatches) -> anyhow::Result<()> { 291 | let key_encoding = encoding_from_args(args, "key_encoding"); 292 | 293 | let key = text_or_error_from_args(args, "key")?; 294 | let key = self::encoding::text_to_binary(key, key_encoding)?; 295 | 296 | database.remove(key)?; 297 | println!("OK"); 298 | 299 | Ok(()) 300 | } 301 | 302 | fn flush_command(database: &mut Database) -> anyhow::Result<()> { 303 | database.flush()?; 304 | println!("OK"); 305 | Ok(()) 306 | } 307 | 308 | fn preview_command<'a>(database: &mut Database, args: &'a ArgMatches) -> anyhow::Result<()> { 309 | let key_encoding = encoding_from_args(args, "key_encoding"); 310 | 311 | let document_format = args.value_of("format").unwrap_or_default().try_into()?; 312 | 313 | let key = text_or_error_from_args(args, "key")?; 314 | let key = self::encoding::text_to_binary(key, key_encoding)?; 315 | 316 | let value = database.get(key)?; 317 | 318 | if let Some(value) = value { 319 | let document = self::encoding::binary_to_document(&value, document_format)?; 320 | 321 | println!("{}", document); 322 | } 323 | 324 | Ok(()) 325 | } 326 | -------------------------------------------------------------------------------- /src/tool/src/verify.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use grebedb::{Database, OpenMode, Options}; 4 | 5 | pub fn verify(database_path: &Path, write: bool, verbose: bool) -> anyhow::Result<()> { 6 | let options = Options { 7 | open_mode: if write { 8 | OpenMode::LoadOnly 9 | } else { 10 | OpenMode::ReadOnly 11 | }, 12 | ..Default::default() 13 | }; 14 | 15 | let mut database = Database::open_path(database_path, options)?; 16 | 17 | database.verify(|current, total| { 18 | if verbose { 19 | let percent = if total > 0 { 20 | current as f64 / total as f64 * 100.0 21 | } else { 22 | 0.0 23 | }; 24 | eprintln!("\t{:.1}%\t{}\t{}", percent, current, total); 25 | } 26 | })?; 27 | 28 | if verbose { 29 | eprintln!("OK"); 30 | } 31 | 32 | Ok(()) 33 | } 34 | --------------------------------------------------------------------------------