├── .github └── workflows │ ├── release.yml │ └── tests.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── block ├── Cargo.lock ├── Cargo.toml ├── README.md ├── src │ └── lib.rs └── tests │ └── block_test.rs ├── block_macro ├── Cargo.toml ├── README.md └── src │ └── lib.rs ├── examples ├── Cargo.lock ├── Cargo.toml ├── README.md └── sync-backend │ ├── Cargo.toml │ └── src │ └── main.rs ├── journal ├── Cargo.toml ├── README.md ├── src │ ├── async_bridge.rs │ ├── async_journal.rs │ ├── error.rs │ ├── journal.rs │ ├── lib.rs │ └── stream.rs └── tests │ └── journal_tests.rs ├── libsqlite-sys ├── Cargo.toml ├── README.md ├── build.rs ├── src │ ├── alloc.rs │ ├── ffi.rs │ ├── iter.rs │ ├── lib.rs │ ├── sqlite_value.rs │ ├── util.rs │ └── vtab.rs ├── tests │ └── allocator_tests.rs └── wrapper.h ├── mycelite ├── .gitignore ├── Cargo.toml └── src │ ├── config.rs │ ├── lib.rs │ ├── replicator │ ├── http_replicator.rs │ ├── mod.rs │ └── noop_replicator.rs │ └── vfs.rs ├── page_parser ├── Cargo.toml ├── README.md ├── src │ ├── database.rs │ ├── header.rs │ ├── lib.rs │ └── page.rs └── tests │ └── header_test.rs ├── serde_sqlite ├── Cargo.toml ├── src │ ├── de.rs │ ├── error.rs │ ├── lib.rs │ └── se.rs └── tests │ ├── de_test.rs │ └── se_test.rs └── utils ├── Cargo.toml └── src └── lib.rs /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | release: 10 | runs-on: ${{ matrix.os }} 11 | 12 | env: 13 | RUSTFLAGS: "-C target-feature=-crt-static" 14 | PKG_CONFIG_ALLOW_CROSS: 1 15 | CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc 16 | CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc 17 | 18 | strategy: 19 | matrix: 20 | include: 21 | - build: linux 22 | os: ubuntu-22.04 23 | target: x86_64-unknown-linux-gnu 24 | 25 | - build: linux 26 | os: ubuntu-22.04 27 | target: x86_64-unknown-linux-musl 28 | 29 | - build: linux 30 | os: ubuntu-22.04 31 | target: aarch64-unknown-linux-gnu 32 | 33 | - build: linux 34 | os: ubuntu-20.04 ## older ubuntu to avoid messing with glibc version 35 | target: arm-unknown-linux-gnueabihf 36 | 37 | - build: macos 38 | os: macos-12 39 | target: x86_64-apple-darwin 40 | 41 | - build: macos 42 | os: macos-12 43 | target: aarch64-apple-darwin 44 | 45 | - build: windows 46 | os: windows-2022 47 | target: x86_64-pc-windows-msvc 48 | 49 | - build: windows 50 | os: windows-2022 51 | target: x86_64-pc-windows-gnu 52 | 53 | 54 | steps: 55 | - name: checkout 56 | uses: actions/checkout@v3 57 | 58 | - name: install packages (linux) 59 | if: matrix.build == 'linux' 60 | run: | 61 | sudo apt update 62 | sudo apt install libsqlite3-dev gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu musl-tools -qy 63 | 64 | - name: install packages (macos) 65 | if: matrix.build == 'macos' 66 | run: | 67 | brew install sqlite3 pkg-config 68 | 69 | - name: install packages (windows) 70 | if: matrix.build == 'windows' 71 | run: | 72 | choco install -y --no-progress --allow-empty-checksums --fail-on-stderr pkgconfiglite 73 | 74 | $env:PKG_CONFIG_PATH="C:/sqlite3/sqlite-amalgamation-3400100/" 75 | $env:SQLITE_URL="https://www.sqlite.org/2022/sqlite-amalgamation-3400100.zip" 76 | mkdir C:\sqlite3\ 77 | curl $env:SQLITE_URL -o C:\sqlite3\sqlite3.zip 78 | Expand-Archive C:\sqlite3\sqlite3.zip -DestinationPath C:\sqlite3\ 79 | @" 80 | Name: sqlite3 81 | Description: sqlite3 amalgamation 82 | Version: 3.40.1 83 | Cflags: -I${env:PKG_CONFIG_PATH} 84 | Libs: 85 | "@ > $env:PKG_CONFIG_PATH\sqlite3.pc 86 | echo "PKG_CONFIG_PATH=$env:PKG_CONFIG_PATH" >> $env:GITHUB_ENV 87 | 88 | - name: setup rust 89 | uses: dtolnay/rust-toolchain@master 90 | with: 91 | toolchain: stable 92 | target: ${{ matrix.target }} 93 | 94 | - name: build binary 95 | run: | 96 | cargo build --verbose --release --target ${{ matrix.target }} 97 | ls target/${{ matrix.target }}/release/ 98 | 99 | - name: build archive 100 | if: matrix.build == 'linux' 101 | run: | 102 | export ARTIFACT_NAME=${{ matrix.target }}.tgz 103 | tar -czf $ARTIFACT_NAME -C "./target/${{ matrix.target }}/release/" libmycelite.so 104 | echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV 105 | 106 | - name: build archive 107 | if: matrix.build == 'macos' 108 | run: | 109 | export ARTIFACT_NAME=${{ matrix.target }}.tgz 110 | tar -czf $ARTIFACT_NAME -C "./target/${{ matrix.target }}/release/" libmycelite.dylib 111 | echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV 112 | 113 | - name: build archive 114 | if: matrix.build == 'windows' 115 | run: | 116 | $env:ARTIFACT_NAME="${{ matrix.target }}.zip" 117 | mv .\target\${{ matrix.target }}\release\mycelite.dll .\target\${{ matrix.target }}\release\libmycelite.dll 118 | Compress-Archive -Path .\target\${{ matrix.target }}\release\libmycelite.dll -DestinationPath $env:ARTIFACT_NAME 119 | echo "ARTIFACT_NAME=$env:ARTIFACT_NAME" >> $env:GITHUB_ENV 120 | 121 | - name: release 122 | uses: ncipollo/release-action@v1 123 | with: 124 | artifacts: ${{ env.ARTIFACT_NAME }} 125 | artifactErrorsFailBuild: true 126 | allowUpdates: true 127 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: [push] 4 | 5 | env: 6 | CARGO_TERM_COLOR: always 7 | 8 | jobs: 9 | linux: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - name: Run tests 14 | run: | 15 | cargo test --verbose --no-default-features 16 | cargo test --verbose 17 | cargo test --release --verbose 18 | 19 | macos: 20 | runs-on: macos-latest 21 | steps: 22 | - uses: actions/checkout@v3 23 | - name: Run tests 24 | run: | 25 | brew install pkg-config 26 | cargo test --verbose --no-default-features 27 | cargo test --verbose 28 | cargo test --release --verbose 29 | 30 | windows: 31 | runs-on: windows-latest 32 | env: 33 | SQLITE_URL: "https://www.sqlite.org/2022/sqlite-amalgamation-3400100.zip" 34 | PKG_CONFIG_PATH: "C:/sqlite3/sqlite-amalgamation-3400100/" 35 | 36 | steps: 37 | - uses: actions/checkout@v3 38 | - name: install pkg-config, download and extract sqlite amalgamation, setup pkg-config file for sqlite3 39 | run: | 40 | choco install -y --no-progress --allow-empty-checksums --fail-on-stderr pkgconfiglite 41 | 42 | mkdir C:\sqlite3\ 43 | curl $env:SQLITE_URL -o C:\sqlite3\sqlite3.zip 44 | Expand-Archive C:\sqlite3\sqlite3.zip -DestinationPath C:\sqlite3\ 45 | @" 46 | Name: sqlite3 47 | Description: sqlite3 amalgamation 48 | Version: 3.40.1 49 | Cflags: -I${env:PKG_CONFIG_PATH} 50 | Libs: 51 | "@ > $env:PKG_CONFIG_PATH\sqlite3.pc 52 | - name: Run tests 53 | run: | 54 | cargo test --verbose --no-default-features 55 | cargo test --verbose 56 | cargo test --release --verbose 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "block", 4 | "block_macro", 5 | "journal", 6 | "libsqlite-sys", 7 | "mycelite", 8 | "page_parser", 9 | "serde_sqlite", 10 | "utils", 11 | ] 12 | resolver = "2" 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![tests](https://github.com/mycelial/mycelite/actions/workflows/tests.yml/badge.svg) 2 | 3 | # Mycelite 4 | 5 | Mycelite implements physical, single-writer replication for SQLite. 6 | 7 | ### Technical details 8 | 9 | - Mycelite is a [VFS](https://www.sqlite.org/vfs.html) extension, which acts 10 | as a proxy for the OS filesystem. 11 | - Mycelite intercepts page writes and creates a binary diff with the old version. 12 | - The binary diffs are then stored in the [journal](./journal/README.md). They can also be sent 13 | over the network to another machine. 14 | - The diffs in the journal can then be sequentially applied, thus achieving 15 | bit-perfect copy of the original database. 16 | 17 | For more details on SQLite binary format see [sqlite-parser-nom](https://github.com/mycelial/sqlite-parser-nom). 18 | In principle, it could be illustrated in the following way: 19 | 20 | ``` 21 | ┌───────────┐ VFS write ┌────────────┐ apply ┌────────────────┐ 22 | │ db.sqlite ├──────────┐ │ db.journal ├───────► replica.sqlite │ 23 | ├───────────┤ │ ├────────────┤ ├────────────────┤ 24 | │ header │ ┌──────▼─────┐ ┌─► diff 0 │ │ header │ 25 | ├───────────┤ │ page 0 new ├─┐ │ ├────────────┤ ├────────────────┤ 26 | │ page 0 ├─┐ └────────────┘ │ ┌──────┐ │ │ ... │ │ page 0 │ 27 | ├───────────┤ │ ├─► diff ├─┘ └────────────┘ ├────────────────┤ 28 | │ page 1 │ │ ┌────────────┐ │ └──────┘ │ ... │ 29 | └───────────┘ └─► page 0 old ├─┘ └────────────────┘ 30 | └────────────┘ 31 | ``` 32 | 33 | This approach comes with both significant upsides and downsides: 34 | - Replica will contain exactly the same object in exactly the same order as in original. 35 | - Out-of-the-box non-deterministic DDLs (e.g., UPDATE with RANDOM() or CURRENT_TIMESTAMP). 36 | - Physical replication is less resource-intensive than logical replication, resulting in 37 | higher throughput with no penalty as the number of replicas grows. 38 | - Time travel by hydrating up to any previous timestamp. 39 | - As there is no locking mechanism currently implemented, only a single writer is supported. 40 | - Replica journal grows linearly, unless compacted. 41 | - VACUUM operation might result in significantly sized journal entry without 42 | actual changes to accessible data. 43 | - Currently, [WAL](https://www.sqlite.org/wal.html)-enabled databases are not supported. 44 | 45 | ### Usage 46 | Refer to the [Quickstart Documentation](https://mycelial.com/docs/get-started/quick-start). 47 | 48 | ### A new type of application 49 | 50 | There is a new type of application called local-first, which combines many of 51 | the best features from both local and client/server applications. 52 | 53 | ### What does local-first offer? 54 | 55 | With local-first applications, you get the speed and responsiveness of a local 56 | application, but at the same time, you get many of the desirable features from 57 | client/server systems. 58 | 59 | ### What do local-first applications look like? 60 | 61 | A good example of a local-first application is [Actual 62 | Budget](https://github.com/actualbudget/actual), an open-source personal finance 63 | application. 64 | 65 | What makes Actual Budget different from its competitors? 66 | 67 | First of all, it's very fast because all the application data is on the local 68 | device - in a SQLite database - but what's most interesting about this app is 69 | that it works on multiple devices. In other words, it has apps for iOS, Android, 70 | Windows, Mac and the Web and it allows you to make concurrent changes on 71 | multiple devices and synchronize those changes to all your devices. 72 | 73 | ### Why aren't more developers creating local-first applications? 74 | 75 | Actual Budget is a good example of a local-first application, but it wasn't very 76 | easy to build. The authors had to write a bunch of synchronization-related code, 77 | that implements and uses 78 | [CRDTs](https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type), and 79 | this start-from-scratch approach just isn't practical for most situations. 80 | Building local-first applications today is too difficult, but we're going to 81 | change that. 82 | -------------------------------------------------------------------------------- /block/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "block" 7 | version = "0.1.0" 8 | dependencies = [ 9 | "block_macro", 10 | ] 11 | 12 | [[package]] 13 | name = "block_macro" 14 | version = "0.1.0" 15 | dependencies = [ 16 | "proc-macro2", 17 | "quote", 18 | "syn", 19 | ] 20 | 21 | [[package]] 22 | name = "proc-macro2" 23 | version = "1.0.49" 24 | source = "registry+https://github.com/rust-lang/crates.io-index" 25 | checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" 26 | dependencies = [ 27 | "unicode-ident", 28 | ] 29 | 30 | [[package]] 31 | name = "quote" 32 | version = "1.0.23" 33 | source = "registry+https://github.com/rust-lang/crates.io-index" 34 | checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" 35 | dependencies = [ 36 | "proc-macro2", 37 | ] 38 | 39 | [[package]] 40 | name = "syn" 41 | version = "1.0.107" 42 | source = "registry+https://github.com/rust-lang/crates.io-index" 43 | checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" 44 | dependencies = [ 45 | "proc-macro2", 46 | "quote", 47 | "unicode-ident", 48 | ] 49 | 50 | [[package]] 51 | name = "unicode-ident" 52 | version = "1.0.6" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" 55 | -------------------------------------------------------------------------------- /block/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "block" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [dependencies] 8 | block_macro = { path = "../block_macro" } 9 | -------------------------------------------------------------------------------- /block/README.md: -------------------------------------------------------------------------------- 1 | ## README 2 | Block trait to indicate block size on the disk for provided structure 3 | -------------------------------------------------------------------------------- /block/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Block trait 2 | pub use block_macro::block; 3 | 4 | pub trait Block { 5 | fn block_size() -> usize; 6 | 7 | /// size of instance of the block, for enums it's tag + size of variant arm 8 | /// 9 | /// only new-type enums are currently supported 10 | fn iblock_size(&self) -> usize { 11 | Self::block_size() 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /block/tests/block_test.rs: -------------------------------------------------------------------------------- 1 | use block::Block; 2 | use block_macro::*; 3 | 4 | #[block(512)] 5 | struct S {} 6 | 7 | #[block] 8 | enum E { 9 | E(S), 10 | } 11 | 12 | #[test] 13 | fn test_block_size() { 14 | assert_eq!(::block_size(), 512); 15 | assert_eq!(::block_size(), 4); 16 | assert_eq!(S::block_size(), 512); 17 | assert_eq!(E::block_size(), 4); 18 | } 19 | 20 | #[block] 21 | enum NewTypeEnum { 22 | S(S), 23 | E(E), 24 | } 25 | 26 | #[test] 27 | fn test_new_type_enum() { 28 | assert_eq!(::block_size(), 4); 29 | 30 | let instance = NewTypeEnum::S(S {}); 31 | assert_eq!(instance.iblock_size(), 512 + 4); 32 | 33 | let instance = NewTypeEnum::E(E::E(S {})); 34 | assert_eq!(instance.iblock_size(), 4 + 4 + 512); 35 | } 36 | -------------------------------------------------------------------------------- /block_macro/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "block_macro" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | proc-macro = true 9 | 10 | [dependencies] 11 | syn = { version = "1", features = ["full"] } 12 | quote = "1" 13 | proc-macro2 = "1" 14 | -------------------------------------------------------------------------------- /block_macro/README.md: -------------------------------------------------------------------------------- 1 | ## Block macro 2 | Attribute macro for `block` crate 3 | 4 | ## Example 5 | ```rust 6 | use block_macro::block; 7 | 8 | #[block(512)] 9 | struct S { 10 | 11 | } 12 | 13 | assert_eq!(S::block_size(), 512) 14 | ``` 15 | -------------------------------------------------------------------------------- /block_macro/src/lib.rs: -------------------------------------------------------------------------------- 1 | use proc_macro::TokenStream; 2 | use proc_macro2::TokenStream as TokenStream2; 3 | use quote::ToTokens; 4 | 5 | /// extract block size from attribute 6 | /// 7 | /// for enums block size should not be specified, tag value is always u32 (due to serde) 8 | fn extract_block_size(args: &syn::AttributeArgs) -> Option { 9 | match args.as_slice() { 10 | [] => None, 11 | [syn::NestedMeta::Lit(syn::Lit::Int(ref int))] => { 12 | Some(int.base10_parse::().expect("invalid block size")) 13 | } 14 | [_] => panic!("expected integer literal"), 15 | _ => panic!("unexpected number of arguments"), 16 | } 17 | } 18 | 19 | /// extact instance block size 20 | /// 21 | /// for structs it's the same as a block size 22 | /// for enums - for now only new-type enums are supported and each arm has size of inner element, 23 | /// which should implement Block trait. 24 | fn extract_instance_block_size( 25 | item: &syn::DeriveInput, 26 | block_size: &Option, 27 | ) -> TokenStream2 { 28 | match item.data { 29 | syn::Data::Struct(_) if block_size.is_some() => { 30 | let block_size = block_size.unwrap(); 31 | quote::quote! { 32 | fn block_size() -> usize { 33 | #block_size 34 | } 35 | } 36 | } 37 | syn::Data::Struct(_) => { 38 | quote::quote! { 39 | std::compile_error!("block for structs require size") 40 | } 41 | } 42 | syn::Data::Enum(ref enum_data) if block_size.is_none() => { 43 | // build iterafor over enum arms 44 | // it's either valid tuple of enum::variant => 45 | // or enum::variant => compile_error!(...) to simplify debug 46 | let enum_arms_iter = enum_data.variants.iter().map(|v| { 47 | let arm_ident = &v.ident; 48 | let arm_ident = quote::quote!{ Self::#arm_ident }; 49 | if let syn::Fields::Unnamed(ref field) = v.fields { 50 | if field.unnamed.len() == 1 { 51 | if let syn::Type::Path(ref type_path) = field.unnamed[0].ty { 52 | let type_ident = type_path.path.get_ident(); 53 | return quote::quote! { 54 | #arm_ident(ref v) => <#type_ident as ::block::Block>::iblock_size(v), 55 | } 56 | } 57 | } 58 | } 59 | let span = v.ident.span(); 60 | quote::quote_spanned!{ span => _ => { 61 | std::compile_error!("only new-type enums with arity of 1 are supported"); 62 | unimplemented!() 63 | },} 64 | }); 65 | let block_size = quote::quote! { ::block_size() }; 66 | quote::quote! { 67 | fn block_size() -> usize { 68 | 4 69 | } 70 | 71 | fn iblock_size(&self) -> usize { 72 | #block_size + match &self { 73 | #(#enum_arms_iter)* 74 | } 75 | } 76 | } 77 | } 78 | syn::Data::Union(_) => { 79 | let span = item.ident.span(); 80 | quote::quote_spanned! { span => std::compiler_error!("unions are not supported") } 81 | } 82 | syn::Data::Enum(_) => { 83 | quote::quote! { 84 | std::compile_error!("enum blocks should not have size and always u32, due to how serde works with enums"); 85 | } 86 | } 87 | } 88 | } 89 | 90 | #[proc_macro_attribute] 91 | pub fn block(args: TokenStream, item: TokenStream) -> TokenStream { 92 | let args = &syn::parse_macro_input!(args as syn::AttributeArgs); 93 | let item = &syn::parse_macro_input!(item as syn::DeriveInput); 94 | 95 | let block_size = extract_block_size(args); 96 | let methods = extract_instance_block_size(item, &block_size); 97 | 98 | let ident = &item.ident; 99 | let block_implementation = quote::quote! { 100 | impl ::block::Block for #ident { 101 | #methods 102 | } 103 | }; 104 | 105 | let mut item = item.to_token_stream(); 106 | item.extend(block_implementation); 107 | item.into() 108 | } 109 | -------------------------------------------------------------------------------- /examples/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["*"] 3 | exclude = ["target"] -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | Each example is a separate crate so its dependencies are clear. 4 | 5 | ### Sync Backend 6 | 7 | It was done for the demo purposes only, uses blocking I/O and supports a single database at a time. 8 | 9 | ```sh 10 | $ cargo run -p sync-backend 11 | ``` 12 | -------------------------------------------------------------------------------- /examples/sync-backend/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sync-backend" 3 | version = "0.1.0" 4 | edition = "2021" 5 | publish = false 6 | 7 | [dependencies] 8 | libsqlite-sys = { path = "../../libsqlite-sys" } 9 | journal = { path = "../../journal", features = ["async_bridge"] } 10 | serde_sqlite = { path = "../../serde_sqlite" } 11 | 12 | tokio = { version = "1", features = ["full"] } 13 | axum = { version = "0.6", features = ["headers"] } 14 | tracing = "0.1" 15 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 16 | futures = { version = "0.3" } 17 | tokio-util = { version = "0.7", features = ["io"] } 18 | serde = { version = "1.0", features = ["derive"] } 19 | -------------------------------------------------------------------------------- /examples/sync-backend/src/main.rs: -------------------------------------------------------------------------------- 1 | //! Example data synchronization backend 2 | //! 3 | //! ** Strictly for the demo purposes only ** 4 | //! ** Known issues **: 5 | //! - It works only for single database 6 | //! 7 | //! Run with 8 | //! 9 | //! ```not_rust 10 | //! cd examples && cargo run -p sync-backend 11 | //! ``` 12 | 13 | use axum::{ 14 | extract::{BodyStream, Path, State, Query}, 15 | http::StatusCode, 16 | body, 17 | response, 18 | routing::get, 19 | Router, Server, 20 | }; 21 | use futures::StreamExt; 22 | use journal::{Journal, AsyncReadJournalStream, AsyncWriteJournalStream}; 23 | use tokio::io::AsyncWriteExt; 24 | use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; 25 | use serde::Deserialize; 26 | 27 | fn to_error(_e: T) -> StatusCode { 28 | StatusCode::INTERNAL_SERVER_ERROR 29 | } 30 | 31 | #[derive(Debug, Default, Deserialize)] 32 | #[allow(dead_code)] 33 | struct Params { 34 | #[serde(rename="snapshot-id")] 35 | snapshot_id: u64, 36 | } 37 | 38 | /// post new journal snapshots 39 | async fn post_snapshot( 40 | State(state): State, 41 | Path(_domain): Path, 42 | mut stream: BodyStream, 43 | ) -> Result<&'static str, StatusCode> { 44 | let mut write_stream = AsyncWriteJournalStream::new(state.journal_path).spawn(); 45 | while let Some(chunk) = stream.next().await { 46 | let chunk = chunk.map_err(to_error)?; 47 | write_stream.write_all(&chunk).await.map_err(to_error)?; 48 | }; 49 | Ok("OK") 50 | } 51 | 52 | /// get latest knowns snapshot num 53 | async fn head_snapshot( 54 | State(state): State, 55 | Path(_domain): Path, 56 | ) -> Result { 57 | let res = tokio::task::spawn_blocking(move ||{ 58 | let journal = Journal::try_from(state.journal_path) 59 | .or_else(|_e| Journal::create(state.journal_path))?; 60 | Ok::<_, journal::Error>(journal.get_header().snapshot_counter) 61 | }); 62 | let snapshot_id = res.await.map_err(to_error)?.map_err(to_error)?; 63 | let headers = response::AppendHeaders([("x-snapshot-id", snapshot_id.to_string())]); 64 | Ok((headers, "head")) 65 | } 66 | 67 | /// get new snapshots 68 | async fn get_snapshot( 69 | State(state): State, 70 | Path(_domain): Path, 71 | params: Option>, 72 | ) -> Result { 73 | let stream = AsyncReadJournalStream::new( 74 | state.journal_path, 75 | params.map(|p| p.snapshot_id).unwrap_or(0) 76 | ).spawn(); 77 | Ok(body::StreamBody::new(tokio_util::io::ReaderStream::new(stream))) 78 | } 79 | 80 | #[derive(Debug, Clone)] 81 | struct AppState { 82 | journal_path: &'static str 83 | } 84 | 85 | impl AppState { 86 | fn new() -> Self { 87 | Self { 88 | journal_path: "/tmp/journal" 89 | } 90 | } 91 | } 92 | 93 | #[tokio::main(flavor = "current_thread")] 94 | async fn main() { 95 | tracing_subscriber::registry() 96 | .with(tracing_subscriber::EnvFilter::new( 97 | std::env::var("RUST_LOG") 98 | .unwrap_or_else(|_| "sync_backend=debug,tower_http=debug".into()), 99 | )) 100 | .with(tracing_subscriber::fmt::layer()) 101 | .init(); 102 | 103 | let app = Router::new() 104 | .route("/domain/:domain", get(get_snapshot).head(head_snapshot).post(post_snapshot)) 105 | .with_state(AppState::new()); 106 | 107 | let addr = std::net::SocketAddr::from(([0, 0, 0, 0], 8080)); 108 | tracing::debug!("listening on {:?}", addr); 109 | Server::bind(&addr) 110 | .serve(app.into_make_service()) 111 | .await 112 | .unwrap() 113 | } 114 | -------------------------------------------------------------------------------- /journal/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "journal" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [features] 8 | default = [] 9 | async = ["dep:tokio", "dep:futures", "dep:tokio-stream", "dep:async-stream"] 10 | async_bridge = ["dep:tokio"] 11 | 12 | [dependencies] 13 | block = { path = "../block" } 14 | serde = { version = "1", features = ["derive"] } 15 | chrono = { version = "0.4", default-features=false, features = ["std", "clock"] } 16 | serde_sqlite = { path = "../serde_sqlite" } 17 | tokio = { version = "1", optional = true, features=["full"]} 18 | futures = {version = "0.3.27", optional = true} 19 | tokio-stream = { version = "0.1.12", optional = true } 20 | async-stream = { version = "0.3.4", optional = true } 21 | 22 | [dev-dependencies] 23 | tempfile = "3" 24 | quickcheck = "1" 25 | spin_sleep = "1" 26 | -------------------------------------------------------------------------------- /journal/README.md: -------------------------------------------------------------------------------- 1 | # The Mycelite Journal 2 | 3 | The Mycelite Journal provides the ability to capture SQLite page changes at the transaction level. 4 | 5 | ![Mycelial VFS Journal V1](https://user-images.githubusercontent.com/504968/204807386-5da165b7-6aef-44ca-ac09-b736c666c297.png) 6 | 7 | The Mycelite Journal is composed of two components: the Header, which begins the Journal, and one or more snapshots. 8 | 9 | ## Mycelite Journal Header 10 | 11 | The first 128 bytes of the Mycelite Journal is the journal header. These 128 bytes contain: 12 | 13 | - Magic (4 bytes) - Constant value of `0x00907a70` (read as "potato"). Why potato? Because the engineer who built this decided to call it potato. (For more information on file signatures known as "Magic Bytes," see [List of File Signatures](https://en.wikipedia.org/wiki/List_of_file_signatures)). 14 | 15 | - Version (4 bytes) - Represents the Journal version number. 16 | 17 | - Snapshot Counter (8 bytes) - The number of Snapshots in the Journal. 18 | 19 | - EOF (8 bytes)- This is the offset position of the commited Snapshot, and designates the end of the file. 20 | 21 | - The balance of the Journal's header bytes are reserved space. 22 | 23 | ## Mycelite Journal Snapshots 24 | 25 | Each Snapshot represents a SQLite transaction that has been captured by the Mycelite Journal. 26 | 27 | Snapshots are comprised of two components: one Snapshot Header and one or more SQLite pages. 28 | 29 | ### Snapshot Header 30 | 31 | The first 32 bytes of a Mycelite Journal Snapshot is the Snopshot Header. These 32 bytes contain: 32 | 33 | - Id (8 bytes) - a unique id(?) for this Journal Snapshot 34 | - Timestamp (8 bytes) - UTC unixtime timestamp in microseconds 35 | - Reserved (16 bytes) - reserved space 36 | 37 | ### Page Header 38 | 39 | Each SQLite page _also_ has a header, called a Page Header. 40 | 41 | The first 16 bytes of a SQLite page is the Page Header. These 16 bytes contain: 42 | 43 | - Offset (8 bytes) - The offset in the database at which this SQLite page was written 44 | - Page Num - Each SQLite database page has its own number. The first page of a Snapshot starts with 0, and the value is incremented as pages are added. 45 | - Page Size - Represents the Journal page size, which is currently set to the underlying SQLite page size. 46 | 47 | ### Snapshot EOF (end of file) 48 | 49 | The Snapshot ends with the last Page Header, whose values are all set to `0`. 50 | -------------------------------------------------------------------------------- /journal/src/async_bridge.rs: -------------------------------------------------------------------------------- 1 | //! Temporary async wrapping to sync journal 2 | 3 | use crate::{Error as JournalError, Journal, Protocol, Stream as JournalStream}; 4 | use serde_sqlite::de; 5 | use tokio::sync::mpsc::error::TrySendError; 6 | use std::io::{BufRead, Read, Write}; 7 | use std::path::PathBuf; 8 | use std::pin::Pin; 9 | use std::task::{Context, Poll, Waker}; 10 | use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; 11 | use tokio::sync::mpsc::{channel, error::TryRecvError, Receiver, Sender}; 12 | 13 | fn to_err(err: E) -> std::io::Error { 14 | std::io::Error::new(std::io::ErrorKind::Other, err) 15 | } 16 | 17 | pub struct AsyncReadJournalStream { 18 | snapshot_id: u64, 19 | journal_path: PathBuf, 20 | } 21 | 22 | impl AsyncReadJournalStream { 23 | pub fn new>(journal_path: P, snapshot_id: u64) -> Self { 24 | AsyncReadJournalStream { 25 | journal_path: journal_path.into(), 26 | snapshot_id, 27 | } 28 | } 29 | 30 | pub fn spawn(self) -> AsyncReadJournalStreamHandle { 31 | let (waker_tx, mut waker_rx) = channel::(1); 32 | let (mut buffer_tx, buffer_rx) = channel::>(1); 33 | let join_handle = 34 | tokio::task::spawn_blocking(move || self.enter_loop(&mut waker_rx, &mut buffer_tx)); 35 | AsyncReadJournalStreamHandle { 36 | tx: waker_tx, 37 | rx: buffer_rx, 38 | buf: None, 39 | read: 0, 40 | join_handle, 41 | } 42 | } 43 | 44 | pub fn enter_loop( 45 | self, 46 | rx: &mut Receiver, 47 | tx: &mut Sender>, 48 | ) -> Result<(), JournalError> { 49 | let mut journal = Journal::try_from(self.journal_path.as_path())?; 50 | let version = journal.get_header().version; 51 | let mut stream = JournalStream::new( 52 | journal.into_iter().skip_snapshots(self.snapshot_id), 53 | version, 54 | ); 55 | 56 | while let Some(waker) = rx.blocking_recv() { 57 | let mut buf = Vec::::with_capacity(0x0001_0000); // 65kb buffer 58 | unsafe { buf.set_len(buf.capacity()) }; 59 | let read = match stream.read(buf.as_mut_slice()) { 60 | Ok(read) => read, 61 | Err(e) => { 62 | waker.wake(); 63 | return Err(e.into()); 64 | } 65 | }; 66 | unsafe { buf.set_len(read) }; 67 | let res = tx.blocking_send(buf); 68 | waker.wake(); 69 | if let Err(tokio::sync::mpsc::error::SendError(_)) = res { 70 | let err = std::io::Error::new(std::io::ErrorKind::Other, "channel closed"); 71 | return Err(err.into()); 72 | } 73 | } 74 | Ok(()) 75 | } 76 | } 77 | 78 | #[derive(Debug)] 79 | pub struct AsyncReadJournalStreamHandle { 80 | buf: Option>, 81 | read: usize, 82 | rx: Receiver>, 83 | tx: Sender, 84 | join_handle: tokio::task::JoinHandle>, 85 | } 86 | 87 | impl AsyncReadJournalStreamHandle { 88 | pub async fn join(self) -> Result, tokio::task::JoinError> { 89 | self.join_handle.await 90 | } 91 | } 92 | 93 | impl AsyncRead for AsyncReadJournalStreamHandle { 94 | fn poll_read( 95 | self: Pin<&mut Self>, 96 | ctx: &mut Context<'_>, 97 | buf: &mut ReadBuf<'_>, 98 | ) -> Poll> { 99 | let p = self.get_mut(); 100 | if p.buf.is_none() { 101 | match p.rx.try_recv() { 102 | // EOF 103 | Ok(buf) if buf.is_empty() => return Poll::Ready(Ok(())), 104 | Ok(buf) => { 105 | p.buf = Some(buf); 106 | p.read = 0; 107 | } 108 | // stream thread quit, FIXME: distinction between thread error and EOF 109 | Err(TryRecvError::Disconnected) => return Poll::Ready(Ok(())), 110 | Err(TryRecvError::Empty) => { 111 | p.tx.try_send(ctx.waker().clone()).map_err(to_err)?; 112 | return Poll::Pending; 113 | } 114 | } 115 | } 116 | 117 | let inner_buf = p.buf.as_ref().unwrap(); 118 | let start = p.read; 119 | let end = p.read + buf.remaining(); 120 | match inner_buf.len() { 121 | len if len == start => { 122 | // inner buf was read to the end 123 | p.buf = None; 124 | p.tx.try_send(ctx.waker().clone()).map_err(to_err)?; 125 | Poll::Pending 126 | } 127 | len if len > end => { 128 | // inner buf have enough data to fill incoming buf to the end 129 | let slice = &inner_buf[start..end]; 130 | p.read = end; 131 | buf.put_slice(slice); 132 | Poll::Ready(Ok(())) 133 | } 134 | len => { 135 | // inner buf doesn't have enough data, to fill incoming buffer completely 136 | let slice = &inner_buf[p.read..]; 137 | p.read = len; 138 | buf.put_slice(slice); 139 | Poll::Ready(Ok(())) 140 | } 141 | } 142 | } 143 | } 144 | 145 | #[derive(Debug)] 146 | enum AsyncWriteProto { 147 | WriteBuf(Vec, Waker), 148 | Shutdown(Waker), 149 | } 150 | 151 | pub struct ReadReceiver { 152 | buf: Vec, 153 | buf_pos: usize, 154 | waker: Option, 155 | rx: Receiver, 156 | } 157 | 158 | impl ReadReceiver { 159 | fn new(rx: Receiver) -> Self { 160 | Self { 161 | buf: vec![], 162 | buf_pos: 0, 163 | waker: None, 164 | rx, 165 | } 166 | } 167 | } 168 | 169 | impl BufRead for ReadReceiver { 170 | fn fill_buf(&mut self) -> std::io::Result<&[u8]> { 171 | if self.buf_pos != self.buf.len() { 172 | return Ok(&self.buf[self.buf_pos..]); 173 | } else { 174 | self.buf_pos = 0; 175 | self.buf.clear(); 176 | } 177 | 178 | loop { 179 | match self.rx.blocking_recv() { 180 | Some(AsyncWriteProto::WriteBuf(buf, waker)) => { 181 | waker.wake(); 182 | self.buf = buf; 183 | self.buf_pos = 0; 184 | break; 185 | }, 186 | Some(AsyncWriteProto::Shutdown(waker)) => { 187 | self.waker = Some(waker); 188 | break; 189 | }, 190 | None => { 191 | return Err(std::io::Error::new( 192 | std::io::ErrorKind::Other, 193 | "channel closed", 194 | )) 195 | } 196 | } 197 | } 198 | Ok(self.buf.as_slice()) 199 | } 200 | 201 | fn consume(&mut self, read: usize) { 202 | self.buf_pos += read; 203 | } 204 | } 205 | 206 | impl Read for ReadReceiver { 207 | fn read(&mut self, write_buf: &mut [u8]) -> std::io::Result { 208 | let mut total = 0; 209 | let mut write_buf_len = write_buf.len(); 210 | let mut write_buf = std::io::Cursor::new(write_buf); 211 | loop { 212 | if write_buf_len == 0 { 213 | break; 214 | }; 215 | let mut read_buf = self.fill_buf()?; 216 | if read_buf.is_empty() { 217 | break; 218 | } 219 | if read_buf.len() >= write_buf_len { 220 | read_buf = &read_buf[..write_buf_len]; 221 | } 222 | let written = write_buf.write(read_buf)?; 223 | total += written; 224 | write_buf_len -= written; 225 | self.consume(written) 226 | } 227 | Ok(total) 228 | } 229 | } 230 | 231 | impl Drop for ReadReceiver { 232 | fn drop(&mut self) { 233 | self.rx.close(); 234 | if let Some(waker) = self.waker.take() { 235 | waker.wake(); 236 | } 237 | while let Ok(message) = self.rx.try_recv() { 238 | match message { 239 | AsyncWriteProto::WriteBuf(_buf, waker) => waker.wake(), 240 | AsyncWriteProto::Shutdown(waker) => waker.wake(), 241 | } 242 | } 243 | } 244 | } 245 | 246 | pub struct AsyncWriteJournalStream { 247 | journal_path: PathBuf, 248 | } 249 | 250 | impl AsyncWriteJournalStream { 251 | pub fn new>(journal_path: P) -> Self { 252 | Self { 253 | journal_path: journal_path.into(), 254 | } 255 | } 256 | 257 | pub fn spawn(mut self) -> AsyncWriteJournalStreamHandle { 258 | let (tx, rx) = channel(1); // enough space to store waker and buf 259 | let read_receiver = ReadReceiver::new(rx); 260 | let join_handle = tokio::task::spawn_blocking(move || self.enter_loop(read_receiver)); 261 | AsyncWriteJournalStreamHandle { tx, join_handle } 262 | } 263 | 264 | pub fn enter_loop(&mut self, mut read_receiver: ReadReceiver) -> Result<(), JournalError> { 265 | let mut journal = match Journal::try_from(self.journal_path.as_path()) { 266 | Ok(j) => j, 267 | Err(e) if e.journal_not_exists() => Journal::create(self.journal_path.as_path())?, 268 | Err(e) => return Err(e), 269 | }; 270 | 271 | let expected = Protocol::JournalVersion(1.into()); 272 | match de::from_reader::(&mut read_receiver).map_err(to_err)? { 273 | msg if msg == expected => (), 274 | other => { 275 | let err = std::io::Error::new( 276 | std::io::ErrorKind::Other, 277 | format!("expected {}, got: {}", expected, other), 278 | ); 279 | return Err(err.into()); 280 | } 281 | } 282 | loop { 283 | match de::from_reader::(&mut read_receiver).map_err(to_err)? { 284 | Protocol::SnapshotHeader(snapshot_header) => { 285 | journal.commit().map_err(to_err)?; 286 | journal.add_snapshot(&snapshot_header).map_err(to_err)?; 287 | } 288 | Protocol::BlobHeader(blob_header) => { 289 | let mut blob = vec![0; blob_header.blob_size as usize]; 290 | read_receiver 291 | .read_exact(blob.as_mut_slice()) 292 | .map_err(to_err)?; 293 | journal 294 | .add_blob(&blob_header, blob.as_slice()) 295 | .map_err(to_err)?; 296 | } 297 | Protocol::EndOfStream(_) => { 298 | journal.commit().map_err(to_err)?; 299 | drop(journal); 300 | return Ok(()); 301 | } 302 | msg => { 303 | return Err(std::io::Error::new( 304 | std::io::ErrorKind::Other, 305 | format!("unexpected message: {msg:?}"), 306 | ) 307 | .into()) 308 | } 309 | } 310 | } 311 | } 312 | } 313 | 314 | #[derive(Debug)] 315 | pub struct AsyncWriteJournalStreamHandle { 316 | tx: Sender, 317 | join_handle: tokio::task::JoinHandle>, 318 | } 319 | 320 | impl AsyncWriteJournalStreamHandle { 321 | pub async fn join(self) -> Result, tokio::task::JoinError> { 322 | self.join_handle.await 323 | } 324 | } 325 | 326 | impl AsyncWrite for AsyncWriteJournalStreamHandle { 327 | fn poll_write( 328 | self: Pin<&mut Self>, 329 | ctx: &mut Context<'_>, 330 | buf: &[u8], 331 | ) -> Poll> { 332 | let me = self.get_mut(); 333 | match me.tx.try_send(AsyncWriteProto::WriteBuf(buf.into(), ctx.waker().clone())) { 334 | Ok(_) => Poll::Ready(Ok(buf.len())), 335 | Err(TrySendError::Full(_)) => Poll::Pending, 336 | Err(e@TrySendError::Closed(_)) => Poll::Ready(Err(to_err(e))), 337 | } 338 | } 339 | 340 | fn poll_flush(self: Pin<&mut Self>, _ctx: &mut Context<'_>) -> Poll> { 341 | Poll::Ready(Ok(())) 342 | } 343 | 344 | fn poll_shutdown(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll> { 345 | let me = self.get_mut(); 346 | match me.tx.try_send(AsyncWriteProto::Shutdown(ctx.waker().clone())) { 347 | Ok(_) => Poll::Pending, 348 | Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { 349 | Poll::Pending 350 | }, 351 | Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { 352 | Poll::Ready(Ok(())) 353 | } 354 | } 355 | } 356 | } 357 | -------------------------------------------------------------------------------- /journal/src/async_journal.rs: -------------------------------------------------------------------------------- 1 | use crate::error::Error; 2 | use crate::{journal::DEFAULT_BUFFER_SIZE, BlobHeader, Header, SnapshotHeader}; 3 | use async_stream::try_stream; 4 | use block::Block; 5 | 6 | use futures::Stream; 7 | use std::{path, pin::Pin}; 8 | 9 | use serde_sqlite::{from_bytes, to_bytes}; 10 | 11 | use tokio::io::{ 12 | AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, AsyncWrite, AsyncWriteExt, SeekFrom, 13 | }; 14 | 15 | type Result = std::result::Result; 16 | 17 | #[derive(Debug, Copy, Clone)] 18 | pub struct AsyncJournal 19 | where 20 | F: AsyncRead + AsyncWrite + AsyncSeek, 21 | { 22 | /// Journal header 23 | header: Header, 24 | /// File 25 | fd: F, 26 | /// snapshot page count 27 | blob_count: Option, 28 | /// Buffer size 29 | buffer_sz: usize, 30 | } 31 | 32 | impl AsyncJournal { 33 | /// Create new journal 34 | pub async fn create>(p: P) -> Result { 35 | let fd = tokio::fs::OpenOptions::new() 36 | .create(true) 37 | .write(true) 38 | .read(true) 39 | .open(p.as_ref()) 40 | .await?; 41 | Self::new(Header::default(), fd, None).await 42 | } 43 | 44 | /// Try to instantiate journal from given path 45 | pub async fn try_from>(p: P) -> Result { 46 | let mut fd = tokio::fs::OpenOptions::new() 47 | .write(true) 48 | .read(true) 49 | .open(p) 50 | .await?; 51 | let header = Self::read_header(&mut fd).await?; 52 | Ok(Self::from(header, fd, None)) 53 | } 54 | } 55 | 56 | impl AsyncJournal { 57 | /// Instantiate journal & force header write 58 | pub async fn new(header: Header, mut fd: F, blob_count: Option) -> Result { 59 | Self::write_header(Box::pin(&mut fd), &header).await?; 60 | Ok(Self::from(header, fd, blob_count)) 61 | } 62 | /// Instantiate journal 63 | pub fn from(header: Header, fd: F, blob_count: Option) -> Self { 64 | Self { 65 | header, 66 | blob_count, 67 | buffer_sz: DEFAULT_BUFFER_SIZE, 68 | fd, 69 | } 70 | } 71 | 72 | /// Set buffer size 73 | pub fn set_buffer_size(&mut self, buffer_sz: usize) { 74 | self.buffer_sz = buffer_sz; 75 | } 76 | 77 | /// Get buffer size 78 | pub fn buffer_size(&self) -> usize { 79 | self.buffer_sz 80 | } 81 | 82 | /// Initiate new snapshot 83 | /// 84 | /// * update journal header to correctly setup offset 85 | /// * to initiate snapshot we seek to current end of the file (value stored in header) 86 | /// * switch fd to buffered mode 87 | /// * write snapshot header with current header counter number 88 | pub async fn new_snapshot(&mut self, page_size: u32) -> Result<()> { 89 | if self.blob_count.is_some() { 90 | return Ok(()); 91 | } 92 | self.update_header().await?; 93 | let snapshot_header = SnapshotHeader::new( 94 | self.header.snapshot_counter, 95 | chrono::Utc::now().timestamp_micros(), 96 | Some(page_size), 97 | ); 98 | self.write_snapshot(&snapshot_header).await 99 | } 100 | 101 | 102 | /// Add existing snapshot 103 | /// 104 | /// Re-syncs journal header 105 | pub async fn add_snapshot(&mut self, snapshot_header: &SnapshotHeader) -> Result<()> { 106 | self.update_header().await?; 107 | self.write_snapshot(snapshot_header).await 108 | } 109 | 110 | /// Add new blob 111 | pub async fn new_blob(&mut self, offset: u64, blob: &[u8]) -> Result<()> { 112 | let blob_num = match self.blob_count { 113 | Some(c) => c, 114 | None => return Err(Error::SnapshotNotStarted), 115 | }; 116 | let blob_header = BlobHeader::new(offset, blob_num, blob.len() as u32); 117 | self.add_blob(&blob_header, blob).await 118 | } 119 | 120 | /// Add blob 121 | pub async fn add_blob(&mut self, blob_header: &BlobHeader, blob: &[u8]) -> Result<()> { 122 | if Some(blob_header.blob_num) != self.blob_count { 123 | return Err(Error::OutOfOrderBlob { 124 | blob_num: blob_header.blob_num, 125 | blob_count: self.blob_count, 126 | }); 127 | } 128 | self.blob_count.as_mut().map(|x| { 129 | *x += 1; 130 | *x 131 | }); 132 | self.fd.write_all(&to_bytes(blob_header)?).await?; 133 | self.fd.write_all(blob).await?; 134 | Ok(()) 135 | } 136 | 137 | pub async fn read_blob_header(&mut self) -> Result { 138 | let mut buf: Vec = Vec::with_capacity(BlobHeader::block_size()); 139 | self.fd.read_buf(&mut buf).await?; 140 | from_bytes::(&buf).map_err(Into::into) 141 | } 142 | 143 | pub async fn read_blob(&mut self, size: u32) -> Result> { 144 | if size == 0 { 145 | let result: Vec = Vec::new(); 146 | return Ok(result); 147 | } 148 | let mut buf: Vec = Vec::with_capacity(size as usize); 149 | self.fd.read_buf(&mut buf).await?; 150 | Ok(buf) 151 | } 152 | 153 | fn snapshot_started(&self) -> bool { 154 | self.blob_count.is_some() 155 | } 156 | 157 | /// Commit snapshot 158 | /// 159 | /// * write final empty page to indicate end of snapshot 160 | /// * flush bufwriter (seek() on BufWriter will force flush) 161 | /// * write new header 162 | /// * flush bufwriter 163 | /// * switch fd back to raw mode 164 | pub async fn commit(&mut self) -> Result<()> { 165 | if !self.snapshot_started() { 166 | return Ok(()); 167 | } 168 | // commit snapshot by writting final empty page 169 | self.fd.write_all(&to_bytes(&BlobHeader::last())?).await?; 170 | self.blob_count = None; 171 | 172 | self.header.snapshot_counter += 1; 173 | self.header.eof = self.fd.stream_position().await?; 174 | 175 | Self::write_header(Box::pin(&mut self.fd), &self.header).await?; 176 | self.fd.flush().await?; 177 | Ok(()) 178 | } 179 | 180 | /// Read header from a given fd 181 | /// 182 | /// * seek to start of the file 183 | /// * read header 184 | async fn read_header( 185 | fd: &mut R, 186 | ) -> Result
{ 187 | fd.rewind().await?; 188 | let mut buf = Vec::with_capacity(Header::block_size()); 189 | fd.read_buf(&mut buf).await?; 190 | 191 | from_bytes::
(&buf).map_err(Into::into) 192 | // from_reader(BufReader::new(fd)).map_err(Into::into).unwrap() 193 | } 194 | 195 | /// Write snapshot to journal 196 | /// 197 | /// This function assumes journal header is up to date 198 | async fn write_snapshot(&mut self, snapshot_header: &SnapshotHeader) -> Result<()> { 199 | if snapshot_header.id != self.header.snapshot_counter { 200 | return Err(Error::OutOfOrderSnapshot { 201 | snapshot_id: snapshot_header.id, 202 | journal_snapshot_id: self.header.snapshot_counter, 203 | }); 204 | } 205 | self.fd.seek(SeekFrom::Start(self.header.eof)).await?; 206 | self.fd.write_all(&to_bytes(snapshot_header)?).await?; 207 | self.blob_count = Some(0); 208 | Ok(()) 209 | } 210 | 211 | pub async fn read_snapshot(&mut self) -> Result { 212 | let mut buf = Vec::with_capacity(SnapshotHeader::block_size()); 213 | self.fd.read_buf(&mut buf).await?; 214 | 215 | from_bytes::(&buf).map_err(Into::into) 216 | } 217 | 218 | /// Write header to a given fd 219 | /// 220 | /// * seek to start of the file 221 | /// * write header 222 | async fn write_header( 223 | mut fd: Pin>, 224 | header: &Header, 225 | ) -> Result<()> { 226 | fd.seek(SeekFrom::Start(0)).await?; 227 | let x = to_bytes(header)?; 228 | fd.write_all(&x).await?; 229 | Ok(()) 230 | } 231 | 232 | /// Return current snapshot counter 233 | pub async fn current_snapshot(&self) -> Option { 234 | match self.header.snapshot_counter { 235 | 0 => None, 236 | v => Some(v), 237 | } 238 | } 239 | 240 | /// Update journal header 241 | pub async fn update_header(&mut self) -> Result<()> { 242 | let h = Self::read_header(&mut self.fd).await?; 243 | self.header = h; 244 | Ok(()) 245 | } 246 | 247 | pub fn get_header(&self) -> &Header { 248 | &self.header 249 | } 250 | 251 | pub fn stream( 252 | &mut self, 253 | ) -> impl Stream)>> + '_ { 254 | try_stream! { 255 | self.update_header().await?; 256 | let mut eoi = self.header.snapshot_counter == 0; 257 | while !eoi { 258 | let snapshot_header = self.read_snapshot().await?; 259 | loop { 260 | let blob_header = self.read_blob_header().await?; 261 | if blob_header.is_last() { 262 | eoi = snapshot_header.id + 1 == self.header.snapshot_counter; 263 | break 264 | } 265 | let blob = self.read_blob(blob_header.blob_size).await?; 266 | yield (snapshot_header, blob_header, blob) 267 | } 268 | } 269 | } 270 | } 271 | } 272 | 273 | #[cfg(test)] 274 | mod tests { 275 | use std::path::Path; 276 | use tokio_stream::StreamExt; 277 | use super::*; 278 | 279 | pub struct DropFile<'a> { 280 | path: &'a Path 281 | } 282 | 283 | impl Drop for DropFile<'_> { 284 | fn drop(&mut self) { 285 | std::fs::remove_file(self.path).ok(); 286 | } 287 | } 288 | 289 | #[tokio::test] 290 | async fn journal_create_works() { 291 | let journal_path = tempfile::NamedTempFile::new().unwrap(); 292 | let journal_path = DropFile{ path: journal_path.path() }; 293 | let result = AsyncJournal::create(journal_path.path).await; 294 | assert!(result.is_ok()); 295 | let journal = result.unwrap(); 296 | assert_eq!(journal.blob_count, None); 297 | assert_eq!(journal.header, Header::default()); 298 | } 299 | 300 | 301 | #[tokio::test] 302 | async fn journal_add_and_commit_works() { 303 | let journal_path = tempfile::NamedTempFile::new().unwrap(); 304 | let journal_path = DropFile{ path: journal_path.path() }; 305 | let result = AsyncJournal::create(journal_path.path).await; 306 | assert!(result.is_ok()); 307 | let mut journal = result.unwrap(); 308 | assert_eq!(journal.blob_count, None); 309 | assert_eq!(journal.header, Header::default()); 310 | 311 | let result = journal.new_snapshot(10).await; 312 | assert!(result.is_ok()); 313 | let result = journal.new_blob(300, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]).await; 314 | assert!(result.is_ok()); 315 | assert_eq!(journal.blob_count, Some(1)); 316 | assert_eq!(journal.header, Header::default()); 317 | 318 | let result = journal.commit().await; 319 | assert!(result.is_ok()); 320 | assert_ne!(journal.header, Header::default()); 321 | } 322 | 323 | #[tokio::test] 324 | async fn journal_empty_stream() { 325 | let journal_path = tempfile::NamedTempFile::new().unwrap(); 326 | let journal_path = DropFile{ path: journal_path.path() }; 327 | let mut journal = AsyncJournal::create(journal_path.path).await.unwrap(); 328 | let stream = journal.stream().collect::>().await; 329 | assert!(stream.len() == 0, "{:#?}", stream); 330 | } 331 | } 332 | -------------------------------------------------------------------------------- /journal/src/error.rs: -------------------------------------------------------------------------------- 1 | //! Journal Error 2 | use serde_sqlite::Error as SerdeSqliteError; 3 | use std::collections::TryReserveError; 4 | use std::io::Error as IOError; 5 | 6 | #[derive(Debug)] 7 | pub enum Error { 8 | /// std::io::Error 9 | IOError(IOError), 10 | /// std::collections::TryReserveError 11 | TryReserveError(TryReserveError), 12 | /// serde_sqlite error 13 | SerdeSqliteError(SerdeSqliteError), 14 | /// attemt to add out of order snapshot 15 | OutOfOrderSnapshot { 16 | snapshot_id: u64, 17 | journal_snapshot_id: u64, 18 | }, 19 | /// Snapshot not started 20 | SnapshotNotStarted, 21 | /// Attemt to add out of order blob 22 | OutOfOrderBlob { 23 | blob_num: u32, 24 | blob_count: Option, 25 | }, 26 | /// Unexpected Journal Version 27 | UnexpectedJournalVersion { expected: u32, got: u32 }, 28 | } 29 | 30 | impl From for Error { 31 | fn from(e: IOError) -> Self { 32 | Self::IOError(e) 33 | } 34 | } 35 | 36 | impl From for Error { 37 | fn from(e: TryReserveError) -> Self { 38 | Self::TryReserveError(e) 39 | } 40 | } 41 | 42 | impl From for Error { 43 | fn from(e: SerdeSqliteError) -> Self { 44 | Self::SerdeSqliteError(e) 45 | } 46 | } 47 | 48 | impl std::fmt::Display for Error { 49 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 50 | write!(f, "{self:?}") 51 | } 52 | } 53 | 54 | impl std::error::Error for Error {} 55 | 56 | impl Error { 57 | /// Check if error caused by absense of journal 58 | pub fn journal_not_exists(&self) -> bool { 59 | match self { 60 | Self::IOError(e) => e.kind() == std::io::ErrorKind::NotFound, 61 | _ => false, 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /journal/src/journal.rs: -------------------------------------------------------------------------------- 1 | //! Journal (v1) 2 | 3 | use crate::error::Error; 4 | use block::{block, Block}; 5 | use serde::{Deserialize, Serialize}; 6 | use serde_sqlite::{from_reader, to_bytes}; 7 | use std::fs; 8 | use std::io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write}; 9 | use std::path; 10 | 11 | pub(crate) const MAGIC: u32 = 0x00907A70; 12 | pub(crate) const DEFAULT_BUFFER_SIZE: usize = 65536; 13 | 14 | type Result = std::result::Result; 15 | 16 | #[derive(Debug)] 17 | pub struct Journal 18 | where 19 | F: Read + Write + Seek, 20 | { 21 | /// Journal header 22 | header: Header, 23 | /// Wrapped into Fd reader/writer/seeker 24 | fd: Fd, BufReader>, 25 | /// snapshot page count 26 | blob_count: Option, 27 | /// Buffer size 28 | buffer_sz: usize, 29 | } 30 | 31 | #[derive(Debug)] 32 | enum Fd { 33 | Raw(F), 34 | Writer(W), 35 | Reader(R), 36 | // placeholder state to aid fd mode switching 37 | Nada, 38 | } 39 | 40 | impl Fd, BufReader> 41 | where 42 | F: Read + Write + Seek, 43 | { 44 | fn as_fd(&mut self) -> F { 45 | match std::mem::replace(self, Self::Nada) { 46 | Self::Reader(fd) => fd.into_inner(), 47 | Self::Writer(fd) => fd.into_parts().0, 48 | Self::Raw(fd) => fd, 49 | Self::Nada => unreachable!(), 50 | } 51 | } 52 | 53 | /// Swith Fd to 'raw' mode 54 | pub fn as_raw(&mut self) { 55 | let fd = self.as_fd(); 56 | let _ = std::mem::replace(self, Fd::Raw(fd)); 57 | } 58 | 59 | /// Switch Fd to buffered write mode 60 | pub fn as_writer(&mut self, buf_size: usize) { 61 | let fd = self.as_fd(); 62 | // FIXME: re-use buffer 63 | let _ = std::mem::replace(self, Fd::Writer(BufWriter::with_capacity(buf_size, fd))); 64 | } 65 | 66 | /// Switch Fd to buffered read mode 67 | pub fn as_reader(&mut self, buf_size: usize) { 68 | let fd = self.as_fd(); 69 | // FIXME: re-use buffer 70 | let _ = std::mem::replace(self, Fd::Reader(BufReader::with_capacity(buf_size, fd))); 71 | } 72 | } 73 | 74 | impl Write for Fd { 75 | fn write(&mut self, buf: &[u8]) -> std::io::Result { 76 | match self { 77 | Self::Raw(fd) => fd.write(buf), 78 | Self::Writer(fd) => fd.write(buf), 79 | Self::Reader(_) => Err(std::io::Error::new( 80 | std::io::ErrorKind::Other, 81 | "can't write into fd in read mode", 82 | )), 83 | Self::Nada => unreachable!(), 84 | } 85 | } 86 | 87 | fn flush(&mut self) -> std::io::Result<()> { 88 | match self { 89 | Self::Raw(fd) => fd.flush(), 90 | Self::Writer(fd) => fd.flush(), 91 | Self::Reader(_) => Err(std::io::Error::new( 92 | std::io::ErrorKind::Other, 93 | "can't flush fd in read mode", 94 | )), 95 | Self::Nada => unreachable!(), 96 | } 97 | } 98 | } 99 | 100 | impl Read for Fd { 101 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 102 | match self { 103 | Self::Raw(fd) => fd.read(buf), 104 | Self::Reader(fd) => fd.read(buf), 105 | Self::Writer(_) => Err(std::io::Error::new( 106 | std::io::ErrorKind::Other, 107 | "can't read from fd in write mode", 108 | )), 109 | Self::Nada => unreachable!(), 110 | } 111 | } 112 | } 113 | 114 | impl Seek for Fd { 115 | fn seek(&mut self, seek: SeekFrom) -> std::io::Result { 116 | match self { 117 | Self::Raw(fd) => fd.seek(seek), 118 | Self::Reader(fd) => fd.seek(seek), 119 | Self::Writer(fd) => fd.seek(seek), 120 | Self::Nada => unreachable!(), 121 | } 122 | } 123 | } 124 | 125 | impl Journal { 126 | /// Create new journal 127 | pub fn create>(p: P) -> Result { 128 | let fd = fs::OpenOptions::new() 129 | .create(true) 130 | .write(true) 131 | .read(true) 132 | .open(p.as_ref())?; 133 | Self::new(Header::default(), fd, None) 134 | } 135 | 136 | /// Try to instantiate journal from given path 137 | pub fn try_from>(p: P) -> Result { 138 | let mut fd = fs::OpenOptions::new().write(true).read(true).open(p)?; 139 | let header = Self::read_header(&mut fd)?; 140 | Ok(Self::from(header, fd, None)) 141 | } 142 | } 143 | 144 | impl Journal { 145 | /// Instantiate journal & force header write 146 | pub fn new(header: Header, mut fd: F, blob_count: Option) -> Result { 147 | Self::write_header(&mut fd, &header)?; 148 | Ok(Self::from(header, fd, blob_count)) 149 | } 150 | 151 | /// Instantiate journal 152 | pub fn from(header: Header, fd: F, blob_count: Option) -> Self { 153 | Self { 154 | header, 155 | fd: Fd::Raw(fd), 156 | blob_count, 157 | buffer_sz: DEFAULT_BUFFER_SIZE, 158 | } 159 | } 160 | 161 | /// Set buffer size 162 | pub fn set_buffer_size(&mut self, buffer_sz: usize) { 163 | self.buffer_sz = buffer_sz; 164 | } 165 | 166 | /// Get buffer size 167 | pub fn buffer_size(&self) -> usize { 168 | self.buffer_sz 169 | } 170 | 171 | /// Initiate new snapshot 172 | /// 173 | /// * update journal header to correctly setup offset 174 | /// * to initiate snapshot we seek to current end of the file (value stored in header) 175 | /// * switch fd to buffered mode 176 | /// * write snapshot header with current header counter number 177 | pub fn new_snapshot(&mut self, page_size: u32) -> Result<()> { 178 | if self.blob_count.is_some() { 179 | return Ok(()); 180 | } 181 | self.update_header()?; 182 | let snapshot_header = SnapshotHeader::new( 183 | self.header.snapshot_counter, 184 | chrono::Utc::now().timestamp_micros(), 185 | Some(page_size), 186 | ); 187 | self.write_snapshot(&snapshot_header) 188 | } 189 | 190 | /// Add new blob 191 | pub fn new_blob(&mut self, offset: u64, blob: &[u8]) -> Result<()> { 192 | let blob_num = match self.blob_count { 193 | Some(c) => c, 194 | None => return Err(Error::SnapshotNotStarted), 195 | }; 196 | let blob_header = BlobHeader::new(offset, blob_num, blob.len() as u32); 197 | self.add_blob(&blob_header, blob) 198 | } 199 | 200 | /// Add existing snapshot 201 | /// 202 | /// Re-syncs journal header 203 | pub fn add_snapshot(&mut self, snapshot_header: &SnapshotHeader) -> Result<()> { 204 | self.update_header()?; 205 | self.write_snapshot(snapshot_header) 206 | } 207 | 208 | /// Write snapshot to journal 209 | /// 210 | /// This function assumes journal header is up to date 211 | fn write_snapshot(&mut self, snapshot_header: &SnapshotHeader) -> Result<()> { 212 | if snapshot_header.id != self.header.snapshot_counter { 213 | return Err(Error::OutOfOrderSnapshot { 214 | snapshot_id: snapshot_header.id, 215 | journal_snapshot_id: self.header.snapshot_counter, 216 | }); 217 | } 218 | self.fd.seek(SeekFrom::Start(self.header.eof))?; 219 | self.fd.as_writer(self.buffer_sz); 220 | self.fd.write_all(&to_bytes(snapshot_header)?)?; 221 | self.blob_count = Some(0); 222 | Ok(()) 223 | } 224 | 225 | /// Add blob 226 | pub fn add_blob(&mut self, blob_header: &BlobHeader, blob: &[u8]) -> Result<()> { 227 | if Some(blob_header.blob_num) != self.blob_count { 228 | return Err(Error::OutOfOrderBlob { 229 | blob_num: blob_header.blob_num, 230 | blob_count: self.blob_count, 231 | }); 232 | } 233 | self.blob_count.as_mut().map(|x| { 234 | *x += 1; 235 | *x 236 | }); 237 | self.fd.write_all(&to_bytes(blob_header)?)?; 238 | self.fd.write_all(blob)?; 239 | Ok(()) 240 | } 241 | 242 | /// Commit snapshot 243 | /// 244 | /// * write final empty page to indicate end of snapshot 245 | /// * flush bufwriter (seek() on BufWriter will force flush) 246 | /// * write new header 247 | /// * flush bufwriter 248 | /// * switch fd back to raw mode 249 | pub fn commit(&mut self) -> Result<()> { 250 | if !self.snapshot_started() { 251 | return Ok(()); 252 | } 253 | // commit snapshot by writting final empty page 254 | self.fd.write_all(&to_bytes(&BlobHeader::last())?)?; 255 | self.blob_count = None; 256 | 257 | self.header.snapshot_counter += 1; 258 | self.header.eof = self.fd.stream_position()?; 259 | 260 | Self::write_header(&mut self.fd, &self.header)?; 261 | self.fd.flush()?; 262 | self.fd.as_raw(); 263 | Ok(()) 264 | } 265 | 266 | /// Get journal header 267 | pub fn get_header(&self) -> &Header { 268 | &self.header 269 | } 270 | 271 | /// Return current snapshot counter 272 | pub fn current_snapshot(&self) -> Option { 273 | match self.header.snapshot_counter { 274 | 0 => None, 275 | v => Some(v), 276 | } 277 | } 278 | 279 | /// Update journal header 280 | pub fn update_header(&mut self) -> Result<()> { 281 | self.fd.as_reader(self.buffer_sz); 282 | self.header = Self::read_header(&mut self.fd)?; 283 | Ok(()) 284 | } 285 | 286 | /// Read header from a given fd 287 | /// 288 | /// * seek to start of the file 289 | /// * read header 290 | fn read_header(fd: &mut R) -> Result
{ 291 | fd.rewind()?; 292 | from_reader(BufReader::new(fd)).map_err(Into::into) 293 | } 294 | 295 | /// Write header to a given fd 296 | /// 297 | /// * seek to start of the file 298 | /// * write header 299 | fn write_header(fd: &mut W, header: &Header) -> Result<()> { 300 | fd.rewind()?; 301 | fd.write_all(&to_bytes(header)?).map_err(Into::into) 302 | } 303 | 304 | /// Check if snapshot was already started 305 | fn snapshot_started(&self) -> bool { 306 | self.blob_count.is_some() 307 | } 308 | } 309 | 310 | #[derive(Debug)] 311 | pub struct IntoIter<'a, F = fs::File> 312 | where 313 | F: Read + Write + Seek, 314 | { 315 | journal: &'a mut Journal, 316 | current_snapshot: Option, 317 | initialized: bool, 318 | eoi: bool, 319 | } 320 | 321 | impl<'a, F: Write + Read + Seek> IntoIter<'a, F> { 322 | pub fn skip_snapshots( 323 | self, 324 | skip: u64, 325 | ) -> impl Iterator as Iterator>::Item> { 326 | self.filter(move |s| match s { 327 | Ok((ref snapshot_h, _, _)) => snapshot_h.id >= skip, 328 | _ => false, 329 | }) 330 | } 331 | } 332 | 333 | impl<'a, F: Read + Write + Seek> IntoIterator for &'a mut Journal { 334 | type IntoIter = IntoIter<'a, F>; 335 | type Item = ::Item; 336 | 337 | fn into_iter<'b>(self) -> Self::IntoIter { 338 | let eoi = self.header.snapshot_counter == 0; 339 | IntoIter { 340 | journal: self, 341 | initialized: false, 342 | current_snapshot: None, 343 | eoi, 344 | } 345 | } 346 | } 347 | 348 | impl<'a, F> Iterator for IntoIter<'a, F> 349 | where 350 | F: Read + Write + Seek, 351 | { 352 | type Item = Result<(SnapshotHeader, BlobHeader, Vec)>; 353 | 354 | fn next(&mut self) -> Option { 355 | if !self.initialized { 356 | if let Err(e) = self.journal.update_header() { 357 | self.eoi = true; 358 | return Some(Err(e)); 359 | } 360 | match self 361 | .journal 362 | .fd 363 | .seek(SeekFrom::Start(Header::block_size() as u64)) 364 | { 365 | Ok(_) => (), 366 | Err(e) => { 367 | self.eoi = true; 368 | return Some(Err(e.into())); 369 | } 370 | }; 371 | self.journal.fd.as_reader(self.journal.buffer_sz); 372 | self.initialized = true; 373 | } 374 | if self.eoi { 375 | return None; 376 | } 377 | if self.current_snapshot.is_none() { 378 | self.current_snapshot = match from_reader::(&mut self.journal.fd) { 379 | Ok(s) => Some(s), 380 | Err(e) => { 381 | self.eoi = true; 382 | return Some(Err(e.into())); 383 | } 384 | }; 385 | } 386 | let blob_header = match from_reader::(&mut self.journal.fd) { 387 | Ok(p) => p, 388 | Err(e) => { 389 | self.eoi = true; 390 | return Some(Err(e.into())); 391 | } 392 | }; 393 | if blob_header.is_last() { 394 | if self.current_snapshot.as_ref().unwrap().id + 1 395 | == self.journal.header.snapshot_counter 396 | { 397 | self.eoi = true; 398 | return None; 399 | } else { 400 | self.current_snapshot = None; 401 | return self.next(); 402 | } 403 | } 404 | let mut buf = vec![]; 405 | match buf.try_reserve(blob_header.blob_size as usize) { 406 | Ok(_) => (), 407 | Err(e) => { 408 | self.eoi = true; 409 | return Some(Err(e.into())); 410 | } 411 | } 412 | buf.resize(blob_header.blob_size as usize, 0); 413 | match self.journal.fd.read_exact(buf.as_mut_slice()) { 414 | Ok(_) => (), 415 | Err(e) => { 416 | self.eoi = true; 417 | return Some(Err(e.into())); 418 | } 419 | } 420 | Some(Ok(( 421 | *self.current_snapshot.as_ref().unwrap(), 422 | blob_header, 423 | buf, 424 | ))) 425 | } 426 | } 427 | 428 | /// Journal Header 429 | #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)] 430 | #[block(128)] 431 | pub struct Header { 432 | /// magic header 433 | pub magic: u32, 434 | /// journal version 435 | pub version: u32, 436 | /// operation counter 437 | pub snapshot_counter: u64, 438 | /// end of last snapshot 439 | pub eof: u64, 440 | } 441 | 442 | impl Default for Header { 443 | fn default() -> Self { 444 | Self { 445 | magic: MAGIC, 446 | version: 1, 447 | snapshot_counter: 0, 448 | eof: ::block_size() as u64, 449 | } 450 | } 451 | } 452 | 453 | /// Transaction Header 454 | #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] 455 | #[block(32)] 456 | pub struct SnapshotHeader { 457 | pub id: u64, 458 | pub timestamp: i64, 459 | #[serde( 460 | serialize_with = "serde_sqlite::se::none_as_zero", 461 | deserialize_with = "serde_sqlite::de::zero_as_none" 462 | )] 463 | pub page_size: Option, 464 | } 465 | 466 | impl SnapshotHeader { 467 | pub fn new(id: u64, timestamp: i64, page_size: Option) -> Self { 468 | Self { 469 | id, 470 | timestamp, 471 | page_size, 472 | } 473 | } 474 | } 475 | 476 | /// Blob Header 477 | #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] 478 | #[block(16)] 479 | pub struct BlobHeader { 480 | pub offset: u64, 481 | pub blob_num: u32, 482 | pub blob_size: u32, 483 | } 484 | 485 | impl BlobHeader { 486 | pub fn new(offset: u64, blob_num: u32, blob_size: u32) -> Self { 487 | Self { 488 | offset, 489 | blob_num, 490 | blob_size, 491 | } 492 | } 493 | 494 | // FIXME: should not be public 495 | pub fn last() -> Self { 496 | Self { 497 | offset: 0, 498 | blob_num: 0, 499 | blob_size: 0, 500 | } 501 | } 502 | 503 | // FIXME: should not be public 504 | pub fn is_last(&self) -> bool { 505 | self.offset == 0 && self.blob_num == 0 && self.blob_size == 0 506 | } 507 | } 508 | -------------------------------------------------------------------------------- /journal/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "async_bridge")] 2 | mod async_bridge; 3 | #[cfg(feature = "async")] 4 | mod async_journal; 5 | 6 | mod error; 7 | mod journal; 8 | mod stream; 9 | 10 | #[cfg(feature = "async_bridge")] 11 | pub use crate::async_bridge::{ 12 | AsyncReadJournalStream, AsyncReadJournalStreamHandle, AsyncWriteJournalStream, 13 | AsyncWriteJournalStreamHandle, 14 | }; 15 | 16 | #[cfg(feature = "async")] 17 | pub use crate::async_journal::AsyncJournal; 18 | 19 | pub use crate::error::Error; 20 | pub use crate::journal::{BlobHeader, Header, Journal, SnapshotHeader}; 21 | pub use crate::stream::{JournalVersion, Protocol, Stream}; 22 | -------------------------------------------------------------------------------- /journal/src/stream.rs: -------------------------------------------------------------------------------- 1 | //! Streaming protocol for journal 2 | 3 | use crate::error::Error as JournalError; 4 | use crate::journal::{BlobHeader, IntoIter, Journal, SnapshotHeader}; 5 | use block::{block, Block}; 6 | use serde::{Deserialize, Serialize}; 7 | use serde_sqlite::to_writer; 8 | use std::io::{BufRead, Cursor, Read, Seek, Write}; 9 | 10 | #[derive(Debug, Serialize, Deserialize, PartialEq)] 11 | #[block(0)] 12 | pub struct End {} 13 | 14 | #[derive(Debug, Serialize, Deserialize, PartialEq)] 15 | #[block] 16 | pub enum Protocol { 17 | SnapshotHeader(SnapshotHeader), 18 | BlobHeader(BlobHeader), 19 | EndOfStream(End), 20 | JournalVersion(JournalVersion), 21 | } 22 | 23 | impl std::fmt::Display for Protocol { 24 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 25 | match self { 26 | Self::SnapshotHeader(_) => write!(f, "SnapshotHeader"), 27 | Self::BlobHeader(_) => write!(f, "BlobHeader"), 28 | Self::EndOfStream(_) => write!(f, "EndOfStream"), 29 | Self::JournalVersion(v) => write!(f, "JournalVersion({})", v.version), 30 | } 31 | } 32 | } 33 | 34 | #[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Copy)] 35 | #[repr(transparent)] 36 | #[block(4)] 37 | pub struct JournalVersion { 38 | version: u32, 39 | } 40 | 41 | impl From for JournalVersion { 42 | fn from(version: u32) -> Self { 43 | Self { version } 44 | } 45 | } 46 | 47 | impl From for u32 { 48 | fn from(val: JournalVersion) -> Self { 49 | val.version 50 | } 51 | } 52 | 53 | impl From for Protocol { 54 | fn from(s: SnapshotHeader) -> Self { 55 | Self::SnapshotHeader(s) 56 | } 57 | } 58 | 59 | impl From for Protocol { 60 | fn from(p: BlobHeader) -> Self { 61 | Self::BlobHeader(p) 62 | } 63 | } 64 | 65 | impl From for Protocol { 66 | fn from(v: JournalVersion) -> Self { 67 | Self::JournalVersion(v) 68 | } 69 | } 70 | 71 | impl Protocol { 72 | fn end() -> Self { 73 | Self::EndOfStream(End {}) 74 | } 75 | } 76 | 77 | #[derive(Debug)] 78 | /// Converts iteration over journal into serialized Protocol stream 79 | pub struct Stream<'a, I: Iterator as Iterator>::Item>> { 80 | iter: I, 81 | version: u32, 82 | version_written: bool, 83 | buf: Vec, 84 | read: usize, 85 | cur_snapshot_id: Option, 86 | finished: bool, 87 | _marker: std::marker::PhantomData<&'a ()>, 88 | } 89 | 90 | // stream, which starts from 'scratch' 91 | impl<'a, F: Read + Write + Seek> From<&'a mut Journal> for Stream<'a, IntoIter<'a, F>> { 92 | fn from(journal: &'a mut Journal) -> Self { 93 | let version = journal.get_header().version; 94 | Stream::new(journal.into_iter(), version) 95 | } 96 | } 97 | 98 | // stream with any iterator with same Item type 99 | impl<'a, I: Iterator as Iterator>::Item>> From<(u32, I)> for Stream<'a, I> { 100 | fn from((version, iter): (u32, I)) -> Self { 101 | Stream::new(iter, version) 102 | } 103 | } 104 | 105 | impl<'a, I: Iterator as Iterator>::Item>> Stream<'a, I> { 106 | pub fn new(iter: I, version: u32) -> Self { 107 | Self { 108 | iter, 109 | version, 110 | version_written: false, 111 | buf: Vec::with_capacity(8192), 112 | read: 0, 113 | cur_snapshot_id: None, 114 | finished: false, 115 | _marker: std::marker::PhantomData, 116 | } 117 | } 118 | 119 | fn to_io_error>(e: E) -> std::io::Error { 120 | let e: JournalError = e.into(); 121 | // FIXME: does it make sense to unwrap error? 122 | match e { 123 | JournalError::IOError(e) => e, 124 | JournalError::SerdeSqliteError(serde_sqlite::Error::IoError(e)) => e, 125 | e => std::io::Error::new(std::io::ErrorKind::Other, e), 126 | } 127 | } 128 | 129 | /// resize own buffer before writting new data chunk into it 130 | fn resize_buf(&mut self, len: usize) { 131 | if self.buf.capacity() < len { 132 | self.buf.reserve(len); 133 | } 134 | // *safe*: 135 | // * reserved for at least 136 | // * used for writing data, no zeroing required 137 | unsafe { self.buf.set_len(len) }; 138 | } 139 | } 140 | 141 | impl<'a, I: Iterator as Iterator>::Item>> BufRead for Stream<'a, I> { 142 | fn fill_buf(&mut self) -> std::io::Result<&[u8]> { 143 | if self.read != self.buf.len() { 144 | return Ok(&self.buf[self.read..]); 145 | } else { 146 | self.read = 0; 147 | self.buf.clear(); 148 | } 149 | 150 | // always write version first 151 | if !self.version_written { 152 | let version: Protocol = JournalVersion::from(self.version).into(); 153 | self.resize_buf(version.iblock_size()); 154 | to_writer(self.buf.as_mut_slice(), &version).map_err(Self::to_io_error)?; 155 | self.version_written = true; 156 | return Ok(self.buf.as_slice()); 157 | } 158 | 159 | // body write 160 | match self.iter.next() { 161 | Some(Ok((snapshot_h, page_h, page))) => { 162 | let snapshot_id = snapshot_h.id; 163 | let snapshot_h: Protocol = snapshot_h.into(); 164 | let page_h: Protocol = page_h.into(); 165 | 166 | // max possible len for given item 167 | let total_len = snapshot_h.iblock_size() + page_h.iblock_size() + page.len(); 168 | self.resize_buf(total_len); 169 | 170 | let mut read_buf = Cursor::new(self.buf.as_mut_slice()); 171 | 172 | if self.cur_snapshot_id != Some(snapshot_id) { 173 | to_writer(&mut read_buf, &snapshot_h).map_err(Self::to_io_error)?; 174 | self.cur_snapshot_id = Some(snapshot_id) 175 | } 176 | to_writer(&mut read_buf, &page_h).map_err(Self::to_io_error)?; 177 | read_buf.write_all(page.as_slice())?; 178 | 179 | // real written value with according buffer resize 180 | let written = read_buf.position(); 181 | self.resize_buf(written as usize); 182 | } 183 | Some(Err(e)) => return Err(Self::to_io_error(e)), 184 | None if !self.finished => { 185 | self.finished = true; 186 | 187 | let eos = Protocol::end(); 188 | self.resize_buf(eos.iblock_size()); 189 | to_writer(self.buf.as_mut_slice(), &eos).map_err(Self::to_io_error)?; 190 | } 191 | None => (), 192 | }; 193 | Ok(self.buf.as_slice()) 194 | } 195 | 196 | fn consume(&mut self, amn: usize) { 197 | self.read += amn 198 | } 199 | } 200 | 201 | impl<'a, I: Iterator as Iterator>::Item>> Read for Stream<'a, I> { 202 | fn read(&mut self, write_buf: &mut [u8]) -> std::io::Result { 203 | let mut total = 0; 204 | let mut write_buf_len = write_buf.len(); 205 | let mut write_buf = Cursor::new(write_buf); 206 | loop { 207 | if write_buf_len == 0 { 208 | break; 209 | } 210 | let mut read_buf = self.fill_buf()?; 211 | if read_buf.is_empty() { 212 | break; 213 | } 214 | if read_buf.len() >= write_buf_len { 215 | read_buf = &read_buf[..write_buf_len]; 216 | }; 217 | let written = write_buf.write(read_buf)?; 218 | total += written; 219 | write_buf_len -= written; 220 | self.consume(written); 221 | } 222 | Ok(total) 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /libsqlite-sys/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "libsqlite-sys" 3 | version = "0.1.0" 4 | edition = "2021" 5 | build = "build.rs" 6 | 7 | [features] 8 | default = [] 9 | 10 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 11 | [dependencies] 12 | 13 | 14 | [build-dependencies] 15 | bindgen = "0.65" 16 | pkg-config = "0.3" 17 | 18 | 19 | [dev-dependencies] 20 | quickcheck = "1" 21 | libc = "0.2" 22 | -------------------------------------------------------------------------------- /libsqlite-sys/README.md: -------------------------------------------------------------------------------- 1 | ## libsqlite-sys 2 | Bindings to sqlite3, relies on system-installed sqlite3. 3 | 4 | ## Doc 5 | ```bash 6 | cargo doc --open 7 | ``` 8 | 9 | ## Access to generated bindings 10 | ```bash 11 | cargo expand 12 | ``` 13 | or 14 | ```bash 15 | cat `find ../target/debug/build/ -name 'bindings.rs' -print` 16 | ``` 17 | -------------------------------------------------------------------------------- /libsqlite-sys/build.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | #[derive(Debug)] 4 | pub struct ParseCallbacks {} 5 | 6 | impl bindgen::callbacks::ParseCallbacks for ParseCallbacks { 7 | fn int_macro(&self, name: &str, _value: i64) -> Option { 8 | if name.starts_with("SQLITE_") { 9 | return Some(bindgen::callbacks::IntKind::Int); 10 | } 11 | None 12 | } 13 | } 14 | 15 | fn main() { 16 | let out_dir = std::env::var("OUT_DIR").unwrap(); 17 | let out_path = Path::new(&out_dir).join("bindings.rs"); 18 | 19 | println!("cargo:rerun-if-changed=wrapper.h"); 20 | println!("cargo:rerun-if-changed=build.rs"); 21 | 22 | let pkg_conf = pkg_config::Config::new() 23 | .print_system_cflags(false) 24 | .print_system_libs(false) 25 | .cargo_metadata(false) 26 | .probe("sqlite3") 27 | .expect("installation of sqlite3 required"); 28 | 29 | let include_paths = pkg_conf 30 | .include_paths 31 | .iter() 32 | .map(|p| format!("-I{}", p.to_str().expect(""))) 33 | .collect::>(); 34 | 35 | let bindings = bindgen::Builder::default() 36 | .header("wrapper.h") 37 | .clang_args(include_paths.as_slice()) 38 | .use_core() 39 | .ctypes_prefix("core::ffi") 40 | .parse_callbacks(Box::new(ParseCallbacks {})) 41 | .generate() 42 | .expect("failed to generate bindings"); 43 | 44 | bindings 45 | .write_to_file(out_path) 46 | .expect("Could'n write bindings"); 47 | } 48 | -------------------------------------------------------------------------------- /libsqlite-sys/src/alloc.rs: -------------------------------------------------------------------------------- 1 | use core::alloc::{GlobalAlloc, Layout}; 2 | use core::ffi::c_void; 3 | 4 | pub struct SQLiteAllocator { 5 | pub malloc: unsafe extern "C" fn(u64) -> *mut c_void, 6 | pub free: unsafe extern "C" fn(*mut c_void), 7 | } 8 | 9 | const PTR_ISIZE: isize = core::mem::size_of::() as isize; 10 | const PTR_USIZE: usize = core::mem::size_of::(); 11 | 12 | unsafe impl GlobalAlloc for SQLiteAllocator { 13 | // v--------------------------| 14 | // ------------------------------------------------------------ 15 | // | padding | ptr | aligned mem block | 16 | // ------------------------------------------------------------ 17 | unsafe fn alloc(&self, layout: Layout) -> *mut u8 { 18 | let align = layout.align(); 19 | let header_size = match align { 20 | align if align <= PTR_USIZE => PTR_USIZE, 21 | align => align, 22 | }; 23 | let size = header_size + layout.size(); 24 | let block = (self.malloc)(size as u64) as *mut u8; 25 | if block.is_null() { 26 | return block; 27 | } 28 | let padding = match (block as usize) % layout.align() { 29 | 0 => header_size, 30 | v => align - v, 31 | } as isize; 32 | *(block.offset(padding - PTR_ISIZE) as *mut usize) = block as usize; 33 | block.offset(padding) 34 | } 35 | 36 | unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { 37 | // address of tag 38 | let addr = (ptr as usize - PTR_USIZE) as *mut usize; 39 | // address of original block 40 | let block = (*addr) as *mut c_void; 41 | (self.free)(block) 42 | } 43 | } 44 | 45 | /// Setup SQLITE3_API3 and SQLITE_ALLOCATOR symbols 46 | #[macro_export] 47 | macro_rules! setup { 48 | () => { 49 | static mut SQLITE3_API: *mut libsqlite_sys::ffi::sqlite3_api_routines = 50 | core::ptr::null_mut(); 51 | 52 | // stub 53 | unsafe extern "C" fn _libsqlite3_stub_malloc(_: u64) -> *mut core::ffi::c_void { 54 | panic!("libsqlite3 not initialized"); 55 | } 56 | 57 | unsafe extern "C" fn _libsqlite_stub_free(_: *mut core::ffi::c_void) { 58 | panic!("libsqlite3 not initialized"); 59 | } 60 | 61 | #[global_allocator] 62 | static mut SQLITE_ALLOCATOR: libsqlite_sys::alloc::SQLiteAllocator = 63 | libsqlite_sys::alloc::SQLiteAllocator { 64 | malloc: _libsqlite3_stub_malloc, 65 | free: _libsqlite_stub_free, 66 | }; 67 | 68 | use core::alloc::GlobalAlloc; 69 | pub unsafe extern "C" fn deallocate(ptr: *mut core::ffi::c_void) { 70 | SQLITE_ALLOCATOR.dealloc( 71 | ptr as *mut u8, 72 | core::alloc::Layout::from_size_align_unchecked(0, 0), 73 | ) 74 | } 75 | }; 76 | } 77 | 78 | /// Init SQLITE3_API and redefine GLOBAL_ALLOC functions to point to sqlite3_malloc64/sqlite3_free 79 | #[macro_export] 80 | macro_rules! init { 81 | ($global:expr) => { 82 | SQLITE3_API = $global; 83 | 84 | SQLITE_ALLOCATOR.malloc = (*$global).malloc64.unwrap(); 85 | SQLITE_ALLOCATOR.free = (*$global).free.unwrap(); 86 | }; 87 | } 88 | -------------------------------------------------------------------------------- /libsqlite-sys/src/ffi.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::all)] 2 | 3 | include!(concat!(env!("OUT_DIR"), "/bindings.rs")); 4 | -------------------------------------------------------------------------------- /libsqlite-sys/src/iter.rs: -------------------------------------------------------------------------------- 1 | //! Iterator over sqlite argc/argv pair 2 | use core::ffi::c_int; 3 | use core::marker::PhantomData; 4 | 5 | /// Iterator over sqlite pointers 6 | #[derive(Debug)] 7 | pub struct PtrIter<'a, T> { 8 | offset: usize, 9 | len: usize, 10 | ptr: *const T, 11 | _marker: PhantomData<&'a ()>, 12 | } 13 | 14 | impl<'a, T> Iterator for PtrIter<'a, T> 15 | where 16 | T: Copy, 17 | { 18 | type Item = T; 19 | 20 | fn next(&mut self) -> Option { 21 | if self.offset >= self.len { 22 | return None; 23 | } 24 | let item = unsafe { *self.ptr.add(self.offset) }; 25 | self.offset += 1; 26 | Some(item) 27 | } 28 | } 29 | 30 | impl<'a, T> PtrIter<'a, T> { 31 | pub fn new(len: c_int, ptr: *const T) -> Self { 32 | Self { 33 | offset: 0, 34 | len: len as usize, 35 | ptr, 36 | _marker: PhantomData, 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /libsqlite-sys/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | pub mod alloc; 3 | #[allow(non_upper_case_globals)] 4 | #[allow(non_camel_case_types)] 5 | #[allow(non_snake_case)] 6 | pub mod ffi; 7 | pub mod iter; 8 | pub mod sqlite_value; 9 | pub mod util; 10 | pub mod vtab; 11 | -------------------------------------------------------------------------------- /libsqlite-sys/src/sqlite_value.rs: -------------------------------------------------------------------------------- 1 | //! Wrapper for sqlite3_value struct 2 | use crate::{ffi, iter}; 3 | use core::ffi::c_int; 4 | 5 | #[derive(Debug, PartialEq)] 6 | pub enum SqliteValue<'a> { 7 | I64(i64), 8 | Double(f64), 9 | Blob(&'a [u8]), 10 | Text(&'a str), 11 | Null, 12 | } 13 | 14 | impl<'a> SqliteValue<'a> { 15 | pub fn is_null(&self) -> bool { 16 | matches!(self, Self::Null) 17 | } 18 | } 19 | 20 | /// Iterator over *mut *mut ffi::sqlite3_value 21 | #[derive(Debug)] 22 | pub struct SqliteValueIter<'a> { 23 | iter: iter::PtrIter<'a, *mut ffi::sqlite3_value>, 24 | api: *mut ffi::sqlite3_api_routines, 25 | } 26 | 27 | impl<'a> SqliteValueIter<'a> { 28 | pub fn new( 29 | argc: c_int, 30 | value: *mut *mut ffi::sqlite3_value, 31 | api: *mut ffi::sqlite3_api_routines, 32 | ) -> Self { 33 | Self { 34 | iter: iter::PtrIter::new(argc, value), 35 | api, 36 | } 37 | } 38 | } 39 | 40 | impl<'a> Iterator for SqliteValueIter<'a> { 41 | type Item = SqliteValue<'a>; 42 | 43 | fn next(&mut self) -> Option { 44 | let value = match self.iter.next() { 45 | Some(v) => v, 46 | None => return None, 47 | }; 48 | let value = unsafe { 49 | match { (*self.api).value_type.unwrap()(value) } { 50 | ffi::SQLITE_TEXT => { 51 | let (text, len) = ( 52 | (*self.api).value_text.unwrap()(value), 53 | (*self.api).value_bytes.unwrap()(value) as usize, 54 | ); 55 | SqliteValue::Text(core::str::from_utf8_unchecked(core::slice::from_raw_parts( 56 | text, len, 57 | ))) 58 | } 59 | ffi::SQLITE_INTEGER => SqliteValue::I64((*self.api).value_int64.unwrap()(value)), 60 | ffi::SQLITE_FLOAT => SqliteValue::Double((*self.api).value_double.unwrap()(value)), 61 | ffi::SQLITE_NULL => SqliteValue::Null, 62 | ffi::SQLITE_BLOB => { 63 | let blob = core::slice::from_raw_parts( 64 | (*self.api).value_blob.unwrap()(value) as *const u8, 65 | (*self.api).value_bytes.unwrap()(value) as usize, 66 | ); 67 | SqliteValue::Blob(blob) 68 | } 69 | _ => unreachable!(), 70 | } 71 | }; 72 | Some(value) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /libsqlite-sys/src/util.rs: -------------------------------------------------------------------------------- 1 | use core::ffi::{c_char, c_int, CStr}; 2 | 3 | /// Iterator over Argv params 4 | pub struct ArgvIter<'a> { 5 | cur: c_int, 6 | argc: c_int, 7 | argv: *mut *const c_char, 8 | _marker: core::marker::PhantomData<&'a ()>, 9 | } 10 | 11 | impl<'a> ArgvIter<'a> { 12 | pub fn new(argc: c_int, argv: *const *const c_char) -> Self { 13 | Self { 14 | cur: 0, 15 | _marker: core::marker::PhantomData, 16 | argc, 17 | argv: argv as *mut _, 18 | } 19 | } 20 | } 21 | 22 | impl<'a> Iterator for ArgvIter<'a> { 23 | type Item = &'a CStr; 24 | 25 | fn next(&mut self) -> Option { 26 | if self.argc <= self.cur { 27 | return None; 28 | } 29 | let cstr = Some(unsafe { CStr::from_ptr(*self.argv) }); 30 | self.cur += 1; 31 | self.argv = unsafe { self.argv.offset(1) }; 32 | cstr 33 | } 34 | } 35 | 36 | #[macro_export] 37 | macro_rules! c_str( 38 | ($e:expr) => { 39 | concat!($e, "\0").as_ptr() as *const core::ffi::c_char 40 | } 41 | ); 42 | -------------------------------------------------------------------------------- /libsqlite-sys/src/vtab.rs: -------------------------------------------------------------------------------- 1 | //! Various helpers around sqlite VTables 2 | use crate::ffi; 3 | use crate::sqlite_value::{SqliteValue, SqliteValueIter}; 4 | use core::ffi::c_int; 5 | 6 | #[derive(Debug)] 7 | pub enum UpdateType<'a> { 8 | Delete { 9 | row_id: SqliteValue<'a>, 10 | }, 11 | Insert { 12 | row_id: SqliteValue<'a>, 13 | columns: SqliteValueIter<'a>, 14 | }, 15 | Update { 16 | row_id: SqliteValue<'a>, 17 | columns: SqliteValueIter<'a>, 18 | }, 19 | } 20 | 21 | // https://www.sqlite.org/vtab.html#xupdate 22 | impl<'a> 23 | From<( 24 | c_int, 25 | *mut *mut ffi::sqlite3_value, 26 | *mut ffi::sqlite3_api_routines, 27 | )> for UpdateType<'a> 28 | { 29 | fn from( 30 | (argc, argv, api): ( 31 | c_int, 32 | *mut *mut ffi::sqlite3_value, 33 | *mut ffi::sqlite3_api_routines, 34 | ), 35 | ) -> Self { 36 | let mut iter = SqliteValueIter::new(argc, argv, api); 37 | let first = iter.next(); 38 | let second = iter.next(); 39 | match argc { 40 | 1 if first.is_some() => Self::Delete { 41 | row_id: first.unwrap(), 42 | }, 43 | v if v > 1 && first.is_some() && first.as_ref().unwrap().is_null() => Self::Insert { 44 | row_id: first.unwrap(), 45 | columns: iter, 46 | }, 47 | v if v > 1 48 | && first.is_some() 49 | && !first.as_ref().unwrap().is_null() 50 | && first == second => 51 | { 52 | Self::Insert { 53 | row_id: first.unwrap(), 54 | columns: iter, 55 | } 56 | } 57 | v if v > 1 58 | && first.is_some() 59 | && !first.as_ref().unwrap().is_null() 60 | && first != second => 61 | { 62 | Self::Update { 63 | row_id: first.unwrap(), 64 | columns: iter, 65 | } 66 | } 67 | _ => unreachable!(), 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /libsqlite-sys/tests/allocator_tests.rs: -------------------------------------------------------------------------------- 1 | use quickcheck::{Arbitrary, Gen, TestResult}; 2 | use std::alloc; 3 | 4 | #[global_allocator] 5 | static mut SQLITE3_ALLOCATOR: libsqlite_sys::alloc::SQLiteAllocator = 6 | libsqlite_sys::alloc::SQLiteAllocator { 7 | malloc: _test_malloc64_wrap, 8 | free: libc::free, 9 | }; 10 | 11 | unsafe extern "C" fn _test_malloc64_wrap(size: u64) -> *mut core::ffi::c_void { 12 | libc::malloc(size as libc::size_t) 13 | } 14 | 15 | #[derive(Debug, Clone)] 16 | struct TestAlloc { 17 | size: usize, 18 | layout: usize, 19 | } 20 | 21 | impl Arbitrary for TestAlloc { 22 | fn arbitrary(gen: &mut Gen) -> Self { 23 | let layouts = (0..10).map(|shf| 1 << shf).collect::>(); 24 | let size = usize::arbitrary(gen) % 0x0001_0000; 25 | let layout = *layouts.get(usize::arbitrary(gen) % layouts.len()).unwrap(); 26 | TestAlloc { size, layout } 27 | } 28 | } 29 | 30 | const PTR_SIZE: isize = std::mem::size_of::() as isize; 31 | 32 | #[test] 33 | // goal of this function to check SQLite Allocator: 34 | // 1. allocated addresses should be aligned 35 | // 2. allocated block should contain requested amount of bytes 36 | // 3. tag should be written properly in initial allocated block 37 | fn test_allocator() { 38 | fn check(allocs: Vec) -> TestResult { 39 | let mut allocated = allocs 40 | .into_iter() 41 | .map(|t| { 42 | let layout = alloc::Layout::from_size_align(t.size, t.layout).unwrap(); 43 | let result = unsafe { alloc::alloc(layout) }; 44 | 45 | // check allocation is not null 46 | assert!(!result.is_null()); 47 | 48 | // check if returned address alligned 49 | assert_eq!(result as usize % t.layout, 0); 50 | 51 | // address is always aligned by at least pointer size 52 | if t.layout < std::mem::size_of::() { 53 | assert_eq!(result as usize % (PTR_SIZE as usize), 0); 54 | } 55 | 56 | // find original block address by reading address of result - usize 57 | let real_block_addr: usize = unsafe { 58 | let addr = ((result as usize) - PTR_SIZE as usize) as *mut usize; 59 | *addr 60 | }; 61 | // +---------------------------------------- 62 | // | real_addr | padding | header | result | 63 | // +---------------------------------------- 64 | // ^___________________| 65 | // real_addr - pointer to block provided by 'malloc' 66 | // padding - calculated offset for block alignment 67 | // header - contains value of real_addr, to call 'free' properly 68 | // result - some value calcuated on top of real_add + layout 69 | 70 | // real block address is always smaller than resulting address, since at least we need 71 | // to store header, which contains real address of the allocated block 72 | assert!(real_block_addr < result as usize); 73 | 74 | // SQLiteAllocator asks for block of size + layout in order to properly align final block 75 | // resulting address should always between (real_address, real_address + layout] 76 | assert!(real_block_addr + t.layout.max(PTR_SIZE as usize) >= result as usize); 77 | 78 | // smoke test? cast allocated block to slice of bytes, zero all stuff 79 | let slice: &mut [u8] = unsafe { std::slice::from_raw_parts_mut(result, t.size) }; 80 | slice.iter_mut().for_each(|x| *x = 0); 81 | assert_eq!(slice.iter().map(|x| (*x) as usize).sum::(), 0); 82 | 83 | (result, layout) 84 | }) 85 | .collect::>(); 86 | 87 | // deallocate 88 | while let Some((addr, layout)) = allocated.pop() { 89 | unsafe { alloc::dealloc(addr, layout) }; 90 | } 91 | TestResult::from_bool(true) 92 | } 93 | quickcheck::quickcheck(check as fn(Vec) -> TestResult); 94 | } 95 | -------------------------------------------------------------------------------- /libsqlite-sys/wrapper.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | -------------------------------------------------------------------------------- /mycelite/.gitignore: -------------------------------------------------------------------------------- 1 | vfs_j 2 | test* 3 | -------------------------------------------------------------------------------- /mycelite/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "mycelite" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | crate-type = ["cdylib", "rlib"] 8 | test = false 9 | 10 | [features] 11 | default = ["replicator"] 12 | replicator = ["dep:ureq", "dep:base64"] 13 | 14 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 15 | 16 | [dependencies] 17 | libsqlite-sys = { path = "../libsqlite-sys" } 18 | journal = { path = "../journal" } 19 | utils = { path = "../utils" } 20 | page_parser = { path = "../page_parser" } 21 | serde_sqlite = { path = "../serde_sqlite" } 22 | once_cell = "1" 23 | 24 | # replicator 25 | ureq = { version = "2.5", optional = true } 26 | base64 = { version = "0.21", optional = true } 27 | 28 | # config 29 | toml = "0.7" 30 | 31 | -------------------------------------------------------------------------------- /mycelite/src/config.rs: -------------------------------------------------------------------------------- 1 | //! mycelite configuration 2 | use crate::{deallocate, SQLITE3_API}; 3 | use libsqlite_sys::{c_str, ffi, sqlite_value::SqliteValue, vtab::UpdateType}; 4 | use once_cell::sync::Lazy; 5 | use std::collections::BTreeMap; 6 | use std::ffi::{c_char, c_int, c_void, CStr, CString}; 7 | use std::mem; 8 | use std::sync::{Arc, Mutex}; 9 | 10 | static CONFIG_REGISTRY: Lazy>>>> = 11 | Lazy::new(|| Mutex::new(BTreeMap::new())); 12 | 13 | #[derive(Debug, Copy, Clone)] 14 | pub(crate) struct ConfigRegistry {} 15 | 16 | impl ConfigRegistry { 17 | pub fn new() -> Self { 18 | Self {} 19 | } 20 | 21 | pub fn register_config(self, database_path: &str) { 22 | let mut map = CONFIG_REGISTRY.lock().unwrap(); 23 | if map.get(database_path).is_some() { 24 | return; 25 | } 26 | let mut config = Config::new(database_path); 27 | // FIXME: error is swallowed 28 | config.read().ok(); 29 | map.insert(database_path.into(), Arc::new(Mutex::new(config))); 30 | } 31 | 32 | #[allow(dead_code)] 33 | pub fn unregister_config(self, database_path: &str) { 34 | CONFIG_REGISTRY 35 | .lock() 36 | .map(|mut map| map.remove(database_path)) 37 | .unwrap(); 38 | } 39 | 40 | pub fn get(self, database_path: &str) -> Arc> { 41 | self.register_config(database_path); 42 | CONFIG_REGISTRY 43 | .lock() 44 | .map(|map| Arc::clone(map.get(database_path).unwrap())) 45 | .unwrap() 46 | } 47 | } 48 | 49 | #[derive(Debug, Clone)] 50 | pub(crate) struct Config { 51 | path: String, 52 | state: BTreeMap, 53 | } 54 | 55 | impl Config { 56 | pub fn new>(database_path: P) -> Self { 57 | let path = { 58 | let mut path: String = database_path.into(); 59 | path.push_str("-mycelite-config"); 60 | path 61 | }; 62 | let mut s = Self { 63 | path, 64 | state: BTreeMap::new(), 65 | }; 66 | s.insert("endpoint", "https://us-east-1.mycelial.com"); 67 | s 68 | } 69 | 70 | pub fn get(&self, key: &str) -> Option<&str> { 71 | self.state.get(key).map(|s| s.as_str()) 72 | } 73 | 74 | fn insert(&mut self, key: &str, value: &str) { 75 | if Self::allowed_keys().contains(&key) { 76 | self.state.insert(key.to_string(), value.to_string()); 77 | } 78 | } 79 | 80 | fn delete(&mut self, pos: usize) { 81 | if let Some(key) = Self::allowed_keys().get(pos) { 82 | self.state.remove(*key); 83 | }; 84 | } 85 | 86 | fn write(&mut self) -> Result<(), Box> { 87 | if !self.path.is_empty() { 88 | let value = toml::to_string(&self.state)?; 89 | std::fs::write(self.path.as_str(), value)?; 90 | } 91 | Ok(()) 92 | } 93 | 94 | fn read(&mut self) -> Result<(), Box> { 95 | let value = match std::fs::read_to_string(self.path.as_str()) { 96 | Ok(value) => value, 97 | Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), 98 | Err(e) => return Err(e.into()), 99 | }; 100 | let map = toml::from_str::>(&value)?; 101 | map.into_iter().for_each(|(key, value)| { 102 | self.state.insert(key, value); 103 | }); 104 | Ok(()) 105 | } 106 | 107 | fn allowed_keys() -> &'static [&'static str] { 108 | &["client_id", "domain", "endpoint", "secret"] 109 | } 110 | 111 | fn rows(&self) -> impl Iterator { 112 | self.state.iter().map(|(k, v)| { 113 | ( 114 | Self::allowed_keys().iter().position(|r| r == k).unwrap() as i64, 115 | k.as_str(), 116 | v.as_str(), 117 | ) 118 | }) 119 | } 120 | } 121 | 122 | #[repr(C)] 123 | struct VTab { 124 | vtab: ffi::sqlite3_vtab, 125 | database_path: String, 126 | } 127 | 128 | impl VTab { 129 | unsafe fn new(database_path: String) -> Self { 130 | Self { 131 | vtab: mem::zeroed(), 132 | database_path, 133 | } 134 | } 135 | 136 | unsafe fn as_mut(ptr: *mut ffi::sqlite3_vtab) -> &'static mut Self { 137 | &mut *ptr.cast::() 138 | } 139 | 140 | unsafe fn from_raw(ptr: *mut ffi::sqlite3_vtab) -> Box { 141 | Box::from_raw(ptr.cast::()) 142 | } 143 | 144 | fn into_raw(self) -> *mut ffi::sqlite3_vtab { 145 | Box::into_raw(Box::new(self)).cast() 146 | } 147 | } 148 | 149 | #[repr(C)] 150 | struct VTabCursor { 151 | cur: ffi::sqlite3_vtab_cursor, 152 | offset: usize, 153 | rows: Vec<(i64, String, String)>, 154 | } 155 | 156 | impl VTabCursor { 157 | unsafe fn new(config_path: &str) -> Self { 158 | let config = ConfigRegistry::new().get(config_path); 159 | let rows = config 160 | .lock() 161 | .unwrap() 162 | .rows() 163 | .map(|(rowid, k, v)| (rowid, k.to_owned(), v.to_owned())) 164 | .collect(); 165 | Self { 166 | cur: mem::zeroed(), 167 | offset: 0, 168 | rows, 169 | } 170 | } 171 | 172 | unsafe fn as_mut(ptr: *mut ffi::sqlite3_vtab_cursor) -> &'static mut Self { 173 | &mut *ptr.cast::() 174 | } 175 | 176 | unsafe fn from_raw(ptr: *mut ffi::sqlite3_vtab_cursor) -> Box { 177 | Box::from_raw(ptr.cast::()) 178 | } 179 | 180 | fn into_raw(self) -> *mut ffi::sqlite3_vtab_cursor { 181 | Box::into_raw(Box::new(self)).cast() 182 | } 183 | } 184 | 185 | unsafe extern "C" fn x_connect( 186 | db: *mut ffi::sqlite3, 187 | _p_aux: *mut c_void, 188 | _argc: c_int, 189 | _argv: *const *const c_char, 190 | pp_vtab: *mut *mut ffi::sqlite3_vtab, 191 | _err: *mut *mut c_char, 192 | ) -> c_int { 193 | let rc = (*SQLITE3_API).declare_vtab.unwrap()( 194 | db, 195 | c_str!("CREATE TABLE mycelite_config(key text, value text)"), 196 | ); 197 | if rc != ffi::SQLITE_OK { 198 | return rc; 199 | }; 200 | let database_path = CStr::from_ptr((*SQLITE3_API).db_filename.unwrap()(db, c_str!("main"))) 201 | .to_string_lossy() 202 | .to_string(); 203 | *pp_vtab = VTab::new(database_path).into_raw(); 204 | ffi::SQLITE_OK 205 | } 206 | 207 | unsafe extern "C" fn x_best_index( 208 | _p_vtab: *mut ffi::sqlite3_vtab, 209 | _index_info: *mut ffi::sqlite3_index_info, 210 | ) -> c_int { 211 | ffi::SQLITE_OK 212 | } 213 | 214 | unsafe extern "C" fn x_disconnect(p_vtab: *mut ffi::sqlite3_vtab) -> c_int { 215 | VTab::from_raw(p_vtab); 216 | ffi::SQLITE_OK 217 | } 218 | 219 | unsafe extern "C" fn x_open( 220 | p_vtab: *mut ffi::sqlite3_vtab, 221 | pp_cursor: *mut *mut ffi::sqlite3_vtab_cursor, 222 | ) -> c_int { 223 | let vtab = VTab::as_mut(p_vtab); 224 | *pp_cursor = VTabCursor::new(vtab.database_path.as_str()).into_raw(); 225 | ffi::SQLITE_OK 226 | } 227 | 228 | unsafe extern "C" fn x_close(p_cursor: *mut ffi::sqlite3_vtab_cursor) -> c_int { 229 | VTabCursor::from_raw(p_cursor); 230 | ffi::SQLITE_OK 231 | } 232 | 233 | unsafe extern "C" fn x_filter( 234 | p_cursor: *mut ffi::sqlite3_vtab_cursor, 235 | _idx_num: c_int, 236 | _idx_str: *const c_char, 237 | _argc: c_int, 238 | _argv: *mut *mut ffi::sqlite3_value, 239 | ) -> c_int { 240 | let mut cursor = VTabCursor::as_mut(p_cursor); 241 | cursor.offset = 0; 242 | ffi::SQLITE_OK 243 | } 244 | 245 | unsafe extern "C" fn x_next(p_cursor: *mut ffi::sqlite3_vtab_cursor) -> c_int { 246 | let cursor = VTabCursor::as_mut(p_cursor); 247 | cursor.offset += 1; 248 | ffi::SQLITE_OK 249 | } 250 | 251 | unsafe extern "C" fn x_column( 252 | p_cursor: *mut ffi::sqlite3_vtab_cursor, 253 | p_ctx: *mut ffi::sqlite3_context, 254 | n: c_int, 255 | ) -> c_int { 256 | let cursor = VTabCursor::as_mut(p_cursor); 257 | let row = match cursor.rows.get(cursor.offset) { 258 | Some(row) => row, 259 | None => return ffi::SQLITE_ERROR, 260 | }; 261 | let value = match n { 262 | 0 => row.1.clone(), 263 | 1 => row.2.clone(), 264 | _ => return ffi::SQLITE_ERROR, 265 | }; 266 | let len = value.len(); 267 | let cs = CString::from_vec_unchecked(value.into_bytes()); 268 | (*SQLITE3_API).result_text.unwrap()(p_ctx, cs.into_raw(), len as c_int, Some(deallocate)); 269 | ffi::SQLITE_OK 270 | } 271 | 272 | unsafe extern "C" fn x_eof(p_cursor: *mut ffi::sqlite3_vtab_cursor) -> c_int { 273 | let cursor = VTabCursor::as_mut(p_cursor); 274 | (cursor.offset >= cursor.rows.len()) as c_int 275 | } 276 | 277 | unsafe extern "C" fn x_rowid( 278 | p_cursor: *mut ffi::sqlite3_vtab_cursor, 279 | p_rowid: *mut ffi::sqlite_int64, 280 | ) -> c_int { 281 | let cursor = VTabCursor::as_mut(p_cursor); 282 | *p_rowid = cursor.rows.get(cursor.offset).unwrap().0; 283 | ffi::SQLITE_OK 284 | } 285 | 286 | unsafe extern "C" fn x_update( 287 | vtab: *mut ffi::sqlite3_vtab, 288 | argc: c_int, 289 | value: *mut *mut ffi::sqlite3_value, 290 | _p_rowid: *mut ffi::sqlite3_int64, 291 | ) -> c_int { 292 | let vtab = VTab::as_mut(vtab); 293 | let config = ConfigRegistry::new().get(vtab.database_path.as_str()); 294 | let mut config = config.lock().unwrap(); 295 | match UpdateType::from((argc, value, SQLITE3_API)) { 296 | UpdateType::Delete { 297 | row_id: SqliteValue::I64(row_id), 298 | } => config.delete(row_id as usize), 299 | UpdateType::Update { mut columns, .. } => match (columns.next(), columns.next()) { 300 | (Some(SqliteValue::Text(key)), Some(SqliteValue::Text(value))) => { 301 | config.insert(key, value) 302 | } 303 | _ => { 304 | return ffi::SQLITE_MISUSE; 305 | } 306 | }, 307 | UpdateType::Insert { mut columns, .. } => match (columns.next(), columns.next()) { 308 | (Some(SqliteValue::Text(key)), Some(SqliteValue::Text(value))) => { 309 | config.insert(key, value) 310 | } 311 | _ => { 312 | return ffi::SQLITE_MISUSE; 313 | } 314 | }, 315 | _ => return ffi::SQLITE_MISUSE, 316 | } 317 | ffi::SQLITE_OK 318 | } 319 | 320 | unsafe extern "C" fn x_begin(_p_vtab: *mut ffi::sqlite3_vtab) -> c_int { 321 | ffi::SQLITE_OK 322 | } 323 | 324 | unsafe extern "C" fn x_sync(p_vtab: *mut ffi::sqlite3_vtab) -> c_int { 325 | let vtab = VTab::as_mut(p_vtab); 326 | let config = ConfigRegistry::new().get(vtab.database_path.as_str()); 327 | let mut config = config.lock().unwrap(); 328 | if config.write().is_err() { 329 | return ffi::SQLITE_ERROR; 330 | }; 331 | ffi::SQLITE_OK 332 | } 333 | 334 | unsafe extern "C" fn x_commit(_p_vtab: *mut ffi::sqlite3_vtab) -> c_int { 335 | ffi::SQLITE_OK 336 | } 337 | 338 | unsafe extern "C" fn x_rollback(_p_vtab: *mut ffi::sqlite3_vtab) -> c_int { 339 | ffi::SQLITE_OK 340 | } 341 | 342 | pub unsafe fn init(db: *mut ffi::sqlite3, _err: *mut *mut c_char) -> c_int { 343 | static CONFIG_VTABLE: ffi::sqlite3_module = ffi::sqlite3_module { 344 | iVersion: 0, 345 | xCreate: None, 346 | xDestroy: None, 347 | xConnect: Some(x_connect), 348 | xDisconnect: Some(x_disconnect), 349 | xBestIndex: Some(x_best_index), 350 | xOpen: Some(x_open), 351 | xClose: Some(x_close), 352 | xFilter: Some(x_filter), 353 | xNext: Some(x_next), 354 | xEof: Some(x_eof), 355 | xColumn: Some(x_column), 356 | xRowid: Some(x_rowid), 357 | xUpdate: Some(x_update), 358 | xBegin: Some(x_begin), 359 | xSync: Some(x_sync), 360 | xCommit: Some(x_commit), 361 | xRollback: Some(x_rollback), 362 | xFindFunction: None, 363 | xRename: None, 364 | xSavepoint: None, 365 | xRelease: None, 366 | xRollbackTo: None, 367 | xShadowName: None, 368 | }; 369 | 370 | (*SQLITE3_API).create_module.unwrap()( 371 | db, 372 | c_str!("mycelite_config"), 373 | &CONFIG_VTABLE, 374 | std::ptr::null_mut() as *mut c_void, 375 | ) 376 | } 377 | -------------------------------------------------------------------------------- /mycelite/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::missing_safety_doc)] 2 | 3 | mod config; 4 | mod replicator; 5 | mod vfs; 6 | use libsqlite_sys::ffi; 7 | use once_cell::sync::OnceCell; 8 | use std::ffi::{c_char, c_int}; 9 | 10 | struct DefaultVfs(*mut ffi::sqlite3_vfs); 11 | 12 | unsafe impl Sync for DefaultVfs {} 13 | unsafe impl Send for DefaultVfs {} 14 | 15 | // The pointer to a vfs that sqlite3 uses by default depending on the OS 16 | // - Linux/MacOS - unix 17 | // - Windows - win32 18 | // 19 | // The pointer is used to properly setup the Mycelite VFS. 20 | static DEFAULT_VFS: OnceCell = OnceCell::new(); 21 | 22 | libsqlite_sys::setup!(); 23 | 24 | #[no_mangle] 25 | pub unsafe fn sqlite3_mycelite_init( 26 | db: *mut ffi::sqlite3, 27 | err: *mut *mut c_char, 28 | api: *mut ffi::sqlite3_api_routines, 29 | ) -> c_int { 30 | mycelite_writer(db, err, api); 31 | mycelite_reader(db, err, api); 32 | ffi::SQLITE_OK_LOAD_PERMANENTLY 33 | } 34 | 35 | #[no_mangle] 36 | pub unsafe fn mycelite_reader( 37 | _db: *mut ffi::sqlite3, 38 | _err: *mut *mut c_char, 39 | api: *mut ffi::sqlite3_api_routines, 40 | ) -> c_int { 41 | libsqlite_sys::init!(api); 42 | let default_vfs = (*SQLITE3_API).vfs_find.unwrap()(std::ptr::null_mut()); 43 | DEFAULT_VFS.set(DefaultVfs(default_vfs)).ok(); 44 | 45 | vfs::MclVFSReader.init(DEFAULT_VFS.get_unchecked().0); 46 | (*SQLITE3_API).vfs_register.unwrap()(vfs::MclVFSReader.as_base(), 1); 47 | ffi::SQLITE_OK_LOAD_PERMANENTLY 48 | } 49 | 50 | #[no_mangle] 51 | pub unsafe fn mycelite_writer( 52 | _db: *mut ffi::sqlite3, 53 | _err: *mut *mut c_char, 54 | api: *mut ffi::sqlite3_api_routines, 55 | ) -> c_int { 56 | libsqlite_sys::init!(api); 57 | let default_vfs = (*SQLITE3_API).vfs_find.unwrap()(std::ptr::null_mut()); 58 | DEFAULT_VFS.set(DefaultVfs(default_vfs)).ok(); 59 | 60 | vfs::MclVFSWriter.init(DEFAULT_VFS.get_unchecked().0); 61 | (*SQLITE3_API).vfs_register.unwrap()(vfs::MclVFSWriter.as_base(), 1); 62 | ffi::SQLITE_OK_LOAD_PERMANENTLY 63 | } 64 | 65 | #[no_mangle] 66 | pub unsafe fn mycelite_config( 67 | db: *mut ffi::sqlite3, 68 | err: *mut *mut c_char, 69 | api: *mut ffi::sqlite3_api_routines, 70 | ) -> c_int { 71 | libsqlite_sys::init!(api); 72 | 73 | // init configuration vtab for given db handle 74 | config::init(db, err) 75 | } 76 | -------------------------------------------------------------------------------- /mycelite/src/replicator/http_replicator.rs: -------------------------------------------------------------------------------- 1 | //! Replicator prototype 2 | //! 3 | //! ** For demo use only! ** 4 | 5 | use crate::config::{Config, ConfigRegistry}; 6 | use base64::engine::{general_purpose::STANDARD as BASE64, Engine}; 7 | use journal::{Journal, Protocol, Stream}; 8 | use serde_sqlite::de; 9 | use std::io::{Seek, SeekFrom, Write}; 10 | use std::path::Path; 11 | use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender}; 12 | use std::sync::{Arc, Mutex}; 13 | use std::thread::JoinHandle; 14 | 15 | enum Message { 16 | /// New snapshot added locally 17 | NewLocalSnapshot, 18 | /// Notification from ReplicatorHandle about closed DB File 19 | Quit, 20 | } 21 | 22 | pub struct Replicator { 23 | database_path: String, 24 | journal: Journal, 25 | read_only: bool, 26 | lock: Arc>, 27 | config: Arc>, 28 | } 29 | 30 | impl Replicator { 31 | pub fn new>( 32 | journal_path: P, 33 | database_path: String, 34 | read_only: bool, 35 | lock: Arc>, 36 | ) -> Self { 37 | let config = ConfigRegistry::new().get(database_path.as_str()); 38 | Self { 39 | journal: Journal::try_from(journal_path).unwrap(), 40 | database_path, 41 | read_only, 42 | lock, 43 | config, 44 | } 45 | } 46 | 47 | pub fn spawn(mut self) -> ReplicatorHandle { 48 | let (tx, mut rx) = channel(); 49 | let local_h = Some(std::thread::spawn(move || self.enter_loop(&mut rx))); 50 | ReplicatorHandle::new(tx, local_h) 51 | } 52 | 53 | /// local loop 54 | /// 55 | /// listens for notifications pulls/pushes snapshots, restores underlying database to latest 56 | /// snapshot 57 | fn enter_loop(&mut self, rx: &mut Receiver) { 58 | loop { 59 | match self.read_only { 60 | true => { 61 | match self.maybe_pull_snapshots() { 62 | Ok((last, new)) if last < new => { 63 | self.restore_latest_snapshot().ok(); 64 | } 65 | Ok(_) => (), 66 | Err(_e) => (), 67 | }; 68 | } 69 | false => { 70 | self.maybe_push_snapshots().ok(); 71 | } 72 | } 73 | match rx.recv_timeout(std::time::Duration::from_secs(5)) { 74 | Err(RecvTimeoutError::Disconnected) => return, 75 | Err(RecvTimeoutError::Timeout) => (), 76 | Ok(Message::Quit) => return, 77 | Ok(Message::NewLocalSnapshot) => (), 78 | }; 79 | } 80 | } 81 | 82 | /// Push local snapshots, if any 83 | fn maybe_push_snapshots(&mut self) -> Result<(), Box> { 84 | // FIXME: unwrap 85 | self.journal.update_header().unwrap(); 86 | let local_snapshot_id = match self.journal.current_snapshot() { 87 | None => return Ok(()), 88 | Some(v) => v, 89 | }; 90 | let url = match self.get_url() { 91 | Some(url) => url, 92 | None => return Ok(()), 93 | }; 94 | // snapshot push always requires authorization (for now) 95 | let client_id = self.get_key("client_id"); 96 | let secret = self.get_key("secret"); 97 | if client_id.is_none() || secret.is_none() { 98 | return Ok(()); 99 | }; 100 | let remote_snapshot_id = match self.get_backend_current_snapshot( 101 | &url, 102 | client_id.as_deref(), 103 | secret.as_deref(), 104 | ) { 105 | Ok(Some(v)) if v >= local_snapshot_id => { 106 | return Ok(()); 107 | } 108 | Ok(Some(v)) => v, 109 | Ok(None) => 0, 110 | Err(_) => return Err("error".into()), 111 | }; 112 | 113 | let mut req = ureq::post(&url); 114 | if let Some(b) = self.get_basic_auth_header(client_id.as_deref(), secret.as_deref()) { 115 | req = req.set("Authorization", &b) 116 | } 117 | 118 | let version = self.journal.get_header().version; 119 | let stream = Stream::from(( 120 | version, 121 | self.journal.into_iter().skip_snapshots(remote_snapshot_id), 122 | )); 123 | 124 | // FIXME: status code are not checked 125 | req.send(stream)?; 126 | Ok(()) 127 | } 128 | 129 | /// Pulls remove snapshots, if any 130 | fn maybe_pull_snapshots( 131 | &mut self, 132 | ) -> Result<(Option, Option), Box> { 133 | let local_snapshot_id = self.journal.current_snapshot(); 134 | let url = match self.get_url() { 135 | Some(url) => url, 136 | None => return Ok((local_snapshot_id, local_snapshot_id)), 137 | }; 138 | 139 | let client_id = self.get_key("client_id"); 140 | let secret = self.get_key("secret"); 141 | 142 | match self.get_backend_current_snapshot(&url, client_id.as_deref(), secret.as_deref())? { 143 | Some(v) if local_snapshot_id < Some(v) => (), 144 | v => return Ok((local_snapshot_id, v)), 145 | }; 146 | 147 | let mut req = 148 | ureq::get(&url).query("snapshot-id", &local_snapshot_id.unwrap_or(0).to_string()); 149 | 150 | if let Some(b) = self.get_basic_auth_header(client_id.as_deref(), secret.as_deref()) { 151 | req = req.set("Authorization", &b) 152 | } 153 | let res = req.call()?; 154 | 155 | let mut reader = res.into_reader(); 156 | 157 | match de::from_reader::(&mut reader)? { 158 | Protocol::JournalVersion(v) if v == 1_u32.into() => (), 159 | Protocol::JournalVersion(v) => { 160 | return Err(format!("unexpected journal version: {v:?}").into()) 161 | } 162 | _ => return Err("expected version header".into()), 163 | }; 164 | loop { 165 | match de::from_reader::(&mut reader)? { 166 | Protocol::SnapshotHeader(snapshot_header) => { 167 | self.journal.commit()?; 168 | self.journal.add_snapshot(&snapshot_header)? 169 | } 170 | Protocol::BlobHeader(blob_header) => { 171 | let mut blob = vec![0; blob_header.blob_size as usize]; 172 | reader.read_exact(blob.as_mut_slice())?; 173 | self.journal.add_blob(&blob_header, blob.as_slice())?; 174 | } 175 | Protocol::EndOfStream(_) => { 176 | self.journal.commit()?; 177 | break; 178 | } 179 | Protocol::JournalVersion(_) => return Err("version header was not expected".into()), 180 | } 181 | } 182 | Ok((local_snapshot_id, self.journal.current_snapshot())) 183 | } 184 | 185 | // FIXME: move to journal API 186 | // FIXME: snapshot is recovered from scratch each time 187 | fn restore_latest_snapshot(&mut self) -> Result<(), Box> { 188 | let lock = self.lock.lock().map_err(|_e| "failed to lock")?; 189 | let mut output = std::io::BufWriter::with_capacity( 190 | 0x0010_0000, 191 | std::fs::OpenOptions::new() 192 | .create(true) 193 | .write(true) 194 | .open(&self.database_path)?, 195 | ); 196 | for data in self.journal.into_iter() { 197 | let (_snapshot_header, page_header, page) = data?; 198 | output.seek(SeekFrom::Start(page_header.offset))?; 199 | output.write_all(&page)?; 200 | } 201 | drop(lock); 202 | Ok(()) 203 | } 204 | 205 | /// Fetch last snapshot id seen by sync backend 206 | fn get_backend_current_snapshot( 207 | &self, 208 | url: &str, 209 | client_id: Option<&str>, 210 | secret: Option<&str>, 211 | ) -> Result, Box> { 212 | let mut req = ureq::head(url).timeout(std::time::Duration::from_secs(5)); 213 | 214 | if let Some(b) = self.get_basic_auth_header(client_id, secret) { 215 | req = req.set("Authorization", &b) 216 | } 217 | let res = req.call()?; 218 | 219 | match res.header("x-snapshot-id") { 220 | Some(value) if value.is_empty() => Ok(None), 221 | Some(value) => Ok(Some(value.parse()?)), 222 | None => Err("backend didn't return x-snapshot-id".into()), 223 | } 224 | } 225 | 226 | fn get_key(&self, key: &str) -> Option { 227 | self.config.lock().unwrap().get(key).map(|s| s.to_owned()) 228 | } 229 | 230 | fn get_url(&self) -> Option { 231 | if let (Some(endpoint), Some(domain)) = (self.get_key("endpoint"), self.get_key("domain")) { 232 | return Some(format!("{endpoint}/domain/{domain}")); 233 | } 234 | None 235 | } 236 | 237 | fn get_basic_auth_header( 238 | &self, 239 | client_id: Option<&str>, 240 | secret: Option<&str>, 241 | ) -> Option { 242 | if let (Some(client_id), Some(secret)) = (client_id, secret) { 243 | return Some(format!( 244 | "Basic {}", 245 | BASE64.encode(format!("{client_id}:{secret}")) 246 | )); 247 | } else { 248 | None 249 | } 250 | } 251 | } 252 | 253 | #[derive(Debug)] 254 | pub struct ReplicatorHandle { 255 | tx: Sender, 256 | handle: Option>, 257 | } 258 | 259 | impl Drop for ReplicatorHandle { 260 | fn drop(&mut self) { 261 | self.tx.send(Message::Quit).ok(); 262 | self.handle.take().map(|h| h.join()); 263 | } 264 | } 265 | 266 | impl ReplicatorHandle { 267 | fn new(tx: Sender, handle: Option>) -> Self { 268 | Self { tx, handle } 269 | } 270 | 271 | pub fn new_snapshot(&mut self) { 272 | self.tx.send(Message::NewLocalSnapshot).ok(); 273 | } 274 | } 275 | -------------------------------------------------------------------------------- /mycelite/src/replicator/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg_attr(not(feature = "replicator"), path = "noop_replicator.rs")] 2 | #[cfg_attr(feature = "replicator", path = "http_replicator.rs")] 3 | mod replicator_impl; 4 | 5 | pub use replicator_impl::*; 6 | -------------------------------------------------------------------------------- /mycelite/src/replicator/noop_replicator.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | use std::sync::{Arc, Mutex}; 3 | 4 | pub struct Replicator {} 5 | 6 | impl Replicator { 7 | pub fn new>( 8 | _journal_path: P, 9 | _database_path: String, 10 | _read_only: bool, 11 | _lock: Arc>, 12 | ) -> Self { 13 | Self {} 14 | } 15 | 16 | pub fn spawn(self) -> ReplicatorHandle { 17 | ReplicatorHandle {} 18 | } 19 | } 20 | 21 | pub struct ReplicatorHandle {} 22 | 23 | impl ReplicatorHandle { 24 | pub fn new_snapshot(&self) {} 25 | } 26 | -------------------------------------------------------------------------------- /mycelite/src/vfs.rs: -------------------------------------------------------------------------------- 1 | use crate::replicator; 2 | use journal::Journal; 3 | use libsqlite_sys::c_str; 4 | use libsqlite_sys::ffi; 5 | use std::ffi::{c_char, c_int, c_void, CStr}; 6 | use std::mem; 7 | use std::ptr; 8 | use std::sync::{Arc, Mutex, MutexGuard}; 9 | 10 | macro_rules! vfs_vtable { 11 | ($name:expr) => { 12 | ffi::sqlite3_vfs { 13 | iVersion: 2, 14 | // initialized on extention load 15 | szOsFile: 0, 16 | // initialized on extention load 17 | mxPathname: 0, 18 | pNext: ptr::null_mut(), 19 | zName: c_str!($name), 20 | pAppData: ptr::null_mut(), 21 | xOpen: Some(mvfs_open), 22 | xDelete: Some(mvfs_delete), 23 | xAccess: Some(mvfs_access), 24 | xFullPathname: Some(mvfs_full_pathname), 25 | xDlOpen: Some(mvfs_dlopen), 26 | xDlError: Some(mvfs_dlerror), 27 | xDlSym: Some(mvfs_dlsym), 28 | xDlClose: Some(mvfs_dlclose), 29 | xRandomness: Some(mvfs_randomness), 30 | xSleep: Some(mvfs_sleep), 31 | xCurrentTime: Some(mvfs_current_time), 32 | xGetLastError: Some(mvfs_get_last_error), 33 | xCurrentTimeInt64: Some(mvfs_current_time_i64), 34 | xSetSystemCall: None, 35 | xGetSystemCall: None, 36 | xNextSystemCall: None, 37 | } 38 | }; 39 | } 40 | 41 | #[repr(C)] 42 | #[derive(Debug)] 43 | pub struct MclVFS { 44 | base: ffi::sqlite3_vfs, 45 | read_only: bool, 46 | // initialized on extention load 47 | real: *mut ffi::sqlite3_vfs, 48 | } 49 | 50 | #[no_mangle] 51 | #[used] 52 | pub static mut MclVFSReader: MclVFS = MclVFS { 53 | base: vfs_vtable!("mycelite_reader"), 54 | read_only: true, 55 | // initialized on extention load 56 | real: ptr::null_mut(), 57 | }; 58 | 59 | #[no_mangle] 60 | #[used] 61 | pub static mut MclVFSWriter: MclVFS = MclVFS { 62 | base: vfs_vtable!("mycelite_writer"), 63 | read_only: false, 64 | // initialized on extention load 65 | real: ptr::null_mut(), 66 | }; 67 | 68 | impl MclVFS { 69 | /// Initialite MclVFS as a proxy to *real* VFS 70 | pub unsafe fn init(&mut self, real: *mut ffi::sqlite3_vfs) { 71 | // init VFS only once 72 | if self.real.is_null() { 73 | self.real = real; 74 | self.base.szOsFile = mem::size_of::() as c_int + (*real).szOsFile; 75 | self.base.mxPathname = (*real).mxPathname; 76 | } 77 | } 78 | 79 | /// Get pointer to base vfs struct 80 | pub unsafe fn as_base(&mut self) -> *mut ffi::sqlite3_vfs { 81 | &mut self.base 82 | } 83 | 84 | /// Get pointer to real vfs 85 | unsafe fn as_real_ptr(base: *mut ffi::sqlite3_vfs) -> *mut ffi::sqlite3_vfs { 86 | (*base.cast::()).real 87 | } 88 | 89 | /// return reference to real vfs 90 | /// 91 | /// reference allow vfs function calls 92 | unsafe fn as_real_ref(base: *mut ffi::sqlite3_vfs) -> &'static mut ffi::sqlite3_vfs { 93 | &mut *Self::as_real_ptr(base) 94 | } 95 | 96 | unsafe fn from_raw_ptr(base: *mut ffi::sqlite3_vfs) -> &'static mut Self { 97 | &mut *(base.cast::()) 98 | } 99 | } 100 | 101 | #[repr(C)] 102 | struct MclVFSFile { 103 | base: ffi::sqlite3_file, 104 | journal: Option>, 105 | read_only: bool, 106 | replicator: Option>, 107 | mutex: Option>>>, 108 | mutex_guard: Option>>, 109 | vfs: *mut ffi::sqlite3_vfs, 110 | real: ffi::sqlite3_file, 111 | } 112 | 113 | impl MclVFSFile { 114 | /// init VFS File 115 | unsafe fn init(&mut self, vfs: *mut ffi::sqlite3_vfs) { 116 | self.vfs = vfs; 117 | self.read_only = MclVFS::from_raw_ptr(vfs).read_only; 118 | self.mutex = Some(mem::ManuallyDrop::new(Arc::new(Mutex::new(())))); 119 | self.mutex_guard = None 120 | } 121 | 122 | /// downcast pfile ptr to MclVFSFile struct ptr 123 | unsafe fn from_ptr(pfile: *mut ffi::sqlite3_file) -> &'static mut Self { 124 | &mut *(pfile as *mut MclVFSFile) 125 | } 126 | 127 | /// bootstrap journal 128 | /// 129 | /// happens only once on journal creation. 130 | fn bootstrap_journal( 131 | &self, 132 | journal: &mut Journal, 133 | database_path: &str, 134 | ) -> Result<(), Box> { 135 | let db = page_parser::Database::new(database_path); 136 | let iter = match db.into_raw_page_iter() { 137 | Ok(iter) => iter, 138 | Err(e) => { 139 | if let Some(err) = e.downcast_ref::() { 140 | if err.kind() == std::io::ErrorKind::NotFound { 141 | // no database file - no need in bootstraping 142 | return Ok(()); 143 | } 144 | } 145 | return Err(e); 146 | } 147 | }; 148 | for res in iter { 149 | let (offset, page) = res?; 150 | let page = page.as_slice(); 151 | // it's ok to call snapshot multiple times, if it's already started, it's noop 152 | journal.new_snapshot(page.len() as u32)?; 153 | for (diff_offset, blob) in utils::get_diff(page, &[]) { 154 | journal.new_blob(offset + diff_offset as u64, blob)?; 155 | } 156 | } 157 | Ok(journal.commit()?) 158 | } 159 | 160 | fn setup_journal( 161 | &mut self, 162 | flags: c_int, 163 | zname: *const c_char, 164 | ) -> Result<(), Box> { 165 | if flags & ffi::SQLITE_OPEN_MAIN_DB == 0 { 166 | self.journal = None; 167 | self.replicator = None; 168 | return Ok(()); 169 | } 170 | 171 | let database_path = unsafe { CStr::from_ptr(zname) }.to_str()?.to_owned(); 172 | let journal_path = { 173 | let mut s = database_path.clone(); 174 | s.push_str("-mycelial"); 175 | s 176 | }; 177 | let (journal, bootstrapped) = match Journal::try_from(&journal_path) { 178 | Ok(j) => (j, false), 179 | Err(e) if e.journal_not_exists() => { 180 | let mut journal = Journal::create(&journal_path)?; 181 | self.bootstrap_journal(&mut journal, &database_path)?; 182 | (journal, true) 183 | } 184 | Err(e) => return Err(e.into()), 185 | }; 186 | self.journal = Some(mem::ManuallyDrop::new(journal)); 187 | 188 | let lock = Arc::clone(self.mutex.as_ref().unwrap()); 189 | self.replicator = Some(mem::ManuallyDrop::new( 190 | replicator::Replicator::new(&journal_path, database_path, self.read_only, lock).spawn(), 191 | )); 192 | 193 | if bootstrapped { 194 | if let Some(r) = self.replicator.as_mut() { 195 | r.new_snapshot(); 196 | } 197 | } 198 | Ok(()) 199 | } 200 | 201 | fn lock(&'static mut self) { 202 | if self.mutex_guard.is_some() { 203 | return; 204 | }; 205 | if let Some(mutex) = self.mutex.as_ref() { 206 | self.mutex_guard = Some(mem::ManuallyDrop::new(mutex.lock().unwrap())) 207 | }; 208 | } 209 | 210 | fn unlock(&mut self) { 211 | if self.mutex_guard.is_some() { 212 | self.mutex_guard.take().map(mem::ManuallyDrop::into_inner); 213 | } 214 | } 215 | } 216 | 217 | // VFS methods 218 | 219 | unsafe extern "C" fn mvfs_open( 220 | vfs: *mut ffi::sqlite3_vfs, 221 | zname: *const c_char, 222 | file: *mut ffi::sqlite3_file, 223 | flags: c_int, 224 | p_out_flags: *mut c_int, 225 | ) -> c_int { 226 | let file = MclVFSFile::from_ptr(file); 227 | file.init(vfs); 228 | if file.setup_journal(flags, zname).is_err() { 229 | return ffi::SQLITE_ERROR; 230 | } 231 | file.base.pMethods = &MclVFSIO as *const _; 232 | MclVFS::as_real_ref(vfs).xOpen.unwrap()( 233 | MclVFS::as_real_ptr(vfs), 234 | zname, 235 | &mut file.real, 236 | flags, 237 | p_out_flags, 238 | ) 239 | } 240 | 241 | unsafe extern "C" fn mvfs_delete( 242 | vfs: *mut ffi::sqlite3_vfs, 243 | zname: *const c_char, 244 | sync_dir: c_int, 245 | ) -> c_int { 246 | MclVFS::as_real_ref(vfs).xDelete.unwrap()(MclVFS::as_real_ptr(vfs), zname, sync_dir) 247 | } 248 | 249 | unsafe extern "C" fn mvfs_access( 250 | vfs: *mut ffi::sqlite3_vfs, 251 | zname: *const c_char, 252 | flags: c_int, 253 | p_res_out: *mut c_int, 254 | ) -> c_int { 255 | MclVFS::as_real_ref(vfs).xAccess.unwrap()(MclVFS::as_real_ptr(vfs), zname, flags, p_res_out) 256 | } 257 | 258 | unsafe extern "C" fn mvfs_full_pathname( 259 | vfs: *mut ffi::sqlite3_vfs, 260 | zname: *const c_char, 261 | n_out: c_int, 262 | z_out: *mut c_char, 263 | ) -> c_int { 264 | MclVFS::as_real_ref(vfs).xFullPathname.unwrap()(MclVFS::as_real_ptr(vfs), zname, n_out, z_out) 265 | } 266 | 267 | unsafe extern "C" fn mvfs_dlopen( 268 | vfs: *mut ffi::sqlite3_vfs, 269 | zfilename: *const c_char, 270 | ) -> *mut c_void { 271 | MclVFS::as_real_ref(vfs).xDlOpen.unwrap()(MclVFS::as_real_ptr(vfs), zfilename) 272 | } 273 | 274 | unsafe extern "C" fn mvfs_dlerror( 275 | vfs: *mut ffi::sqlite3_vfs, 276 | n_byte: c_int, 277 | z_err_msg: *mut c_char, 278 | ) { 279 | MclVFS::as_real_ref(vfs).xDlError.unwrap()(MclVFS::as_real_ptr(vfs), n_byte, z_err_msg) 280 | } 281 | 282 | unsafe extern "C" fn mvfs_dlsym( 283 | vfs: *mut ffi::sqlite3_vfs, 284 | p: *mut c_void, 285 | z_symbol: *const c_char, 286 | ) -> Option 287 | { 288 | MclVFS::as_real_ref(vfs).xDlSym.unwrap()(MclVFS::as_real_ptr(vfs), p, z_symbol) 289 | } 290 | 291 | unsafe extern "C" fn mvfs_dlclose(vfs: *mut ffi::sqlite3_vfs, p_handle: *mut c_void) { 292 | MclVFS::as_real_ref(vfs).xDlClose.unwrap()(MclVFS::as_real_ptr(vfs), p_handle); 293 | } 294 | 295 | unsafe extern "C" fn mvfs_randomness( 296 | vfs: *mut ffi::sqlite3_vfs, 297 | n_byte: c_int, 298 | z_buf_out: *mut c_char, 299 | ) -> c_int { 300 | MclVFS::as_real_ref(vfs).xRandomness.unwrap()(MclVFS::as_real_ptr(vfs), n_byte, z_buf_out) 301 | } 302 | 303 | unsafe extern "C" fn mvfs_sleep(vfs: *mut ffi::sqlite3_vfs, micros: c_int) -> c_int { 304 | MclVFS::as_real_ref(vfs).xSleep.unwrap()(MclVFS::as_real_ptr(vfs), micros) 305 | } 306 | 307 | unsafe extern "C" fn mvfs_current_time(vfs: *mut ffi::sqlite3_vfs, p_timeout: *mut f64) -> c_int { 308 | MclVFS::as_real_ref(vfs).xCurrentTime.unwrap()(MclVFS::as_real_ptr(vfs), p_timeout) 309 | } 310 | 311 | unsafe extern "C" fn mvfs_get_last_error( 312 | vfs: *mut ffi::sqlite3_vfs, 313 | a: c_int, 314 | b: *mut c_char, 315 | ) -> c_int { 316 | MclVFS::as_real_ref(vfs).xGetLastError.unwrap()(MclVFS::as_real_ptr(vfs), a, b) 317 | } 318 | 319 | unsafe extern "C" fn mvfs_current_time_i64( 320 | vfs: *mut ffi::sqlite3_vfs, 321 | p: *mut ffi::sqlite3_int64, 322 | ) -> c_int { 323 | MclVFS::as_real_ref(vfs).xCurrentTimeInt64.unwrap()(MclVFS::as_real_ptr(vfs), p) 324 | } 325 | 326 | #[no_mangle] 327 | #[used] 328 | static MclVFSIO: ffi::sqlite3_io_methods = ffi::sqlite3_io_methods { 329 | iVersion: 1, 330 | xClose: Some(mvfs_io_close), 331 | xRead: Some(mvfs_io_read), 332 | xWrite: Some(mvfs_io_write), 333 | xTruncate: Some(mvfs_io_truncate), 334 | xSync: Some(mvfs_io_sync), 335 | xFileSize: Some(mvfs_io_file_size), 336 | xLock: Some(mvfs_io_lock), 337 | xUnlock: Some(mvfs_io_unlock), 338 | xCheckReservedLock: Some(mvfs_io_check_reserved_lock), 339 | xFileControl: Some(mvfs_io_file_control), 340 | xSectorSize: Some(mvfs_io_sector_size), 341 | xDeviceCharacteristics: Some(mvfs_io_device_characteristics), 342 | 343 | xShmMap: None, 344 | xShmLock: None, 345 | xShmBarrier: None, 346 | xShmUnmap: None, 347 | xFetch: None, 348 | xUnfetch: None, 349 | }; 350 | 351 | unsafe extern "C" fn mvfs_io_close(pfile: *mut ffi::sqlite3_file) -> c_int { 352 | let file = MclVFSFile::from_ptr(pfile); 353 | file.unlock(); 354 | file.mutex.take().map(mem::ManuallyDrop::into_inner); 355 | file.journal.take().map(mem::ManuallyDrop::into_inner); 356 | file.replicator.take().map(mem::ManuallyDrop::into_inner); 357 | (*file.real.pMethods).xClose.unwrap()(&mut file.real) 358 | } 359 | 360 | unsafe extern "C" fn mvfs_io_read( 361 | pfile: *mut ffi::sqlite3_file, 362 | buf: *mut c_void, 363 | amt: c_int, 364 | offset: ffi::sqlite_int64, 365 | ) -> c_int { 366 | let file = MclVFSFile::from_ptr(pfile); 367 | (*file.real.pMethods).xRead.unwrap()(&mut file.real, buf, amt, offset) 368 | } 369 | 370 | unsafe extern "C" fn mvfs_io_write( 371 | pfile: *mut ffi::sqlite3_file, 372 | buf: *const c_void, 373 | amt: c_int, 374 | offset: ffi::sqlite_int64, 375 | ) -> c_int { 376 | let file = MclVFSFile::from_ptr(pfile); 377 | if file.read_only && file.journal.is_some() { 378 | // FIXME: this is a hack for reader-only and virtual table 379 | if offset == 0 { 380 | return ffi::SQLITE_OK; 381 | } else { 382 | return ffi::SQLITE_READONLY; 383 | } 384 | } 385 | let result = match file.journal.as_mut() { 386 | Some(journal) => { 387 | let new_page = std::slice::from_raw_parts(buf.cast::(), amt as usize); 388 | let mut old_page = vec![0_u8; amt as usize]; 389 | let mut iter = 390 | match MclVFSIO.xRead.unwrap()(pfile, old_page.as_mut_ptr().cast(), amt, offset) { 391 | // existing page 392 | ffi::SQLITE_OK => utils::get_diff(new_page, &old_page), 393 | // new page 394 | ffi::SQLITE_IOERR_SHORT_READ => utils::get_diff(new_page, &[]), 395 | _other => return ffi::SQLITE_ERROR, 396 | }; 397 | iter.try_for_each(|(diff_offset, diff)| { 398 | let diff_offset = diff_offset as i64 + offset; 399 | journal 400 | .new_snapshot(amt as u32) 401 | .and_then(|_| journal.new_blob(diff_offset as u64, diff)) 402 | }) 403 | } 404 | None => Ok(()), 405 | }; 406 | if let Err(_e) = result { 407 | return ffi::SQLITE_ERROR; 408 | } 409 | (*file.real.pMethods).xWrite.unwrap()(&mut file.real, buf, amt, offset) 410 | } 411 | 412 | unsafe extern "C" fn mvfs_io_truncate( 413 | pfile: *mut ffi::sqlite3_file, 414 | size: ffi::sqlite3_int64, 415 | ) -> c_int { 416 | let file = MclVFSFile::from_ptr(pfile); 417 | (*file.real.pMethods).xTruncate.unwrap()(&mut file.real, size) 418 | } 419 | 420 | unsafe extern "C" fn mvfs_io_sync(pfile: *mut ffi::sqlite3_file, flags: c_int) -> c_int { 421 | let file = MclVFSFile::from_ptr(pfile); 422 | match file.journal.as_mut().map(|journal| journal.commit()) { 423 | None | Some(Ok(_)) => (), 424 | Some(Err(_e)) => return ffi::SQLITE_ERROR, 425 | }; 426 | if let Some(replicator) = file.replicator.as_mut() { 427 | replicator.new_snapshot(); 428 | } 429 | (*file.real.pMethods).xSync.unwrap()(&mut file.real, flags) 430 | } 431 | 432 | unsafe extern "C" fn mvfs_io_file_size( 433 | pfile: *mut ffi::sqlite3_file, 434 | psize: *mut ffi::sqlite3_int64, 435 | ) -> c_int { 436 | let file = MclVFSFile::from_ptr(pfile); 437 | (*file.real.pMethods).xFileSize.unwrap()(&mut file.real, psize) 438 | } 439 | 440 | unsafe extern "C" fn mvfs_io_lock(pfile: *mut ffi::sqlite3_file, elock: c_int) -> c_int { 441 | let file = MclVFSFile::from_ptr(pfile); 442 | let real = (&mut file.real) as *mut ffi::sqlite3_file; 443 | // lock only main database file 444 | if file.journal.is_some() { 445 | file.lock(); 446 | } 447 | (*(*real).pMethods).xLock.unwrap()(real, elock) 448 | } 449 | 450 | unsafe extern "C" fn mvfs_io_unlock(pfile: *mut ffi::sqlite3_file, elock: c_int) -> c_int { 451 | let file = MclVFSFile::from_ptr(pfile); 452 | if file.journal.is_some() { 453 | file.unlock() 454 | } 455 | (*file.real.pMethods).xUnlock.unwrap()(&mut file.real, elock) 456 | } 457 | 458 | unsafe extern "C" fn mvfs_io_check_reserved_lock( 459 | pfile: *mut ffi::sqlite3_file, 460 | out: *mut c_int, 461 | ) -> c_int { 462 | let file = MclVFSFile::from_ptr(pfile); 463 | (*file.real.pMethods).xCheckReservedLock.unwrap()(&mut file.real, out) 464 | } 465 | 466 | unsafe extern "C" fn mvfs_io_file_control( 467 | pfile: *mut ffi::sqlite3_file, 468 | op: c_int, 469 | p_arg: *mut c_void, 470 | ) -> c_int { 471 | let file = MclVFSFile::from_ptr(pfile); 472 | (*file.real.pMethods).xFileControl.unwrap()(&mut file.real, op, p_arg) 473 | } 474 | 475 | unsafe extern "C" fn mvfs_io_sector_size(pfile: *mut ffi::sqlite3_file) -> c_int { 476 | let file = MclVFSFile::from_ptr(pfile); 477 | (*file.real.pMethods).xSectorSize.unwrap()(&mut file.real) 478 | } 479 | 480 | unsafe extern "C" fn mvfs_io_device_characteristics(pfile: *mut ffi::sqlite3_file) -> c_int { 481 | let file = MclVFSFile::from_ptr(pfile); 482 | (*file.real.pMethods).xDeviceCharacteristics.unwrap()(&mut file.real) 483 | } 484 | -------------------------------------------------------------------------------- /page_parser/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "page_parser" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | serde = { version = "1", features = ["derive"] } 10 | serde_sqlite = { path = "../serde_sqlite" } 11 | block = { path = "../block" } 12 | -------------------------------------------------------------------------------- /page_parser/README.md: -------------------------------------------------------------------------------- 1 | ## Sqlite Page Parser 2 | -------------------------------------------------------------------------------- /page_parser/src/database.rs: -------------------------------------------------------------------------------- 1 | //! Sqlite Database 2 | use crate::header::Header; 3 | use crate::page::RawPage; 4 | use serde_sqlite::from_bytes; 5 | use std::io::BufReader; 6 | use std::io::{Read, Seek}; 7 | use std::path::PathBuf; 8 | 9 | #[derive(Debug)] 10 | pub struct Database { 11 | path: PathBuf, 12 | } 13 | 14 | impl Database { 15 | pub fn new>(p: P) -> Self { 16 | Self { path: p.into() } 17 | } 18 | 19 | /// Initialize iterator over raw sqlite pages 20 | pub fn into_raw_page_iter(&self) -> Result> { 21 | let mut fd = std::fs::OpenOptions::new() 22 | .read(true) 23 | .open(self.path.as_path())?; 24 | let db_size = fd.metadata()?.len(); 25 | let (page_size, pages_left) = match db_size { 26 | 0 => (0, 0), 27 | _ => { 28 | let mut buf = [0_u8; 100]; 29 | fd.read_exact(buf.as_mut_slice())?; 30 | let header = from_bytes::
(buf.as_slice())?; 31 | let page_size = header.page_size() as u64; 32 | (page_size, db_size / page_size) 33 | } 34 | }; 35 | fd.rewind()?; 36 | Ok(RawPageIter { 37 | fd: BufReader::new(fd), 38 | page_size, 39 | pages_left, 40 | }) 41 | } 42 | } 43 | 44 | #[derive(Debug)] 45 | pub struct RawPageIter { 46 | // for now only file iter, but in-memory option also can be supported 47 | fd: BufReader, 48 | page_size: u64, 49 | pages_left: u64, 50 | } 51 | 52 | impl Iterator for RawPageIter { 53 | type Item = Result<(u64, RawPage), std::io::Error>; 54 | 55 | fn next(&mut self) -> Option { 56 | if self.pages_left == 0 { 57 | return None; 58 | }; 59 | self.pages_left -= 1; 60 | let offset = match self.fd.stream_position() { 61 | Err(e) => return Some(Err(e)), 62 | Ok(offset) => offset, 63 | }; 64 | let mut page = vec![0; self.page_size as usize]; 65 | match self.fd.read_exact(page.as_mut_slice()) { 66 | Ok(_) => Some(Ok((offset, RawPage::new(page)))), 67 | Err(e) => { 68 | self.pages_left = 0; 69 | Some(Err(e)) 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /page_parser/src/header.rs: -------------------------------------------------------------------------------- 1 | //! [Sqlite Database Header] 2 | 3 | use block::block; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | /// sqlite database header 7 | #[derive(Debug, Clone, Serialize, Deserialize)] 8 | #[block(100)] 9 | pub struct Header { 10 | /// sqlite header magic: 'SQLite format 3\0' 11 | pub magic: [u8; 16], 12 | /// sqlite page size, values of power of two between 512 and 32768 inclusive, or the value 1 representing a page size of 65536 13 | pub page_size: u16, 14 | /// file format write version: 1 for legacy, 2 for WAL 15 | pub write_version: u8, 16 | /// file format read vresion: 1 for legacy, 2 for WAL 17 | pub read_version: u8, 18 | // reserved 19 | _reserved_1: u8, 20 | /// max embedded payload fraction, must be 64 21 | pub max_embedded_payload_fraction: u8, 22 | /// min embedded payload fraction, must be 32 23 | pub min_embedded_payload_fraction: u8, 24 | /// leaf payload fraction 25 | pub leaf_payload_fraction: u8, 26 | /// file change counter 27 | pub file_change_counter: u32, 28 | /// size of the database file in pages, the "in-header database size" 29 | pub database_size: u32, 30 | /// page number of the first freelist trunk page 31 | #[serde( 32 | deserialize_with = "serde_sqlite::de::zero_as_none", 33 | serialize_with = "serde_sqlite::se::none_as_zero" 34 | )] 35 | pub first_freelist_page_num: Option, 36 | /// total number of freelist pages 37 | pub freelist_pages_total: u32, 38 | /// schema cookie 39 | pub schema_cookie: u32, 40 | /// schema format number, supported values are 1, 2, 3 and 4 41 | pub schema_format_num: u32, 42 | /// default page cache size 43 | pub default_page_cache_size: u32, 44 | /// page number of largest root b-tree page when in auto-vacuum or incremental vacuum modes, zero otherwise 45 | pub largest_root: u32, 46 | /// db text encoding: 47 | /// UTF-8 - 1 48 | /// UTF-16le - 2 49 | /// UTF-16be - 3 50 | pub text_encoding: u32, 51 | /// user version, set by user version pragma 52 | pub user_version: u32, 53 | /// incremental vacuum mode flag, true if not 0, false otherwize 54 | pub inc_vacuum_mode: u32, 55 | /// application id, set by pragma application id 56 | pub application_id: u32, 57 | // reserved 58 | _reserved_2: [u8; 20], 59 | // offset: 72, size: 20 60 | /// version of sqlite which modified database recently 61 | pub version_valid_for_number: u32, 62 | /// sqlite version number 63 | pub version: u32, 64 | } 65 | 66 | impl Header { 67 | pub fn page_size(&self) -> u32 { 68 | match self.page_size { 69 | 1 => 0x10000, 70 | v => v as u32, 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /page_parser/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod database; 2 | pub(crate) mod header; 3 | pub(crate) mod page; 4 | 5 | pub use database::Database; 6 | pub use header::Header; 7 | pub use page::RawPage; 8 | -------------------------------------------------------------------------------- /page_parser/src/page.rs: -------------------------------------------------------------------------------- 1 | //! Sqlite Page 2 | 3 | /// Sqlite Raw Page 4 | /// 5 | /// Just a chunk of bytes representing sqlite database page 6 | #[derive(Debug)] 7 | pub struct RawPage(Vec); 8 | 9 | impl RawPage { 10 | pub fn new(page: Vec) -> Self { 11 | Self(page) 12 | } 13 | 14 | pub fn as_slice(&self) -> &[u8] { 15 | self.0.as_slice() 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /page_parser/tests/header_test.rs: -------------------------------------------------------------------------------- 1 | //! validate sqlite header deserializer/serialiser. 2 | //! deserialized version compared against manually parsed version. 3 | //! serialized version should produce exact header is was deserialized from. 4 | 5 | use page_parser::Header; 6 | 7 | use std::ffi::CStr; 8 | 9 | // real sqlite3 header 10 | static HEADER: [u8; 100] = [ 11 | 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, 12 | 0x10, 0x00, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 13 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 14 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 15 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 16 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 17 | 0x00, 0x2e, 0x63, 0x00, 18 | ]; 19 | 20 | /// manually parsed header 21 | #[derive(Debug, Clone)] 22 | pub struct TestHeader { 23 | pub magic: [u8; 16], 24 | pub page_size: u32, 25 | pub write_version: u8, 26 | pub read_version: u8, 27 | pub max_embedded_payload_fraction: u8, 28 | pub min_embedded_payload_fraction: u8, 29 | pub leaf_payload_fraction: u8, 30 | pub file_change_counter: u32, 31 | pub db_size: u32, 32 | pub first_free_page_num: Option, 33 | pub freelist_total: u32, 34 | pub schema_cookie: u32, 35 | pub schema_format_num: u32, 36 | pub default_page_cache_size: u32, 37 | pub largest_root: u32, 38 | pub text_encoding: u32, 39 | pub user_version: u32, 40 | pub inc_vacuum_mode: u32, 41 | pub application_id: u32, 42 | pub version_valid_for_number: u32, 43 | pub version: u32, 44 | } 45 | 46 | macro_rules! slc { 47 | ($buf:ident, $offset:expr, $len:expr) => { 48 | $buf[$offset..($offset + $len)] 49 | }; 50 | ($buf:ident, $offset:expr, $len:expr, $t:ty) => { 51 | <$t>::from_be_bytes(slc!($buf, $offset, $len).try_into()?) 52 | }; 53 | } 54 | 55 | impl TryFrom<&[u8; 100]> for TestHeader { 56 | type Error = Box; 57 | 58 | fn try_from(buf: &[u8; 100]) -> Result { 59 | Ok(Self::new( 60 | { 61 | let mut magic = [0_u8; 16]; 62 | magic.copy_from_slice(&buf[..16]); 63 | magic 64 | }, // header 65 | slc!(buf, 16, 2, u16), // page size 66 | slc!(buf, 18, 1, u8), // write_version 67 | slc!(buf, 19, 1, u8), // read_version 68 | slc!(buf, 21, 1, u8), // max_embedded_payload_fraction 69 | slc!(buf, 22, 1, u8), // min_embedded_payload_fraction 70 | slc!(buf, 23, 1, u8), // leaf_payload_fraction 71 | slc!(buf, 24, 4, u32), // file_change_counter 72 | slc!(buf, 28, 4, u32), // db_size 73 | slc!(buf, 32, 4, u32).checked_sub(1).map(|x| x + 1), // first_free_page_num 74 | slc!(buf, 36, 4, u32), // freelist_total 75 | slc!(buf, 40, 4, u32), // schema_cookie 76 | slc!(buf, 44, 4, u32), // schema_format_num 77 | slc!(buf, 48, 4, u32), // default_page_cache 78 | slc!(buf, 52, 4, u32), // largest_root 79 | slc!(buf, 56, 4, u32), // text_encoding 80 | slc!(buf, 60, 4, u32), // user_version 81 | slc!(buf, 64, 4, u32), // inc_vacuum_mode 82 | slc!(buf, 68, 4, u32), // application_id 83 | slc!(buf, 92, 4, u32), // version_valid_for_number 84 | slc!(buf, 96, 4, u32), // version 85 | )) 86 | } 87 | } 88 | 89 | impl TestHeader { 90 | pub fn new( 91 | magic: [u8; 16], 92 | page_size: u16, 93 | write_version: u8, 94 | read_version: u8, 95 | max_embedded_payload_fraction: u8, 96 | min_embedded_payload_fraction: u8, 97 | leaf_payload_fraction: u8, 98 | file_change_counter: u32, 99 | db_size: u32, 100 | first_free_page_num: Option, 101 | freelist_total: u32, 102 | schema_cookie: u32, 103 | schema_format_num: u32, 104 | default_page_cache_size: u32, 105 | largest_root: u32, 106 | text_encoding: u32, 107 | user_version: u32, 108 | inc_vacuum_mode: u32, 109 | application_id: u32, 110 | version_valid_for_number: u32, 111 | version: u32, 112 | ) -> Self { 113 | Self { 114 | magic, 115 | page_size: Self::to_page_size(page_size), 116 | write_version, 117 | read_version, 118 | max_embedded_payload_fraction, 119 | min_embedded_payload_fraction, 120 | leaf_payload_fraction, 121 | file_change_counter, 122 | db_size, 123 | first_free_page_num, 124 | freelist_total, 125 | schema_cookie, 126 | schema_format_num, 127 | default_page_cache_size, 128 | largest_root, 129 | text_encoding, 130 | user_version, 131 | inc_vacuum_mode, 132 | application_id, 133 | version_valid_for_number, 134 | version, 135 | } 136 | } 137 | 138 | /// Get real page size 139 | /// 140 | /// Field stores only 2 bytes, to max value to represent is 65535 141 | /// To specify page size of value 65536 - 0x0001 value is used 142 | fn to_page_size(value: u16) -> u32 { 143 | match value { 144 | 1 => 65536, 145 | v => v as u32, 146 | } 147 | } 148 | } 149 | 150 | #[test] 151 | fn header_deserialize_serialize() { 152 | let header = serde_sqlite::from_bytes::
(HEADER.as_slice()); 153 | assert!(header.is_ok(), "{header:?}"); 154 | let header = header.unwrap(); 155 | 156 | let test_header = >::try_from(&HEADER); 157 | assert!(test_header.is_ok(), "{test_header:?}"); 158 | let test_header = test_header.unwrap(); 159 | 160 | // check magic 161 | assert_eq!(header.magic, test_header.magic); 162 | assert_eq!( 163 | CStr::from_bytes_with_nul(header.magic.as_slice()), 164 | CStr::from_bytes_with_nul(b"SQLite format 3\0") 165 | ); 166 | 167 | // check page size 168 | assert_eq!(header.page_size(), test_header.page_size); 169 | 170 | // check write version 171 | assert_eq!(header.write_version, test_header.write_version); 172 | 173 | // check read version 174 | assert_eq!(header.read_version, test_header.read_version); 175 | 176 | // check max embedded payload fraction 177 | assert_eq!( 178 | header.max_embedded_payload_fraction, 179 | test_header.max_embedded_payload_fraction 180 | ); 181 | 182 | // check min embedded payload fraction 183 | assert_eq!( 184 | header.min_embedded_payload_fraction, 185 | test_header.min_embedded_payload_fraction 186 | ); 187 | 188 | // check leaf payload fraction 189 | assert_eq!( 190 | header.leaf_payload_fraction, 191 | test_header.leaf_payload_fraction 192 | ); 193 | 194 | // check file change counter 195 | assert_eq!(header.file_change_counter, test_header.file_change_counter); 196 | 197 | // check db size 198 | assert_eq!(header.database_size, test_header.db_size); 199 | 200 | // check first free page num 201 | assert_eq!( 202 | header.first_freelist_page_num, 203 | test_header.first_free_page_num 204 | ); 205 | 206 | // check freelist total 207 | assert_eq!(header.freelist_pages_total, test_header.freelist_total); 208 | 209 | // check schema cookie 210 | assert_eq!(header.schema_cookie, test_header.schema_cookie); 211 | 212 | // check schema format num 213 | assert_eq!(header.schema_format_num, test_header.schema_format_num); 214 | 215 | // check default page cache size 216 | assert_eq!( 217 | header.default_page_cache_size, 218 | test_header.default_page_cache_size 219 | ); 220 | 221 | // check largest root 222 | assert_eq!(header.largest_root, test_header.largest_root); 223 | 224 | // check text encoding 225 | assert_eq!(header.text_encoding, { test_header.text_encoding }); 226 | 227 | // check user version 228 | assert_eq!(header.user_version, test_header.user_version); 229 | 230 | // check inc vacuum mode 231 | assert_eq!(header.inc_vacuum_mode, test_header.inc_vacuum_mode); 232 | 233 | // check application id 234 | assert_eq!(header.application_id, test_header.application_id); 235 | 236 | // check version valid for number 237 | assert_eq!( 238 | header.version_valid_for_number, 239 | test_header.version_valid_for_number 240 | ); 241 | 242 | // check version 243 | assert_eq!(header.version, test_header.version); 244 | 245 | // check serialized header equals original header 246 | let bytes = serde_sqlite::to_bytes(&header); 247 | assert!(bytes.is_ok(), "{bytes:?}"); 248 | let bytes = bytes.unwrap(); 249 | assert_eq!(bytes, HEADER); 250 | } 251 | -------------------------------------------------------------------------------- /serde_sqlite/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "serde_sqlite" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | serde = { version = "1", features = ["derive"] } 10 | block = { path = "../block" } 11 | -------------------------------------------------------------------------------- /serde_sqlite/src/de.rs: -------------------------------------------------------------------------------- 1 | //! SQLite data format deserializer 2 | 3 | use crate::error::Error; 4 | use block::Block; 5 | use serde::{ 6 | de, de::DeserializeSeed, de::IntoDeserializer, de::Visitor, Deserialize, Deserializer, 7 | }; 8 | use std::io::Read; 9 | 10 | struct SqliteDe { 11 | reader: R, 12 | } 13 | 14 | impl SqliteDe { 15 | fn from_reader(reader: R) -> Self { 16 | Self { reader } 17 | } 18 | } 19 | 20 | impl<'de, 'a, R> Deserializer<'de> for &'a mut SqliteDe 21 | where 22 | R: Read, 23 | { 24 | type Error = Error; 25 | 26 | fn deserialize_any(self, _v: V) -> Result 27 | where 28 | V: Visitor<'de>, 29 | { 30 | Err(Self::Error::Unsupported("Deserializer::deserialize_any")) 31 | } 32 | 33 | fn deserialize_bool(self, v: V) -> Result 34 | where 35 | V: Visitor<'de>, 36 | { 37 | let mut buf = [0_u8; 1]; 38 | self.reader.read_exact(&mut buf)?; 39 | v.visit_bool(buf[0] == 1) 40 | } 41 | 42 | fn deserialize_i8(self, v: V) -> Result 43 | where 44 | V: Visitor<'de>, 45 | { 46 | let mut buf = [0; 1]; 47 | self.reader.read_exact(buf.as_mut_slice())?; 48 | v.visit_i8(i8::from_be_bytes(buf)) 49 | } 50 | 51 | fn deserialize_i16(self, v: V) -> Result 52 | where 53 | V: Visitor<'de>, 54 | { 55 | let mut buf = [0; 2]; 56 | self.reader.read_exact(buf.as_mut_slice())?; 57 | v.visit_i16(i16::from_be_bytes(buf)) 58 | } 59 | 60 | fn deserialize_i32(self, v: V) -> Result 61 | where 62 | V: Visitor<'de>, 63 | { 64 | let mut buf = [0; 4]; 65 | self.reader.read_exact(buf.as_mut_slice())?; 66 | v.visit_i32(i32::from_be_bytes(buf)) 67 | } 68 | 69 | fn deserialize_i64(self, v: V) -> Result 70 | where 71 | V: Visitor<'de>, 72 | { 73 | let mut buf = [0; 8]; 74 | self.reader.read_exact(buf.as_mut_slice())?; 75 | v.visit_i64(i64::from_be_bytes(buf)) 76 | } 77 | 78 | fn deserialize_u8(self, v: V) -> Result 79 | where 80 | V: Visitor<'de>, 81 | { 82 | let mut buf = [0; 1]; 83 | self.reader.read_exact(buf.as_mut_slice())?; 84 | v.visit_u8(u8::from_be_bytes(buf)) 85 | } 86 | 87 | fn deserialize_u16(self, v: V) -> Result 88 | where 89 | V: Visitor<'de>, 90 | { 91 | let mut buf = [0; 2]; 92 | self.reader.read_exact(buf.as_mut_slice())?; 93 | v.visit_u16(u16::from_be_bytes(buf)) 94 | } 95 | 96 | fn deserialize_u32(self, v: V) -> Result 97 | where 98 | V: Visitor<'de>, 99 | { 100 | let mut buf = [0; 4]; 101 | self.reader.read_exact(buf.as_mut_slice())?; 102 | v.visit_u32(u32::from_be_bytes(buf)) 103 | } 104 | 105 | fn deserialize_u64(self, v: V) -> Result 106 | where 107 | V: Visitor<'de>, 108 | { 109 | let mut buf = [0; 8]; 110 | self.reader.read_exact(&mut buf)?; 111 | v.visit_u64(u64::from_be_bytes(buf)) 112 | } 113 | 114 | fn deserialize_f32(self, v: V) -> Result 115 | where 116 | V: Visitor<'de>, 117 | { 118 | let mut buf = [0; 4]; 119 | self.reader.read_exact(&mut buf)?; 120 | v.visit_f32(f32::from_be_bytes(buf)) 121 | } 122 | 123 | fn deserialize_f64(self, v: V) -> Result 124 | where 125 | V: Visitor<'de>, 126 | { 127 | let mut buf = [0; 8]; 128 | self.reader.read_exact(&mut buf)?; 129 | v.visit_f64(f64::from_be_bytes(buf)) 130 | } 131 | 132 | fn deserialize_char(self, _v: V) -> Result 133 | where 134 | V: Visitor<'de>, 135 | { 136 | Err(Self::Error::Unsupported("Deserializer::deserialize_char")) 137 | } 138 | 139 | fn deserialize_str(self, _v: V) -> Result 140 | where 141 | V: Visitor<'de>, 142 | { 143 | Err(Error::Unsupported("Deserializer::deserialize_str")) 144 | } 145 | 146 | fn deserialize_string(self, _v: V) -> Result 147 | where 148 | V: Visitor<'de>, 149 | { 150 | Err(Error::Unsupported("Deserializer::deserialize_string")) 151 | } 152 | 153 | fn deserialize_bytes(self, _v: V) -> Result 154 | where 155 | V: Visitor<'de>, 156 | { 157 | Err(Error::Unsupported("Deserializer::deserialize_bytes")) 158 | } 159 | 160 | fn deserialize_byte_buf(self, _v: V) -> Result 161 | where 162 | V: Visitor<'de>, 163 | { 164 | Err(Error::Unsupported("Deserializer::deserialize_byte_buf")) 165 | } 166 | 167 | fn deserialize_option(self, _v: V) -> Result 168 | where 169 | V: Visitor<'de>, 170 | { 171 | Err(Error::Unsupported("Deserializer::deserialize_option")) 172 | } 173 | 174 | fn deserialize_unit(self, _v: V) -> Result 175 | where 176 | V: Visitor<'de>, 177 | { 178 | Err(Error::Unsupported("Deserializer::deserialize_unit")) 179 | } 180 | 181 | fn deserialize_unit_struct(self, _name: &str, _v: V) -> Result 182 | where 183 | V: Visitor<'de>, 184 | { 185 | Err(Error::Unsupported("Deserializer::deserialize_unit_struct")) 186 | } 187 | 188 | fn deserialize_newtype_struct(self, _name: &str, _v: V) -> Result 189 | where 190 | V: Visitor<'de>, 191 | { 192 | Err(Error::Unsupported( 193 | "Deserializer::deserialize_newtype_struct", 194 | )) 195 | } 196 | 197 | fn deserialize_seq(self, _v: V) -> Result 198 | where 199 | V: Visitor<'de>, 200 | { 201 | Err(Error::Unsupported("Deserializer::deserialize_seq")) 202 | } 203 | 204 | fn deserialize_tuple(self, len: usize, v: V) -> Result 205 | where 206 | V: Visitor<'de>, 207 | { 208 | v.visit_seq(SeqAccess { de: self, len }) 209 | } 210 | 211 | fn deserialize_tuple_struct( 212 | self, 213 | _name: &str, 214 | _len: usize, 215 | _v: V, 216 | ) -> Result 217 | where 218 | V: Visitor<'de>, 219 | { 220 | Err(Error::Unsupported("Deserializer::deserialize_tuple_struct")) 221 | } 222 | 223 | fn deserialize_map(self, _v: V) -> Result 224 | where 225 | V: Visitor<'de>, 226 | { 227 | Err(Error::Unsupported("Deserializer::deserialize_map")) 228 | } 229 | 230 | fn deserialize_struct( 231 | self, 232 | _name: &str, 233 | fields: &[&str], 234 | v: V, 235 | ) -> Result 236 | where 237 | V: Visitor<'de>, 238 | { 239 | self.deserialize_tuple(fields.len(), v) 240 | } 241 | 242 | fn deserialize_enum( 243 | self, 244 | _name: &str, 245 | _variants: &[&str], 246 | v: V, 247 | ) -> Result 248 | where 249 | V: Visitor<'de>, 250 | { 251 | v.visit_enum(EnumAccess::new(self)) 252 | } 253 | 254 | fn deserialize_identifier(self, _v: V) -> Result 255 | where 256 | V: Visitor<'de>, 257 | { 258 | Err(Error::Unsupported("Deserializer::deserialize_identifier")) 259 | } 260 | 261 | fn deserialize_ignored_any(self, _v: V) -> Result 262 | where 263 | V: Visitor<'de>, 264 | { 265 | Err(Error::Unsupported("Deserializer::deserialize_ignored_any")) 266 | } 267 | } 268 | 269 | struct EnumAccess<'a, R: 'a> { 270 | de: &'a mut SqliteDe, 271 | } 272 | 273 | impl<'a, R> EnumAccess<'a, R> { 274 | fn new(de: &'a mut SqliteDe) -> Self { 275 | Self { de } 276 | } 277 | } 278 | 279 | impl<'a, 'de, R: Read> de::EnumAccess<'de> for EnumAccess<'a, R> { 280 | type Error = Error; 281 | type Variant = VariantAccess<'a, R>; 282 | 283 | fn variant(self) -> Result<(V, Self::Variant), Self::Error> 284 | where 285 | V: Deserialize<'de>, 286 | { 287 | let mut buf = [0_u8; 4]; 288 | self.de.reader.read_exact(&mut buf)?; 289 | let tag = u32::from_be_bytes(buf) as u64; 290 | let de = IntoDeserializer::::into_deserializer(tag); 291 | let tag = V::deserialize(de)?; 292 | Ok((tag, VariantAccess { de: self.de })) 293 | } 294 | 295 | fn variant_seed(self, _seed: V) -> Result<(V::Value, Self::Variant), Error> 296 | where 297 | V: DeserializeSeed<'de>, 298 | { 299 | Err(Error::Unsupported("EnumAccess::variant_seed")) 300 | } 301 | } 302 | 303 | struct VariantAccess<'a, R: 'a> { 304 | de: &'a mut SqliteDe, 305 | } 306 | 307 | impl<'a, 'de, R: Read> de::VariantAccess<'de> for VariantAccess<'a, R> { 308 | type Error = Error; 309 | 310 | fn unit_variant(self) -> Result<(), Self::Error> { 311 | Err(Error::Unsupported("VariantAccess::unit_variant")) 312 | } 313 | 314 | fn newtype_variant_seed(self, _seed: T) -> Result 315 | where 316 | T: DeserializeSeed<'de>, 317 | { 318 | Err(Error::Unsupported("VariantAccess::newtype_variant_seed")) 319 | } 320 | 321 | fn tuple_variant(self, _len: usize, _visitor: V) -> Result 322 | where 323 | V: Visitor<'de>, 324 | { 325 | Err(Error::Unsupported("VariantAccess::tuple_variant")) 326 | } 327 | 328 | fn struct_variant( 329 | self, 330 | _fields: &'static [&'static str], 331 | _visitor: V, 332 | ) -> Result 333 | where 334 | V: Visitor<'de>, 335 | { 336 | Err(Error::Unsupported("VariantAccess::struct_variant")) 337 | } 338 | 339 | fn newtype_variant(self) -> Result 340 | where 341 | T: Deserialize<'de>, 342 | { 343 | T::deserialize(self.de) 344 | } 345 | } 346 | 347 | /// SeqAccess Visitor 348 | struct SeqAccess<'a, R: 'a> { 349 | de: &'a mut SqliteDe, 350 | len: usize, 351 | } 352 | 353 | impl<'a, 'de, R: Read> de::SeqAccess<'de> for SeqAccess<'a, R> { 354 | type Error = Error; 355 | 356 | fn next_element_seed(&mut self, _seed: T) -> Result, Self::Error> 357 | where 358 | T: de::DeserializeSeed<'de>, 359 | { 360 | Err(Error::Unsupported("SeqAccess::next_element")) 361 | } 362 | 363 | fn next_element(&mut self) -> Result, Self::Error> 364 | where 365 | T: Deserialize<'de>, 366 | { 367 | if self.len > 0 { 368 | self.len -= 1; 369 | T::deserialize(&mut *self.de).map(Some) 370 | } else { 371 | Ok(None) 372 | } 373 | } 374 | } 375 | 376 | struct CountingReader { 377 | reader: R, 378 | read: usize, 379 | } 380 | 381 | impl CountingReader { 382 | fn new(reader: R) -> Self { 383 | Self { reader, read: 0 } 384 | } 385 | 386 | fn discard_padding(&mut self, left: usize) -> std::io::Result<()> { 387 | if left == 0 { 388 | return Ok(()); 389 | } 390 | let mut buf = vec![0; left]; 391 | self.read_exact(buf.as_mut_slice())?; 392 | Ok(()) 393 | } 394 | } 395 | 396 | impl Read for CountingReader { 397 | fn read(&mut self, buf: &mut [u8]) -> std::io::Result { 398 | let read = self.reader.read(buf)?; 399 | self.read += read; 400 | Ok(read) 401 | } 402 | } 403 | 404 | /// Deserialize default value (zero) as None 405 | pub fn zero_as_none<'de, D, T>(d: D) -> Result, D::Error> 406 | where 407 | D: Deserializer<'de>, 408 | T: Deserialize<'de> + Default + Copy + PartialEq + Eq, 409 | { 410 | match T::deserialize(d) { 411 | Ok(value) if value == T::default() => Ok(None), 412 | Ok(value) => Ok(Some(value)), 413 | Err(e) => Err(e), 414 | } 415 | } 416 | 417 | pub fn from_bytes<'de, T>(input: &'de [u8]) -> Result 418 | where 419 | T: Deserialize<'de> + Block, 420 | { 421 | from_reader(input) 422 | } 423 | 424 | pub fn from_reader<'de, T, R>(reader: R) -> Result 425 | where 426 | T: Deserialize<'de> + Block, 427 | R: Read, 428 | { 429 | let mut cbr = CountingReader::new(reader); 430 | let res = T::deserialize(&mut SqliteDe::from_reader(&mut cbr))?; 431 | cbr.discard_padding(res.iblock_size() - cbr.read)?; 432 | Ok(res) 433 | } -------------------------------------------------------------------------------- /serde_sqlite/src/error.rs: -------------------------------------------------------------------------------- 1 | //! Sqlite Data Format Serialize/Deserialize Error 2 | 3 | use serde::{de, ser}; 4 | use std::fmt; 5 | 6 | #[derive(Debug)] 7 | pub enum Error { 8 | Message(String), 9 | IoError(std::io::Error), 10 | Incomplete, 11 | Unexpected, 12 | Unsupported(&'static str), 13 | OutOfMemory(std::collections::TryReserveError), 14 | } 15 | 16 | impl fmt::Display for Error { 17 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 18 | write!(f, "{self:?}") 19 | } 20 | } 21 | 22 | impl std::error::Error for Error {} 23 | 24 | impl ser::Error for Error { 25 | fn custom(msg: T) -> Self { 26 | Self::Message(msg.to_string()) 27 | } 28 | } 29 | 30 | impl de::Error for Error { 31 | fn custom(msg: T) -> Self { 32 | Self::Message(msg.to_string()) 33 | } 34 | } 35 | 36 | impl From for Error { 37 | fn from(e: std::io::Error) -> Self { 38 | Self::IoError(e) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /serde_sqlite/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod de; 2 | mod error; 3 | pub mod se; 4 | 5 | pub use de::{from_bytes, from_reader}; 6 | pub use error::Error; 7 | pub use se::{to_bytes, to_writer}; 8 | -------------------------------------------------------------------------------- /serde_sqlite/src/se.rs: -------------------------------------------------------------------------------- 1 | //! Sqlite data format serializer 2 | use crate::error::Error; 3 | use block::Block; 4 | use serde::{ 5 | ser::SerializeMap, ser::SerializeSeq, ser::SerializeStruct, ser::SerializeStructVariant, 6 | ser::SerializeTuple, ser::SerializeTupleStruct, ser::SerializeTupleVariant, Serialize, 7 | Serializer, 8 | }; 9 | use std::io::{BufWriter, Write}; 10 | 11 | struct SqliteSe { 12 | writer: W, 13 | } 14 | 15 | impl<'a, W: Write> Serializer for &'a mut SqliteSe { 16 | type Ok = (); 17 | type Error = Error; 18 | 19 | type SerializeSeq = Self; 20 | type SerializeTuple = Self; 21 | type SerializeTupleStruct = Self; 22 | type SerializeTupleVariant = Self; 23 | type SerializeMap = Self; 24 | type SerializeStruct = Self; 25 | type SerializeStructVariant = Self; 26 | 27 | fn serialize_bool(self, value: bool) -> Result { 28 | self.writer 29 | .write_all(&(value as u8).to_be_bytes()) 30 | .map_err(Into::into) 31 | } 32 | 33 | fn serialize_i8(self, value: i8) -> Result { 34 | self.writer 35 | .write_all(&value.to_be_bytes()) 36 | .map_err(Into::into) 37 | } 38 | 39 | fn serialize_i16(self, value: i16) -> Result { 40 | self.writer 41 | .write_all(&value.to_be_bytes()) 42 | .map_err(Into::into) 43 | } 44 | 45 | fn serialize_i32(self, value: i32) -> Result { 46 | self.writer 47 | .write_all(&value.to_be_bytes()) 48 | .map_err(Into::into) 49 | } 50 | 51 | fn serialize_i64(self, value: i64) -> Result { 52 | self.writer 53 | .write_all(&value.to_be_bytes()) 54 | .map_err(Into::into) 55 | } 56 | 57 | fn serialize_u8(self, value: u8) -> Result { 58 | self.writer 59 | .write_all(&value.to_be_bytes()) 60 | .map_err(Into::into) 61 | } 62 | 63 | fn serialize_u16(self, value: u16) -> Result { 64 | self.writer 65 | .write_all(&value.to_be_bytes()) 66 | .map_err(Into::into) 67 | } 68 | 69 | fn serialize_u32(self, value: u32) -> Result { 70 | self.writer 71 | .write_all(&value.to_be_bytes()) 72 | .map_err(Into::into) 73 | } 74 | 75 | fn serialize_u64(self, value: u64) -> Result { 76 | self.writer 77 | .write_all(&value.to_be_bytes()) 78 | .map_err(Into::into) 79 | } 80 | 81 | fn serialize_f32(self, value: f32) -> Result { 82 | self.writer 83 | .write_all(&value.to_be_bytes()) 84 | .map_err(Into::into) 85 | } 86 | 87 | fn serialize_f64(self, value: f64) -> Result { 88 | self.writer 89 | .write_all(&value.to_be_bytes()) 90 | .map_err(Into::into) 91 | } 92 | 93 | fn serialize_char(self, value: char) -> Result { 94 | // char is always 4 bytes long 95 | self.writer 96 | .write_all(&(value as u32).to_be_bytes()) 97 | .map_err(Into::into) 98 | } 99 | 100 | fn serialize_str(self, _: &str) -> Result { 101 | Err(Error::Unsupported("Serializer::serialize_str")) 102 | } 103 | 104 | fn serialize_bytes(self, _: &[u8]) -> Result { 105 | Err(Error::Unsupported("Serializer::serialize_bytes")) 106 | } 107 | 108 | fn serialize_none(self) -> Result { 109 | Err(Error::Unsupported("Serializer::serialize_none")) 110 | } 111 | 112 | fn serialize_some(self, _: &T) -> Result { 113 | Err(Error::Unsupported("Serializer::serialize_some")) 114 | } 115 | 116 | fn serialize_unit(self) -> Result { 117 | Err(Error::Unsupported("Serializer::serialize_unit")) 118 | } 119 | 120 | fn serialize_unit_struct(self, _name: &str) -> Result { 121 | Err(Error::Unsupported("Serializer::serialize_unit_struct")) 122 | } 123 | 124 | fn serialize_unit_variant( 125 | self, 126 | _name: &str, 127 | _variant_index: u32, 128 | _variant: &str, 129 | ) -> Result { 130 | Err(Error::Unsupported("Serializer::serialize_unit_variant")) 131 | } 132 | 133 | fn serialize_newtype_struct( 134 | self, 135 | _name: &str, 136 | _value: &T, 137 | ) -> Result { 138 | Err(Error::Unsupported("Serializer::serialize_newtype_struct")) 139 | } 140 | 141 | fn serialize_newtype_variant( 142 | self, 143 | _name: &str, 144 | variant_index: u32, 145 | _variant: &str, 146 | value: &T, 147 | ) -> Result { 148 | self.writer.write_all(&variant_index.to_be_bytes())?; 149 | value.serialize(self) 150 | } 151 | 152 | fn serialize_seq(self, _len: Option) -> Result { 153 | Ok(self) 154 | } 155 | 156 | fn serialize_tuple(self, _len: usize) -> Result { 157 | Ok(self) 158 | } 159 | 160 | fn serialize_tuple_struct( 161 | self, 162 | _name: &str, 163 | _len: usize, 164 | ) -> Result { 165 | Ok(self) 166 | } 167 | 168 | fn serialize_tuple_variant( 169 | self, 170 | _name: &str, 171 | _variant_index: u32, 172 | _variant: &str, 173 | _len: usize, 174 | ) -> Result { 175 | Ok(self) 176 | } 177 | 178 | fn serialize_map(self, _len: Option) -> Result { 179 | Ok(self) 180 | } 181 | 182 | fn serialize_struct( 183 | self, 184 | _name: &str, 185 | _len: usize, 186 | ) -> Result { 187 | Ok(self) 188 | } 189 | 190 | fn serialize_struct_variant( 191 | self, 192 | _name: &str, 193 | _variant_index: u32, 194 | _variant: &str, 195 | _len: usize, 196 | ) -> Result { 197 | Ok(self) 198 | } 199 | } 200 | 201 | impl<'a, W: Write> SerializeSeq for &'a mut SqliteSe { 202 | type Ok = (); 203 | type Error = Error; 204 | 205 | fn serialize_element(&mut self, _value: &T) -> Result 206 | where 207 | T: ?Sized + Serialize, 208 | { 209 | Err(Error::Unsupported("SerializeSeq::serialize_element")) 210 | } 211 | 212 | fn end(self) -> Result { 213 | Err(Error::Unsupported("SerializeSeq::end")) 214 | } 215 | } 216 | 217 | impl<'a, W: Write> SerializeTuple for &'a mut SqliteSe { 218 | type Ok = (); 219 | type Error = Error; 220 | 221 | fn serialize_element(&mut self, value: &T) -> Result 222 | where 223 | T: ?Sized + Serialize, 224 | { 225 | value.serialize(&mut **self) 226 | } 227 | 228 | fn end(self) -> Result { 229 | Ok(()) 230 | } 231 | } 232 | 233 | impl<'a, W: Write> SerializeTupleStruct for &'a mut SqliteSe { 234 | type Ok = (); 235 | type Error = Error; 236 | 237 | fn serialize_field(&mut self, _value: &T) -> Result 238 | where 239 | T: ?Sized + Serialize, 240 | { 241 | Err(Error::Unsupported("SerializeTupleStruct::serialize_field")) 242 | } 243 | 244 | fn end(self) -> Result { 245 | Err(Error::Unsupported("SerializeTupleStruct::end")) 246 | } 247 | } 248 | 249 | impl<'a, W: Write> SerializeTupleVariant for &'a mut SqliteSe { 250 | type Ok = (); 251 | type Error = Error; 252 | 253 | fn serialize_field(&mut self, _value: &T) -> Result 254 | where 255 | T: ?Sized + Serialize, 256 | { 257 | Err(Error::Unsupported("SerializeTupleVariant::serialize_field")) 258 | } 259 | 260 | fn end(self) -> Result { 261 | Err(Error::Unsupported("SerializeTupleVariant::end")) 262 | } 263 | } 264 | 265 | impl<'a, W: Write> SerializeMap for &'a mut SqliteSe { 266 | type Ok = (); 267 | type Error = Error; 268 | 269 | fn serialize_key(&mut self, _key: &T) -> Result 270 | where 271 | T: ?Sized + Serialize, 272 | { 273 | Err(Error::Unsupported("SerializeMap::serialize_key")) 274 | } 275 | 276 | fn serialize_value(&mut self, _value: &T) -> Result 277 | where 278 | T: ?Sized + Serialize, 279 | { 280 | Err(Error::Unsupported("SerializeMap::serialize_value")) 281 | } 282 | 283 | fn end(self) -> Result { 284 | Err(Error::Unsupported("SerializeMap::end")) 285 | } 286 | } 287 | 288 | impl<'a, W: Write> SerializeStruct for &'a mut SqliteSe { 289 | type Ok = (); 290 | type Error = Error; 291 | 292 | fn serialize_field(&mut self, _key: &str, value: &T) -> Result 293 | where 294 | T: ?Sized + Serialize, 295 | { 296 | value.serialize(&mut **self) 297 | } 298 | 299 | fn end(self) -> Result { 300 | Ok(()) 301 | } 302 | } 303 | 304 | impl<'a, W: Write> SerializeStructVariant for &'a mut SqliteSe { 305 | type Ok = (); 306 | type Error = Error; 307 | 308 | fn serialize_field(&mut self, _key: &str, _value: &T) -> Result 309 | where 310 | T: ?Sized + Serialize, 311 | { 312 | Err(Error::Unsupported( 313 | "SerializeStructVariant::serialize_field", 314 | )) 315 | } 316 | 317 | fn end(self) -> Result { 318 | Err(Error::Unsupported("SerializeStructVariant::end")) 319 | } 320 | } 321 | 322 | // serialize None as zero 323 | pub fn none_as_zero(field: &Option, s: S) -> Result 324 | where 325 | S: Serializer, 326 | T: Serialize + Copy + Default, 327 | { 328 | let value = match field.as_ref() { 329 | None => T::default(), 330 | Some(&v) => v, 331 | }; 332 | value.serialize(s) 333 | } 334 | 335 | struct CountingBufWriter { 336 | writer: BufWriter, 337 | written: usize, 338 | block_size: usize, 339 | } 340 | 341 | impl CountingBufWriter { 342 | fn new(writer: W, block_size: usize) -> Self { 343 | Self { 344 | writer: BufWriter::new(writer), 345 | written: 0, 346 | block_size, 347 | } 348 | } 349 | 350 | fn pad(&mut self) -> std::io::Result<()> { 351 | let mut left = self.block_size - self.written; 352 | if left == 0 { 353 | return Ok(()); 354 | } 355 | let buf_size = 4096; 356 | let mut buf = vec![0; 4096]; 357 | while left > 0 { 358 | let to_write = buf_size.min(left); 359 | // *safe* since vec is pre-allocated and initialized 360 | unsafe { buf.set_len(to_write) }; 361 | self.write_all(buf.as_mut_slice())?; 362 | left -= to_write 363 | } 364 | Ok(()) 365 | } 366 | } 367 | 368 | impl Write for CountingBufWriter { 369 | fn write(&mut self, buf: &[u8]) -> std::io::Result { 370 | if self.written + buf.len() > self.block_size { 371 | // FIXME: 372 | return Err(std::io::Error::new( 373 | std::io::ErrorKind::Other, 374 | "block size overflow", 375 | )); 376 | } 377 | let written = self.writer.write(buf)?; 378 | self.written += written; 379 | Ok(written) 380 | } 381 | 382 | fn flush(&mut self) -> std::io::Result<()> { 383 | self.writer.flush() 384 | } 385 | } 386 | 387 | pub fn to_bytes(value: &T) -> Result, Error> 388 | where 389 | T: Serialize + Block, 390 | { 391 | let mut buf = Vec::::new(); 392 | buf.try_reserve(value.iblock_size()) 393 | .map_err(Error::OutOfMemory)?; 394 | buf.resize(value.iblock_size(), 0); 395 | to_writer(buf.as_mut_slice(), value)?; 396 | Ok(buf) 397 | } 398 | 399 | pub fn to_writer(writer: W, value: &T) -> Result<(), Error> 400 | where 401 | T: Serialize + Block, 402 | { 403 | let mut cbw = CountingBufWriter::new(writer, value.iblock_size()); 404 | value.serialize(&mut SqliteSe { writer: &mut cbw })?; 405 | cbw.pad()?; 406 | Ok(cbw.flush()?) 407 | } 408 | -------------------------------------------------------------------------------- /serde_sqlite/tests/de_test.rs: -------------------------------------------------------------------------------- 1 | use block::block; 2 | use serde::Deserialize; 3 | use serde_sqlite::Error; 4 | use serde_sqlite::{from_bytes, from_reader}; 5 | 6 | #[derive(Debug, Deserialize, PartialEq)] 7 | #[block(64)] 8 | struct ValidStruct { 9 | b: bool, 10 | u_8: u8, 11 | u_16: u16, 12 | u_32: u32, 13 | u_64: u64, 14 | i_8: i8, 15 | i_16: i16, 16 | i_32: i32, 17 | i_64: i64, 18 | f_32: f32, 19 | f_64: f64, 20 | #[serde(deserialize_with = "serde_sqlite::de::zero_as_none")] 21 | n: Option, 22 | #[serde(deserialize_with = "serde_sqlite::de::zero_as_none")] 23 | s: Option, 24 | } 25 | 26 | #[test] 27 | #[rustfmt::skip] 28 | fn test_deserialization_from_bytes() { 29 | let block = &[ 30 | /* b */ 0x01, 31 | /* u_8 */ 0x02, 32 | /* u_16 */ 0x01, 0x02, 33 | /* u_32 */ 0x01, 0x02, 0x03, 0x04, 34 | /* u_64 */ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 35 | /* i_8 */ 0xff, 36 | /* i_16 */ 0xff, 0xfe, 37 | /* i_32 */ 0xff, 0xff, 0xff, 0xfd, 38 | /* i_64 */ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 39 | /* f_32 */ 0x80, 0x00, 0x00, 0x00, 40 | /* f_64 */ 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 41 | /* n */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 42 | /* s */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 43 | /* block */ 44 | /* padding */ 0x01, 0x02, 0x03, 0x04, 0x05 45 | ]; 46 | let decoded = from_bytes::(block); 47 | assert!(decoded.is_ok()); 48 | let decoded = decoded.unwrap(); 49 | let header = ValidStruct { 50 | b: true, 51 | u_8: 2, 52 | u_16: 0x0102_u16, 53 | u_32: 0x01020304_u32, 54 | u_64: 0x0102030405060708_u64, 55 | i_8: -1, 56 | i_16: -2, 57 | i_32: -3, 58 | i_64: -4, 59 | f_32: -0.0, 60 | f_64: f64::INFINITY, 61 | n: None, 62 | s: Some(1), 63 | }; 64 | assert_eq!(decoded, header); 65 | } 66 | 67 | #[test] 68 | #[rustfmt::skip] 69 | fn test_deserialization_from_reader() { 70 | let block = &[ 71 | /* b */ 0x01, 72 | /* u_8 */ 0x02, 73 | /* u_16 */ 0x01, 0x02, 74 | /* u_32 */ 0x01, 0x02, 0x03, 0x04, 75 | /* u_64 */ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 76 | /* i_8 */ 0xff, 77 | /* i_16 */ 0xff, 0xfe, 78 | /* i_32 */ 0xff, 0xff, 0xff, 0xfd, 79 | /* i_64 */ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 80 | /* f_32 */ 0x80, 0x00, 0x00, 0x00, 81 | /* f_64 */ 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 82 | /* n */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 83 | /* s */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 84 | /* block */ 85 | /* padding */ 0x00, 0x00, 0x00, 0x00, 0x00 86 | ]; 87 | let decoded = from_reader::(std::io::Cursor::new(block)); 88 | assert!(decoded.is_ok()); 89 | let decoded = decoded.unwrap(); 90 | let header = ValidStruct { 91 | b: true, 92 | u_8: 2, 93 | u_16: 0x0102_u16, 94 | u_32: 0x01020304_u32, 95 | u_64: 0x0102030405060708_u64, 96 | i_8: -1, 97 | i_16: -2, 98 | i_32: -3, 99 | i_64: -4, 100 | f_32: -0.0, 101 | f_64: f64::INFINITY, 102 | n: None, 103 | s: Some(1), 104 | }; 105 | assert_eq!(decoded, header); 106 | } 107 | 108 | #[test] 109 | #[rustfmt::skip] 110 | fn test_deserialization_error() { 111 | // incomplete block (padding is missing) 112 | let block = &[ 113 | /* b */ 0x01, 114 | /* u_8 */ 0x02, 115 | /* u_16 */ 0x01, 0x02, 116 | /* u_32 */ 0x01, 0x02, 0x03, 0x04, 117 | /* u_64 */ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 118 | /* i_8 */ 0xff, 119 | /* i_16 */ 0xff, 0xfe, 120 | /* i_32 */ 0xff, 0xff, 0xff, 0xfd, 121 | /* i_64 */ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 122 | /* f_32 */ 0x80, 0x00, 0x00, 0x00, 123 | /* f_64 */ 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 124 | /* n */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 125 | /* s */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 126 | ]; 127 | assert!(matches!(from_bytes::(block), Err(Error::IoError(_)))); 128 | assert!(matches!( 129 | from_reader::(std::io::Cursor::new(block)), 130 | Err(Error::IoError(_))) 131 | ); 132 | } 133 | 134 | #[derive(Debug, Deserialize, PartialEq)] 135 | #[block(8)] 136 | struct S {} 137 | 138 | #[derive(Debug, Deserialize, PartialEq)] 139 | #[block] 140 | enum A { 141 | F(ValidStruct), 142 | S(S), 143 | } 144 | 145 | #[derive(Debug, Deserialize, PartialEq)] 146 | #[block] 147 | enum B { 148 | A(A), 149 | } 150 | 151 | #[test] 152 | #[rustfmt::skip] 153 | fn test_deserialization_newtype_enum() { 154 | // test first variant 155 | let block = &[ 156 | /* tag */ 0x00, 0x00, 0x00, 0x00, 157 | /* b */ 0x01, 158 | /* u_8 */ 0x02, 159 | /* u_16 */ 0x01, 0x02, 160 | /* u_32 */ 0x01, 0x02, 0x03, 0x04, 161 | /* u_64 */ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 162 | /* i_8 */ 0xff, 163 | /* i_16 */ 0xff, 0xfe, 164 | /* i_32 */ 0xff, 0xff, 0xff, 0xfd, 165 | /* i_64 */ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 166 | /* f_32 */ 0x80, 0x00, 0x00, 0x00, 167 | /* f_64 */ 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 168 | /* n */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 169 | /* s */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 170 | /* block */ 171 | /* padding */ 0x01, 0x02, 0x03, 0x04, 0x05 172 | ]; 173 | let res = from_bytes::(block); 174 | assert!(res.is_ok(), "{res:?}"); 175 | let f = res.unwrap(); 176 | assert_eq!( 177 | A::F(ValidStruct{ 178 | b: true, 179 | u_8: 2, 180 | u_16: 0x0102_u16, 181 | u_32: 0x01020304_u32, 182 | u_64: 0x0102030405060708_u64, 183 | i_8: -1, 184 | i_16: -2, 185 | i_32: -3, 186 | i_64: -4, 187 | f_32: -0.0, 188 | f_64: f64::INFINITY, 189 | n: None, 190 | s: Some(1), 191 | }), 192 | f 193 | ); 194 | 195 | 196 | let block = &[ 197 | /* tag */ 0x00, 0x00, 0x00, 0x01, 198 | /* padding */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 199 | ]; 200 | let res = from_bytes::(block); 201 | assert!(res.is_ok(), "{res:?}"); 202 | let s = res.unwrap(); 203 | assert_eq!(A::S(S{}), s); 204 | 205 | 206 | let block = &[ 207 | /* tag B */ 0x00, 0x00, 0x00, 0x00, 208 | /* tag A */ 0x00, 0x00, 0x00, 0x01, 209 | /* padding */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 210 | ]; 211 | 212 | let res = from_bytes::(block); 213 | assert!(res.is_ok(), "{res:?}"); 214 | let b = res.unwrap(); 215 | assert_eq!(B::A(A::S(S{})), b); 216 | } 217 | -------------------------------------------------------------------------------- /serde_sqlite/tests/se_test.rs: -------------------------------------------------------------------------------- 1 | use block::{block, Block}; 2 | use serde::Serialize; 3 | use serde_sqlite::{to_bytes, to_writer, Error}; 4 | 5 | #[derive(Debug, Serialize)] 6 | #[block(64)] 7 | struct ValidStruct { 8 | b: bool, 9 | u_8: u8, 10 | u_16: u16, 11 | u_32: u32, 12 | u_64: u64, 13 | i_8: i8, 14 | i_16: i16, 15 | i_32: i32, 16 | i_64: i64, 17 | f_32: f32, 18 | f_64: f64, 19 | #[serde(serialize_with = "serde_sqlite::se::none_as_zero")] 20 | n: Option, 21 | #[serde(serialize_with = "serde_sqlite::se::none_as_zero")] 22 | s: Option, 23 | } 24 | 25 | #[test] 26 | #[rustfmt::skip] 27 | fn test_valid_serialization() { 28 | let header = ValidStruct { 29 | b: true, 30 | u_8: 2, 31 | u_16: 0x0102_u16, 32 | u_32: 0x01020304_u32, 33 | u_64: 0x0102030405060708_u64, 34 | i_8: -1, 35 | i_16: -2, 36 | i_32: -3, 37 | i_64: -4, 38 | f_32: -0.0, 39 | f_64: f64::INFINITY, 40 | n: None, 41 | s: Some(1), 42 | }; 43 | // to_bytes 44 | let res = to_bytes(&header); 45 | assert!(res.is_ok(), "{res:?}"); 46 | 47 | let header = res.unwrap(); 48 | assert_eq!(header.len(), ValidStruct::block_size()); 49 | assert_eq!( 50 | header.as_slice(), 51 | &[ 52 | /* b */ 0x01, 53 | /* u_8 */ 0x02, 54 | /* u_16 */ 0x01, 0x02, 55 | /* u_32 */ 0x01, 0x02, 0x03, 0x04, 56 | /* u_64 */ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 57 | /* i_8 */ 0xff, 58 | /* i_16 */ 0xff, 0xfe, 59 | /* i_32 */ 0xff, 0xff, 0xff, 0xfd, 60 | /* i_64 */ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 61 | /* f_32 */ 0x80, 0x00, 0x00, 0x00, 62 | /* f_64 */ 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 63 | /* n */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 64 | /* s */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 65 | /* block */ 66 | /* padding */ 0x00, 0x00, 0x00, 0x00, 0x00, 67 | ] 68 | ); 69 | } 70 | 71 | #[test] 72 | #[rustfmt::skip] 73 | fn test_valid_serialization_to_writer() { 74 | let header = ValidStruct { 75 | b: true, 76 | u_8: 2, 77 | u_16: 0x0102_u16, 78 | u_32: 0x01020304_u32, 79 | u_64: 0x0102030405060708_u64, 80 | i_8: -1, 81 | i_16: -2, 82 | i_32: -3, 83 | i_64: -4, 84 | f_32: -0.0, 85 | f_64: f64::INFINITY, 86 | n: None, 87 | s: Some(1), 88 | }; 89 | 90 | let mut buf = vec![0xff; 72]; 91 | let res = to_writer(buf.as_mut_slice(), &header); 92 | assert!(res.is_ok(), "{res:?}"); 93 | 94 | assert_eq!( 95 | buf.as_slice(), 96 | &[ 97 | /* b */ 0x01, 98 | /* u_8 */ 0x02, 99 | /* u_16 */ 0x01, 0x02, 100 | /* u_32 */ 0x01, 0x02, 0x03, 0x04, 101 | /* u_64 */ 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 102 | /* i_8 */ 0xff, 103 | /* i_16 */ 0xff, 0xfe, 104 | /* i_32 */ 0xff, 0xff, 0xff, 0xfd, 105 | /* i_64 */ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 106 | /* f_32 */ 0x80, 0x00, 0x00, 0x00, 107 | /* f_64 */ 0x7f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 108 | /* n */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 109 | /* s */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 110 | /* block */ 111 | /* padding */ 0x00, 0x00, 0x00, 0x00, 0x00, 112 | /* extra */ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 113 | ] 114 | ); 115 | } 116 | 117 | #[derive(Debug, Serialize)] 118 | #[block(4)] 119 | struct InvalidStruct { 120 | v: u64, 121 | } 122 | 123 | #[test] 124 | /// serialized struct contains more bytes than size provided to block macro 125 | fn test_invalid_serialization() { 126 | let err = to_bytes(&InvalidStruct { v: 0 }); 127 | assert!(matches!(err, Err(Error::IoError(_)))); 128 | let err = err.unwrap_err(); 129 | assert_eq!( 130 | err.to_string(), 131 | "IoError(Custom { kind: Other, error: \"block size overflow\" })" 132 | ); 133 | } 134 | 135 | #[test] 136 | fn test_invalid_serialization_to_writer() { 137 | let mut buf = vec![0xff; 128]; 138 | let err = to_writer(buf.as_mut_slice(), &InvalidStruct { v: 0 }); 139 | assert!(matches!(err, Err(Error::IoError(_)))); 140 | let err = err.unwrap_err(); 141 | assert_eq!( 142 | err.to_string(), 143 | "IoError(Custom { kind: Other, error: \"block size overflow\" })" 144 | ); 145 | } 146 | 147 | // enum serialization 148 | 149 | #[derive(Serialize)] 150 | #[block(32)] 151 | struct FirstVariant { 152 | f: u64, 153 | s: u32, 154 | t: [u8; 2], 155 | } 156 | 157 | #[derive(Serialize)] 158 | #[block(16)] 159 | struct SecondVariant { 160 | f: i64, 161 | s: i64, 162 | } 163 | 164 | // newtype enum, wraps existing structures 165 | // serializer needs to produce blocks of different size + prefix to hold enum discriminant 166 | #[derive(Serialize)] 167 | #[block] 168 | enum NewTypeEnum { 169 | F(FirstVariant), 170 | S(SecondVariant), 171 | } 172 | 173 | #[test] 174 | #[rustfmt::skip] 175 | fn test_enum_newtype_serialization() { 176 | let fv = NewTypeEnum::F(FirstVariant { f: 0, s: 1, t: [0; 2] }); 177 | let fv_res= serde_sqlite::to_bytes(&fv); 178 | assert!(fv_res.is_ok()); 179 | let fv_bytes = fv_res.unwrap(); 180 | assert_eq!(fv_bytes.len(), fv.iblock_size()); 181 | assert_eq!( 182 | fv_bytes.as_slice(), 183 | &[ 184 | /* tag */ 0x00, 0x00, 0x00, 0x00, 185 | /* f */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 186 | /* s */ 0x00, 0x00, 0x00, 0x01, 187 | /* t */ 0x00, 0x00, 188 | 189 | /* block */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 190 | /* */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 191 | /* padding */ 0x00, 0x00 192 | ] 193 | ); 194 | 195 | let sv = NewTypeEnum::S(SecondVariant{ f: 0, s: 1}); 196 | let sv_res= serde_sqlite::to_bytes(&sv); 197 | assert!(sv_res.is_ok()); 198 | let sv_bytes = sv_res.unwrap(); 199 | assert_eq!( 200 | sv_bytes.as_slice(), 201 | &[ 202 | /* tag */ 0x00, 0x00, 0x00, 0x01, 203 | /* f */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 204 | /* s */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 205 | ] 206 | ); 207 | assert_eq!(sv_bytes.len(), sv.iblock_size()); 208 | } 209 | -------------------------------------------------------------------------------- /utils/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "utils" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | 10 | [dev-dependencies] 11 | quickcheck = "1.0.3" -------------------------------------------------------------------------------- /utils/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::iter; 2 | 3 | const GAP: usize = 16; 4 | 5 | pub fn get_diff<'a>( 6 | new_page: &'a [u8], 7 | old_page: &'a [u8], 8 | ) -> impl Iterator + 'a { 9 | let iter = old_page 10 | .iter() 11 | .chain(iter::repeat::<&u8>(&0)) 12 | .zip(new_page) 13 | .map(|(&old, &new)| (old, new)) 14 | .enumerate(); 15 | 16 | Diff { 17 | iter, 18 | gap: GAP, 19 | range: None, 20 | } 21 | .map(|(start, end)| (start, &new_page[start..=end])) 22 | } 23 | 24 | pub struct Diff { 25 | iter: I, 26 | gap: usize, 27 | range: Option<(usize, usize)>, 28 | } 29 | 30 | impl Iterator for Diff 31 | where 32 | I: Iterator, 33 | { 34 | type Item = (usize, usize); 35 | 36 | fn next(&mut self) -> Option { 37 | for item in self.iter.by_ref() { 38 | match item { 39 | (i, (old, new)) if old != new => { 40 | self.range = match self.range { 41 | None => Some((i, i)), 42 | Some((start, _)) => Some((start, i)), 43 | } 44 | } 45 | (i, _) => match self.range { 46 | Some((_, end)) if end + self.gap < i => { 47 | return self.range.take(); 48 | } 49 | _ => {} 50 | }, 51 | } 52 | } 53 | self.range.take() 54 | } 55 | } 56 | 57 | #[cfg(test)] 58 | mod tests { 59 | use super::*; 60 | use quickcheck::{quickcheck, TestResult}; 61 | 62 | #[test] 63 | fn it_works() { 64 | let expected: Vec<(usize, &[u8])> = vec![]; 65 | let results = get_diff(&[], &[]); 66 | assert_eq!(results.collect::>(), expected); 67 | } 68 | 69 | #[test] 70 | fn test_it_works_with_actual_data() { 71 | let old_page: &[u8] = &[ 72 | 83, 81, 76, 105, 116, 101, 32, 102, 111, 114, 109, 97, 116, 32, 51, 0, 16, 0, 1, 1, 0, 73 | 64, 32, 32, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 74 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 46, 99, 1, 13, 0, 0, 0, 1, 15, 76 | 201, 0, 15, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1, 6, 23, 21, 21, 79 | 1, 79, 116, 97, 98, 108, 101, 116, 101, 115, 116, 116, 101, 115, 116, 2, 67, 82, 69, 80 | 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 116, 101, 115, 116, 40, 110, 117, 109, 98, 101, 81 | 114, 32, 105, 110, 116, 101, 103, 101, 114, 41, 82 | ]; 83 | let new_page: &[u8] = &[ 84 | 83, 81, 76, 105, 116, 101, 32, 102, 111, 114, 109, 97, 116, 32, 51, 0, 16, 0, 1, 1, 0, 85 | 64, 32, 32, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 86 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 46, 99, 1, 13, 0, 0, 0, 1, 15, 88 | 201, 0, 15, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1, 6, 23, 21, 21, 91 | 1, 79, 116, 97, 98, 108, 101, 116, 101, 115, 116, 116, 101, 115, 116, 2, 67, 82, 69, 92 | 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 116, 101, 115, 116, 40, 110, 117, 109, 98, 101, 93 | 114, 32, 105, 110, 116, 101, 103, 101, 114, 41, 94 | ]; 95 | let results = get_diff(new_page, old_page); 96 | let expected: Vec<(usize, &[u8])> = vec![(27, &[5]), (95, &[5])]; 97 | assert_eq!(results.collect::>(), expected); 98 | } 99 | 100 | #[test] 101 | fn test_it_works_with_small_gap_between_changed_values() { 102 | let old_page: &[u8] = &[ 103 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 104 | ]; 105 | let new_page: &[u8] = &[ 106 | 0, 1, 20, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 190, 20, 107 | ]; 108 | let results = get_diff(new_page, old_page); 109 | let expected: Vec<(usize, &[u8])> = vec![( 110 | 2, 111 | &[ 112 | 20, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 190, 113 | ], 114 | )]; 115 | 116 | assert_eq!(results.collect::>(), expected); 117 | } 118 | #[test] 119 | fn test_it_works_with_big_gap_between_changed_values() { 120 | let old_page: &[u8] = &[ 121 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 122 | ]; 123 | let new_page: &[u8] = &[ 124 | 0, 1, 20, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 200, 125 | ]; 126 | 127 | let results = get_diff(new_page, old_page); 128 | let expected: Vec<(usize, &[u8])> = vec![(2, &[20]), (20, &[200])]; 129 | 130 | assert_eq!(results.collect::>(), expected); 131 | } 132 | 133 | #[test] 134 | fn test_it_works_with_values_at_end_changed() { 135 | let old_page: &[u8] = &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; 136 | let new_page: &[u8] = &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]; 137 | 138 | let results = get_diff(new_page, old_page); 139 | let expected: Vec<(usize, &[u8])> = vec![(11, &[1])]; 140 | 141 | assert_eq!(results.collect::>(), expected); 142 | } 143 | 144 | #[test] 145 | fn test_it_works_with_empty_old_page_and_new_page_all_zeros() { 146 | let old_page: &[u8] = &[]; 147 | let new_page: &[u8] = &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; 148 | 149 | let results = get_diff(new_page, old_page); 150 | let expected: Vec<(usize, &[u8])> = vec![]; 151 | 152 | assert_eq!(results.collect::>(), expected); 153 | } 154 | 155 | #[test] 156 | fn test_it_works_with_empty_old_page() { 157 | let old_page: &[u8] = &[]; 158 | let new_page: &[u8] = &[0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0]; 159 | 160 | let results = get_diff(new_page, old_page); 161 | let expected: Vec<(usize, &[u8])> = vec![(3, &[1, 1, 1, 0, 1])]; 162 | 163 | assert_eq!(results.collect::>(), expected); 164 | } 165 | 166 | quickcheck! { 167 | fn prop_get_diff_when_pages_exist(new: Vec, old: Vec) -> TestResult { 168 | if new.len() != old.len() { 169 | return TestResult::discard(); 170 | } 171 | let diff = get_diff(&new, &old); 172 | let mut brand_new = old.clone(); 173 | 174 | for (offset, bytes) in diff { 175 | for (i, val) in bytes.iter().enumerate() { 176 | brand_new[offset + i] = *val; 177 | } 178 | } 179 | TestResult::from_bool(new == brand_new) 180 | } 181 | 182 | fn prop_get_diff_when_old_page_not_exists(new: Vec) -> TestResult { 183 | let old: Vec = vec![]; 184 | let diff = get_diff(&new, &old); 185 | let mut brand_new = vec![0; new.len()]; 186 | for (offset, bytes) in diff { 187 | for (i, val) in bytes.iter().enumerate() { 188 | brand_new[offset + i] = *val; 189 | } 190 | } 191 | TestResult::from_bool(new == brand_new) 192 | } 193 | } 194 | } 195 | --------------------------------------------------------------------------------