├── .github ├── dependabot.yml └── workflows │ ├── ci.yml │ └── dependabot.yml ├── .gitignore ├── .pre-commit-config.yaml ├── Cargo.lock ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── benches └── bench.rs ├── clippy.toml ├── justfile ├── src ├── brotli.rs ├── bsdiff4.rs ├── bsdiffraw.rs ├── bzip2.rs ├── cdylib │ └── cdylib.rs ├── common.rs ├── common_diff.rs ├── gzip.rs └── lib.rs └── tests ├── test-ext.sh └── test.rs /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | 4 | # Maintain dependencies for GitHub Actions 5 | - package-ecosystem: github-actions 6 | directory: "/" 7 | schedule: 8 | interval: weekly 9 | groups: 10 | all-actions-version-updates: 11 | applies-to: version-updates 12 | patterns: 13 | - "*" 14 | all-actions-security-updates: 15 | applies-to: security-updates 16 | patterns: 17 | - "*" 18 | 19 | # Update Rust dependencies 20 | - package-ecosystem: cargo 21 | directory: "/" 22 | schedule: 23 | interval: daily 24 | time: "02:00" 25 | open-pull-requests-limit: 10 26 | groups: 27 | all-cargo-version-updates: 28 | applies-to: version-updates 29 | patterns: 30 | - "*" 31 | all-cargo-security-updates: 32 | applies-to: security-updates 33 | patterns: 34 | - "*" 35 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | release: 9 | types: [ published ] 10 | workflow_dispatch: 11 | 12 | defaults: 13 | run: 14 | shell: bash 15 | 16 | jobs: 17 | test: 18 | name: Test 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | - if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' 23 | uses: Swatinem/rust-cache@v2 24 | - uses: taiki-e/install-action@v2 25 | with: { tool: just } 26 | - if: github.event_name == 'release' 27 | name: Ensure this crate has not yet been published (on release) 28 | run: just check-if-published 29 | - run: just ci-test 30 | - name: Check semver 31 | uses: obi1kenobi/cargo-semver-checks-action@v2 32 | 33 | test-msrv: 34 | name: Test MSRV 35 | runs-on: ubuntu-latest 36 | steps: 37 | - uses: actions/checkout@v4 38 | - if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' 39 | uses: Swatinem/rust-cache@v2 40 | - uses: taiki-e/install-action@v2 41 | with: { tool: just } 42 | - name: Read MSRV 43 | id: msrv 44 | run: echo "value=$(just get-msrv)" >> $GITHUB_OUTPUT 45 | - name: Install MSRV Rust ${{ steps.msrv.outputs.value }} 46 | uses: dtolnay/rust-toolchain@stable 47 | with: 48 | toolchain: ${{ steps.msrv.outputs.value }} 49 | components: rustfmt 50 | - run: just ci_mode=0 ci-test-msrv # Ignore warnings in MSRV 51 | 52 | coverage: 53 | name: Code Coverage 54 | if: github.event_name != 'release' 55 | runs-on: ubuntu-latest 56 | steps: 57 | - uses: actions/checkout@v4 58 | - uses: Swatinem/rust-cache@v2 59 | - uses: taiki-e/install-action@v2 60 | with: { tool: 'just,cargo-llvm-cov' } 61 | - name: Generate code coverage 62 | run: just ci-coverage 63 | - name: Upload coverage to Codecov 64 | uses: codecov/codecov-action@v5 65 | with: 66 | token: ${{ secrets.CODECOV_TOKEN }} 67 | files: target/llvm-cov/codecov.info 68 | fail_ci_if_error: true 69 | 70 | build: 71 | name: Build ${{ matrix.target }} 72 | needs: [ test ] 73 | runs-on: ${{ matrix.os }} 74 | strategy: 75 | fail-fast: false 76 | matrix: 77 | include: 78 | - target: aarch64-apple-darwin 79 | file: libsqlite_compressions.dylib 80 | download: 'https://www.sqlite.org/2023/sqlite-tools-osx-x64-3440200.zip' 81 | os: macOS-latest 82 | sqlite3: ./sqlite3 83 | - target: x86_64-apple-darwin 84 | file: libsqlite_compressions.dylib 85 | download: 'https://www.sqlite.org/2023/sqlite-tools-osx-x64-3440200.zip' 86 | os: macOS-latest 87 | sqlite3: ./sqlite3 88 | - target: x86_64-pc-windows-msvc 89 | file: sqlite_compressions.dll 90 | download: 'https://www.sqlite.org/2023/sqlite-tools-win-x64-3440200.zip' 91 | os: windows-latest 92 | sqlite3: ./sqlite3.exe 93 | - target: x86_64-unknown-linux-gnu 94 | file: libsqlite_compressions.so 95 | os: ubuntu-latest 96 | sqlite3: sqlite3 97 | steps: 98 | - uses: actions/checkout@v4 99 | - if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' 100 | uses: Swatinem/rust-cache@v2 101 | - uses: taiki-e/install-action@v2 102 | with: { tool: just } 103 | - if: matrix.download 104 | name: Download SQLite 105 | uses: carlosperate/download-file-action@v2 106 | with: 107 | file-url: '${{ matrix.download }}' 108 | file-name: sqlite.zip 109 | location: ./tmp-downloads 110 | - if: matrix.download 111 | name: Install SQLite 112 | run: | 113 | cd tmp-downloads 114 | if [[ "${{ runner.os }}" == "Windows" ]]; then 115 | 7z x sqlite.zip 116 | else 117 | unzip sqlite.zip 118 | chmod +x ${{ matrix.sqlite3 }} 119 | fi 120 | mv ${{ matrix.sqlite3 }} ../ 121 | cd .. 122 | rm -rf ./tmp-downloads 123 | - name: SQLite Info 124 | run: | 125 | which ${{ matrix.sqlite3 }} 126 | ${{ matrix.sqlite3 }} --version 127 | ${{ matrix.sqlite3 }} <"] 6 | repository = "https://github.com/nyurik/sqlite-compressions" 7 | edition = "2021" 8 | license = "MIT OR Apache-2.0" 9 | keywords = ["sqlite", "compression", "gzip", "brotli", "bsdiff"] 10 | categories = ["database", "compression"] 11 | rust-version = "1.82" 12 | 13 | [lib] 14 | name = "sqlite_compressions" 15 | 16 | # Loadable extension is a cdylib (lib), but Rust does not allow multiple libs per crate, so using an example instead. 17 | # See https://github.com/rust-lang/cargo/issues/8628 18 | [[example]] 19 | name = "sqlite_compressions" 20 | path = "src/cdylib/cdylib.rs" 21 | crate-type = ["cdylib"] 22 | required-features = ["loadable_extension"] 23 | 24 | [features] 25 | default = ["trace", "brotli", "bsdiff4", "bsdiffraw", "bzip2", "gzip"] 26 | # Use this feature to build loadable extension. 27 | # Assumes --no-default-features. 28 | default_loadable_extension = ["loadable_extension", "brotli", "bsdiff4", "bsdiffraw", "bzip2", "gzip"] 29 | # 30 | # Enable Trace Logging 31 | trace = ["dep:log"] 32 | # 33 | # Build loadable extension. 34 | # See https://github.com/rusqlite/rusqlite/discussions/1423 35 | # This feature does not work with "rusqlite/modern_sqlite" 36 | loadable_extension = ["rusqlite/loadable_extension", "rusqlite/trace"] 37 | # 38 | # Encoding algorithms 39 | brotli = ["dep:brotli"] 40 | bsdiff4 = ["dep:qbsdiff"] 41 | bsdiffraw = ["dep:bsdiff"] 42 | bzip2 = ["dep:bzip2"] 43 | gzip = ["dep:flate2"] 44 | 45 | [dependencies] 46 | brotli = { version = ">=5.0, <9.0", optional = true } 47 | bsdiff = { version = "0.2.1", optional = true } 48 | bzip2 = { version = "0.5.2", optional = true } 49 | flate2 = { version = "1.1.1", optional = true } 50 | log = { version = "0.4.27", optional = true } 51 | qbsdiff = { version = "1.4.3", optional = true } 52 | 53 | # There are multiple versions that could work. However, sqlx requires a specific one, so don't limit it here 54 | # Note that cdylib requires >= 0.32.0 (controlled by the lock file) 55 | # The `set-min-rusqlite-version` just recipe will parse the minimum version from here, so it must be 3 part 56 | rusqlite = { version = ">=0.30.0", features = ["functions"] } 57 | 58 | [dev-dependencies] 59 | criterion = { version = "0.6", features = ["html_reports"] } 60 | ctor = "0.4" 61 | env_logger = "0.11" 62 | hex = "0.4" 63 | insta = { version = "1", features = [] } 64 | rstest = "0.25" 65 | 66 | [lints.rust] 67 | unused_qualifications = "warn" 68 | 69 | [lints.clippy] 70 | cargo = { level = "warn", priority = -1 } 71 | pedantic = { level = "warn", priority = -1 } 72 | missing_errors_doc = "allow" 73 | module_name_repetitions = "allow" 74 | 75 | [[bench]] 76 | name = "bench" 77 | harness = false 78 | 79 | #[patch.crates-io] 80 | #rusqlite = { path = "../rusqlite" } 81 | #libsqlite3-sys = { path = "../rusqlite/libsqlite3-sys" } 82 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023-2025 Yuri Astrakhan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sqlite-compressions 2 | 3 | [![GitHub](https://img.shields.io/badge/github-sqlite--compressions-8da0cb?logo=github)](https://github.com/nyurik/sqlite-compressions) 4 | [![crates.io version](https://img.shields.io/crates/v/sqlite-compressions.svg)](https://crates.io/crates/sqlite-compressions) 5 | [![docs.rs docs](https://docs.rs/sqlite-compressions/badge.svg)](https://docs.rs/sqlite-compressions) 6 | [![license](https://img.shields.io/crates/l/sqlite-compressions.svg)](https://github.com/nyurik/sqlite-compressions/blob/main/LICENSE-APACHE) 7 | [![CI build](https://github.com/nyurik/sqlite-compressions/actions/workflows/ci.yml/badge.svg)](https://github.com/nyurik/sqlite-compressions/actions) 8 | [![Codecov](https://img.shields.io/codecov/c/github/nyurik/sqlite-compressions)](https://app.codecov.io/gh/nyurik/sqlite-compressions) 9 | 10 | Implement `SQLite` compression, decompression, and testing functions for Brotli, bzip2, and gzip encodings, as well as 11 | [bsdiff4](https://github.com/mendsley/bsdiff#readme) and [raw bsdiff](https://github.com/space-wizards/bsdiff-rs#readme) 12 | binary diffing and patching support. 13 | Functions are available as a loadable extension, or as a Rust library. 14 | 15 | See also [SQLite-hashes](https://github.com/nyurik/sqlite-hashes) extension for `MD5, SHA1, SHA224, SHA256, SHA384, 16 | SHA512, FNV1a, xxHash` hashing functions. 17 | 18 | ## Usage 19 | 20 | This `SQLite` extension adds functions for brotli, bzip2, and gzip compressions like `gzip(data, [quality])`, 21 | decoding `gzip_decode(data)`, and testing `gzip_test(data)` functions. Both encoding and decoding functions return 22 | blobs, and the 23 | testing function returns a true/false. The encoding functions can encode text and blob values, but will raise an error 24 | on other types like integers and floating point numbers. All functions will return `NULL` if the input data is `NULL`. 25 | 26 | `bsdiff4(source, target)` will return a binary diff between two blobs, and `bspatch4(source, diff)` will apply the diff 27 | to the source blob to produce the target blob. The diff and patch functions will raise an error if the input data is not 28 | blobs or if the diff is invalid. If either input is `NULL`, the diff and patch functions will return `NULL`. 29 | 30 | Similar `bsdiffraw(source, target)` and `bspatchraw(source, diff)` functions are available for raw bsdiff format. Raw 31 | format is not compressed and does not have any magic number prefix. If the internal format provided 32 | by [bsdiff crate](https://github.com/space-wizards/bsdiff-rs#readme) changes, we will add a separate function for it. 33 | 34 | ### Extension 35 | 36 | To use as an extension, load the `libsqlite_compressions.so` shared library into `SQLite`. 37 | 38 | ```bash 39 | $ sqlite3 40 | sqlite> .load ./libsqlite_compressions 41 | sqlite> SELECT hex(brotli('Hello world!')); 42 | 8B058048656C6C6F20776F726C642103 43 | sqlite> SELECT brotli_decode(x'8B058048656C6C6F20776F726C642103'); 44 | Hello world! 45 | sqlite> SELECT brotli_test(x'8B058048656C6C6F20776F726C642103'); 46 | 1 47 | ``` 48 | 49 | ### Rust library 50 | 51 | To use as a Rust library, add `sqlite-compressions` to your `Cargo.toml` dependencies. Then, register the needed 52 | functions with `register_compression_functions(&db)`. This will register all available functions, or you can 53 | use `register_gzip_functions(&db)`, `register_brotli_functions(&db)`, `register_bzip2_functions(&db)` to register just 54 | the needed ones (you may also 55 | disable the default features to reduce compile time and binary size). 56 | 57 | ```rust 58 | use sqlite_compressions::{register_compression_functions, rusqlite::Connection}; 59 | 60 | // Connect to SQLite DB and register needed functions 61 | let db = Connection::open_in_memory().unwrap(); 62 | // can also use encoding-specific ones like register_gzip_functions(&db) 63 | register_compression_functions(&db).unwrap(); 64 | 65 | // Encode 'password' using GZIP, and dump resulting BLOB as a HEX string 66 | let sql = "SELECT hex(gzip('password'));"; 67 | let res: String = db.query_row_and_then(&sql, [], |r| r.get(0)).unwrap(); 68 | assert_eq!(res, "1F8B08000000000000FF2B482C2E2ECF2F4A0100D546C23508000000"); 69 | 70 | // Encode 'password' using Brotli, decode it, and convert the blob to text 71 | let sql = "SELECT CAST(brotli_decode(brotli('password')) AS TEXT);"; 72 | let res: String = db.query_row_and_then(&sql, [], |r| r.get(0)).unwrap(); 73 | assert_eq!(res, "password"); 74 | 75 | // Test that Brotli-encoded value is correct. 76 | let sql = "SELECT brotli_test(brotli('password'));"; 77 | let res: bool = db.query_row_and_then(&sql, [], |r| r.get(0)).unwrap(); 78 | assert!(res); 79 | 80 | // Test that diffing source and target blobs can be applied to source to get target. 81 | let sql = "SELECT bspatch4('source', bsdiff4('source', 'target'));"; 82 | let res: Vec = db.query_row_and_then(&sql, [], |r| r.get(0)).unwrap(); 83 | assert_eq!(res, b"target"); 84 | 85 | // Test that diffing source and target blobs can be applied 86 | // to source to get target when using raw bsdiff format. 87 | let sql = "SELECT bspatchraw('source', bsdiffraw('source', 'target'));"; 88 | let res: Vec = db.query_row_and_then(&sql, [], |r| r.get(0)).unwrap(); 89 | assert_eq!(res, b"target"); 90 | ``` 91 | 92 | #### Using with `SQLx` 93 | 94 | To use with [SQLx](https://crates.io/crates/sqlx), you need to get the raw handle from the 95 | `SqliteConnection` and pass it to the registration function. 96 | 97 | ```rust,ignore 98 | use rusqlite::Connection; 99 | use sqlite_compressions::register_compression_functions; 100 | use sqlx::sqlite::SqliteConnection; 101 | 102 | async fn register_functions(sqlx_conn: &mut SqliteConnection) { 103 | // SAFETY: No query must be performed on `sqlx_conn` until `handle_lock` is dropped. 104 | let mut handle_lock = sqlx_conn.lock_handle().await.unwrap(); 105 | let handle = handle_lock.as_raw_handle().as_ptr(); 106 | 107 | // SAFETY: this is safe as long as handle_lock is valid. 108 | let rusqlite_conn = unsafe { Connection::from_handle(handle) }.unwrap(); 109 | 110 | // Registration is attached to the connection, not to rusqlite_conn, 111 | // so it will be available for the entire lifetime of the `sqlx_conn`. 112 | // Registration will be automatically dropped when SqliteConnection is dropped. 113 | register_compression_functions(&rusqlite_conn).unwrap(); 114 | } 115 | ``` 116 | 117 | ## Crate features 118 | 119 | By default, this crate will compile with all features. You can enable just the ones you need to reduce compile time and 120 | binary size. 121 | 122 | ```toml 123 | [dependencies] 124 | sqlite-compressions = { version = "0.3", default-features = false, features = ["brotli"] } 125 | ``` 126 | 127 | * **trace** - enable tracing support, logging all function calls and their arguments 128 | * **brotli** - enable Brotli compression support 129 | * **bzip2** - enable bzip2 compression support 130 | * **gzip** - enable GZIP compression support 131 | * **bsdiff4** - enable bsdiff4 binary diffing and patching support 132 | * **bsdiffraw** - enable bsdiff binary diffing and patching support using raw format 133 | 134 | The **`loadable_extension`** feature should only be used when building 135 | a `.so` / `.dylib` / `.dll` extension file that can be loaded directly into sqlite3 executable. 136 | 137 | ## Development 138 | 139 | * You must install `sqlite3` and `libsqlite3-dev`, e.g. `sudo apt install -y libsqlite3-dev sqlite3` on Ubuntu/Mint. 140 | * This project is easier to develop with [just](https://github.com/casey/just#readme), a modern alternative to `make`. 141 | Install it with `cargo install just`. 142 | * To get a list of available commands, run `just`. 143 | * To run tests, use `just test`. 144 | 145 | ## License 146 | 147 | Licensed under either of 148 | 149 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or ) 150 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or ) 151 | at your option. 152 | 153 | ### Contribution 154 | 155 | Unless you explicitly state otherwise, any contribution intentionally 156 | submitted for inclusion in the work by you, as defined in the 157 | Apache-2.0 license, shall be dual licensed as above, without any 158 | additional terms or conditions. 159 | -------------------------------------------------------------------------------- /benches/bench.rs: -------------------------------------------------------------------------------- 1 | use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; 2 | use sqlite_compressions::{BrotliEncoder, Bzip2Encoder, Encoder as _, GzipEncoder}; 3 | 4 | macro_rules! enc_test { 5 | ($func_name:ident, $enc_type:ident, $func:literal) => { 6 | fn $func_name(c: &mut Criterion) { 7 | let mut group = c.benchmark_group("test"); 8 | group.sample_size(60); 9 | for size in [10, 10 * 1024, 1024 * 1024] { 10 | let data = $enc_type::encode(gen_data(size).as_slice(), None).unwrap(); 11 | group.bench_function(BenchmarkId::new($func, size), |b| { 12 | b.iter(|| $enc_type::test(&data)); 13 | }); 14 | } 15 | group.finish(); 16 | } 17 | }; 18 | } 19 | 20 | enc_test!(gzip_test, GzipEncoder, "gzip"); 21 | enc_test!(brotli_test, BrotliEncoder, "brotli"); 22 | enc_test!(bzip2_test, Bzip2Encoder, "bzip2"); 23 | 24 | fn gen_data(size: usize) -> Vec { 25 | let mut byte_data: Vec = Vec::with_capacity(size); 26 | for i in 0..size { 27 | #[allow(clippy::cast_possible_truncation)] 28 | byte_data.push((i % 256) as u8); 29 | } 30 | byte_data 31 | } 32 | 33 | criterion_group!(benches, gzip_test, brotli_test, bzip2_test); 34 | criterion_main!(benches); 35 | -------------------------------------------------------------------------------- /clippy.toml: -------------------------------------------------------------------------------- 1 | allow-unwrap-in-tests = true 2 | avoid-breaking-exported-api = false 3 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env just --justfile 2 | 3 | main_crate := 'sqlite-compressions' 4 | bin_name := 'sqlite_compressions' 5 | sqlite3 := 'sqlite3' 6 | 7 | # if running in CI, treat warnings as errors by setting RUSTFLAGS and RUSTDOCFLAGS to '-D warnings' unless they are already set 8 | # Use `CI=true just ci-test` to run the same tests as in GitHub CI. 9 | # Use `just env-info` to see the current values of RUSTFLAGS and RUSTDOCFLAGS 10 | ci_mode := if env('CI', '') != '' { '1' } else { '' } 11 | export RUSTFLAGS := env('RUSTFLAGS', if ci_mode == '1' {'-D warnings'} else {''}) 12 | export RUSTDOCFLAGS := env('RUSTDOCFLAGS', if ci_mode == '1' {'-D warnings'} else {''}) 13 | export RUST_BACKTRACE := env('RUST_BACKTRACE', if ci_mode == '1' {'1'} else {''}) 14 | 15 | @_default: 16 | just --list 17 | 18 | # Run benchmarks 19 | bench: 20 | cargo bench 21 | open target/criterion/report/index.html 22 | 23 | # Run integration tests and save its output as the new expected output 24 | bless *args: (cargo-install 'cargo-insta') 25 | cargo insta test --accept --unreferenced=delete {{args}} 26 | 27 | # Build the project 28 | build: build-lib build-ext 29 | 30 | # Build extension binary 31 | build-ext *args: 32 | cargo build --example {{bin_name}} --no-default-features --features default_loadable_extension {{args}} 33 | 34 | # Build the lib 35 | build-lib: 36 | cargo build --workspace --all-targets 37 | 38 | # Quick compile without building a binary 39 | check: 40 | cargo check --workspace --all-targets 41 | 42 | # Verify that the current version of the crate is not the same as the one published on crates.io 43 | check-if-published: (assert 'jq') 44 | #!/usr/bin/env bash 45 | set -euo pipefail 46 | LOCAL_VERSION="$({{just_executable()}} get-crate-field version)" 47 | echo "Detected crate version: '$LOCAL_VERSION'" 48 | CRATE_NAME="$({{just_executable()}} get-crate-field name)" 49 | echo "Detected crate name: '$CRATE_NAME'" 50 | PUBLISHED_VERSION="$(cargo search ${CRATE_NAME} | grep "^${CRATE_NAME} =" | sed -E 's/.* = "(.*)".*/\1/')" 51 | echo "Published crate version: '$PUBLISHED_VERSION'" 52 | if [ "$LOCAL_VERSION" = "$PUBLISHED_VERSION" ]; then 53 | echo "ERROR: The current crate version has already been published." 54 | exit 1 55 | else 56 | echo "The current crate version has not yet been published." 57 | fi 58 | 59 | # Quick compile - lib-only 60 | check-lib: 61 | cargo check --workspace 62 | 63 | # Generate code coverage report to upload to codecov.io 64 | ci-coverage: && \ 65 | (coverage '--codecov --output-path target/llvm-cov/codecov.info') 66 | # ATTENTION: the full file path above is used in the CI workflow 67 | mkdir -p target/llvm-cov 68 | 69 | # Run all tests as expected by CI 70 | ci-test: env-info test-fmt check clippy test test-ext test-doc 71 | #!/usr/bin/env bash 72 | set -euo pipefail 73 | if [ -n "$(git status --untracked-files --porcelain)" ]; then 74 | >&2 echo 'ERROR: git repo is no longer clean. Make sure compilation and tests artifacts are in the .gitignore, and no repo files are modified.' 75 | >&2 echo '######### git status ##########' 76 | git status 77 | exit 1 78 | fi 79 | 80 | # Run minimal subset of tests to ensure compatibility with MSRV 81 | ci-test-msrv: env-info check-lib test 82 | 83 | # Clean all build artifacts 84 | clean: 85 | cargo clean 86 | 87 | # Run cargo clippy to lint the code 88 | clippy *args: 89 | cargo clippy --workspace --all-targets {{args}} 90 | cargo clippy --no-default-features --features default_loadable_extension {{args}} 91 | 92 | # Generate code coverage report. Will install `cargo llvm-cov` if missing. 93 | coverage *args='--no-clean --open': (cargo-install 'cargo-llvm-cov') 94 | cargo llvm-cov --workspace --all-targets --include-build-script {{args}} 95 | # TODO: add test coverage for the loadable extension too, and combine them 96 | # cargo llvm-cov --example {{bin_name}} --no-default-features --features default_loadable_extension --codecov --output-path codecov.info 97 | 98 | cross-build-ext *args: 99 | cross build --example {{bin_name}} --no-default-features --features default_loadable_extension {{args}} 100 | 101 | cross-build-ext-aarch64: (cross-build-ext '--target=aarch64-unknown-linux-gnu' '--release') 102 | 103 | cross-test-ext-aarch64: 104 | docker run \ 105 | --rm \ 106 | -v "$(pwd):/workspace" \ 107 | -w /workspace \ 108 | --entrypoint sh \ 109 | -e EXTENSION_FILE=target/aarch64-unknown-linux-gnu/release/examples/lib{{bin_name}} \ 110 | --platform linux/arm64 \ 111 | arm64v8/ubuntu \ 112 | -c 'apt-get update && apt-get install -y sqlite3 && tests/test-ext.sh' 113 | 114 | # Build and open code documentation 115 | docs: 116 | cargo doc --no-deps --open 117 | 118 | # Print environment info 119 | env-info: 120 | @echo "Running {{if ci_mode == '1' {'in CI mode'} else {'in dev mode'} }} on {{os()}} / {{arch()}}" 121 | {{just_executable()}} --version 122 | rustc --version 123 | cargo --version 124 | rustup --version 125 | @echo "RUSTFLAGS='$RUSTFLAGS'" 126 | @echo "RUSTDOCFLAGS='$RUSTDOCFLAGS'" 127 | 128 | # Reformat all code `cargo fmt`. If nightly is available, use it for better results 129 | fmt: 130 | #!/usr/bin/env bash 131 | set -euo pipefail 132 | if rustup component list --toolchain nightly | grep rustfmt &> /dev/null; then 133 | echo 'Reformatting Rust code using nightly Rust fmt to sort imports' 134 | cargo +nightly fmt --all -- --config imports_granularity=Module,group_imports=StdExternalCrate 135 | else 136 | echo 'Reformatting Rust with the stable cargo fmt. Install nightly with `rustup install nightly` for better results' 137 | cargo fmt --all 138 | fi 139 | 140 | # Get any package's field from the metadata 141 | get-crate-field field package=main_crate: 142 | cargo metadata --format-version 1 | jq -r '.packages | map(select(.name == "{{package}}")) | first | .{{field}}' 143 | 144 | # Get the minimum supported Rust version (MSRV) for the crate 145 | get-msrv: (get-crate-field 'rust_version') 146 | 147 | # Find the minimum supported Rust version (MSRV) using cargo-msrv extension, and update Cargo.toml 148 | msrv: (cargo-install 'cargo-msrv') 149 | cargo msrv find --write-msrv --ignore-lockfile 150 | 151 | # Check semver compatibility with prior published version. Install it with `cargo install cargo-semver-checks` 152 | semver *args: (cargo-install 'cargo-semver-checks') 153 | cargo semver-checks {{args}} 154 | 155 | # Switch to the minimum rusqlite version 156 | set-min-rusqlite-version: (assert "jq") 157 | #!/usr/bin/env bash 158 | set -euo pipefail 159 | MIN_RUSQL_VER="$(grep '^rusqlite =.*version = ">=' Cargo.toml | sed -E 's/.*version = "[^"0-9]*([0-9.-]+).*/\1/')" 160 | echo "Switching to minimum rusqlite version: $MIN_RUSQL_VER" 161 | cargo update -p rusqlite --precise "$MIN_RUSQL_VER" 162 | 163 | # Run all unit and integration tests 164 | test: \ 165 | ( test-one-lib ) \ 166 | ( test-one-lib "--no-default-features" "--features" "gzip,brotli,bzip2,bsdiff4,bsdiffraw" ) \ 167 | ( test-one-lib "--no-default-features" "--features" "trace,brotli" ) \ 168 | ( test-one-lib "--no-default-features" "--features" "trace,bsdiff4" ) \ 169 | ( test-one-lib "--no-default-features" "--features" "trace,bsdiffraw" ) \ 170 | ( test-one-lib "--no-default-features" "--features" "trace,bzip2" ) \ 171 | ( test-one-lib "--no-default-features" "--features" "trace,gzip" ) 172 | 173 | # Test documentation 174 | test-doc: 175 | cargo test --doc 176 | cargo doc --no-deps 177 | 178 | # Test extension by loading it into sqlite and running SQL tests 179 | test-ext: build-ext 180 | ./tests/test-ext.sh 181 | 182 | # Test code formatting 183 | test-fmt: 184 | cargo fmt --all -- --check 185 | 186 | # Find unused dependencies. Install it with `cargo install cargo-udeps` 187 | udeps: (cargo-install 'cargo-udeps') 188 | cargo +nightly udeps --all-targets --workspace --all-features 189 | 190 | # Update all dependencies, including breaking changes. Requires nightly toolchain (install with `rustup install nightly`) 191 | update: 192 | cargo +nightly -Z unstable-options update --breaking 193 | cargo update 194 | 195 | # Ensure that a certain command is available 196 | [private] 197 | assert command: 198 | @if ! type {{command}} > /dev/null; then \ 199 | echo "Command '{{command}}' could not be found. Please make sure it has been installed on your computer." ;\ 200 | exit 1 ;\ 201 | fi 202 | 203 | # Check if a certain Cargo command is installed, and install it if needed 204 | [private] 205 | cargo-install $COMMAND $INSTALL_CMD='' *args='': 206 | #!/usr/bin/env bash 207 | set -euo pipefail 208 | if ! command -v $COMMAND > /dev/null; then 209 | if ! command -v cargo-binstall > /dev/null; then 210 | echo "$COMMAND could not be found. Installing it with cargo install ${INSTALL_CMD:-$COMMAND} --locked {{args}}" 211 | cargo install ${INSTALL_CMD:-$COMMAND} --locked {{args}} 212 | else 213 | echo "$COMMAND could not be found. Installing it with cargo binstall ${INSTALL_CMD:-$COMMAND} --locked {{args}}" 214 | cargo binstall ${INSTALL_CMD:-$COMMAND} --locked {{args}} 215 | fi 216 | fi 217 | 218 | [private] 219 | is-sqlite3-available: 220 | #!/usr/bin/env bash 221 | set -euo pipefail 222 | if ! command -v {{sqlite3}} > /dev/null; then 223 | echo "{{sqlite3}} executable could not be found" 224 | exit 1 225 | fi 226 | echo "Found {{sqlite3}} executable:" 227 | {{sqlite3}} --version 228 | 229 | [private] 230 | test-one-lib *args: 231 | @echo "### TEST {{args}} #######################################################################################################################" 232 | cargo test {{args}} 233 | -------------------------------------------------------------------------------- /src/brotli.rs: -------------------------------------------------------------------------------- 1 | use std::io::{Read, Write}; 2 | 3 | use brotli::{CompressorWriter, Decompressor}; 4 | use rusqlite::Error::UserFunctionError; 5 | 6 | use crate::common::{register_compression, Encoder}; 7 | use crate::rusqlite::{Connection, Result}; 8 | 9 | /// Register the `brotli` SQL function with the given `SQLite` connection. 10 | /// The function takes a single argument and returns the [Brotli compression](https://en.wikipedia.org/wiki/Brotli) (blob) of that argument. 11 | /// The argument can be either a string or a blob. 12 | /// If the argument is `NULL`, the result is `NULL`. 13 | /// 14 | /// # Example 15 | /// 16 | /// ``` 17 | /// # use sqlite_compressions::rusqlite::{Connection, Result}; 18 | /// # use sqlite_compressions::register_brotli_functions; 19 | /// # fn main() -> Result<()> { 20 | /// let db = Connection::open_in_memory()?; 21 | /// register_brotli_functions(&db)?; 22 | /// let result: Vec = db.query_row("SELECT brotli('hello')", [], |r| r.get(0))?; 23 | /// let expected = b"\x0b\x02\x80\x68\x65\x6c\x6c\x6f\x03"; 24 | /// assert_eq!(result, expected); 25 | /// let result: String = db.query_row("SELECT CAST(brotli_decode(brotli('world')) AS TEXT)", [], |r| r.get(0))?; 26 | /// let expected = "world"; 27 | /// assert_eq!(result, expected); 28 | /// let result: bool = db.query_row("SELECT brotli_test(brotli('world'))", [], |r| r.get(0))?; 29 | /// let expected = true; 30 | /// assert_eq!(result, expected); 31 | /// # Ok(()) 32 | /// # } 33 | /// ``` 34 | pub fn register_brotli_functions(conn: &Connection) -> Result<()> { 35 | register_compression::(conn) 36 | } 37 | 38 | pub struct BrotliEncoder; 39 | 40 | impl Encoder for BrotliEncoder { 41 | fn enc_name() -> &'static str { 42 | "brotli" 43 | } 44 | fn dec_name() -> &'static str { 45 | "brotli_decode" 46 | } 47 | fn test_name() -> &'static str { 48 | "brotli_test" 49 | } 50 | 51 | fn encode(data: &[u8], quality: Option) -> Result> { 52 | let mut encoder = CompressorWriter::new(Vec::new(), 4 * 1024, quality.unwrap_or(11), 22); 53 | encoder 54 | .write_all(data) 55 | .map_err(|e| UserFunctionError(e.into()))?; 56 | Ok(encoder.into_inner()) 57 | } 58 | 59 | fn decode(data: &[u8]) -> Result> { 60 | let mut decompressed = Vec::new(); 61 | Decompressor::new(data, 4 * 1024) 62 | .read_to_end(&mut decompressed) 63 | .map_err(|e| UserFunctionError(e.into()))?; 64 | Ok(decompressed) 65 | } 66 | 67 | fn test(data: &[u8]) -> bool { 68 | // reuse the same buffer when decompressing 69 | // ideally we should use some null buffer, but flate2 doesn't seem to support that 70 | // note that buffer size does affect performance and depend on the input data size 71 | let mut buffer = [0u8; 4 * 1024]; 72 | let mut decoder = Decompressor::new(data, 4 * 1024); 73 | while let Ok(len) = decoder.read(&mut buffer) { 74 | if len == 0 { 75 | return true; 76 | } 77 | } 78 | false 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/bsdiff4.rs: -------------------------------------------------------------------------------- 1 | use std::io::Cursor; 2 | 3 | use qbsdiff::bsdiff::Bsdiff; 4 | use qbsdiff::bspatch::Bspatch; 5 | use rusqlite::Error::UserFunctionError; 6 | 7 | use crate::common_diff::{register_differ, Differ}; 8 | use crate::rusqlite::{Connection, Result}; 9 | 10 | /// Register the `bsdiff4` and `bspatch4` SQL functions with the given `SQLite` connection. 11 | /// The `bsdiff4` function takes two arguments, and returns the [BSDiff delta](https://github.com/mendsley/bsdiff#readme) (blob) of the binary difference. 12 | /// The arguments can be either a string or a blob. 13 | /// If any of the arguments are `NULL`, the result is `NULL`. 14 | /// 15 | /// # Example 16 | /// 17 | /// ``` 18 | /// # use sqlite_compressions::rusqlite::{Connection, Result}; 19 | /// # use sqlite_compressions::register_bsdiff4_functions; 20 | /// # fn main() -> Result<()> { 21 | /// let db = Connection::open_in_memory()?; 22 | /// register_bsdiff4_functions(&db)?; 23 | /// let result: Vec = db.query_row("SELECT bspatch4('013479', bsdiff4('013479', '23456789'))", [], |r| r.get(0))?; 24 | /// let expected = b"23456789"; 25 | /// assert_eq!(result, expected); 26 | /// # Ok(()) 27 | /// # } 28 | /// ``` 29 | pub fn register_bsdiff4_functions(conn: &Connection) -> Result<()> { 30 | register_differ::(conn) 31 | } 32 | 33 | pub struct Bsdiff4Differ; 34 | 35 | impl Differ for Bsdiff4Differ { 36 | fn diff_name() -> &'static str { 37 | "bsdiff4" 38 | } 39 | 40 | fn patch_name() -> &'static str { 41 | "bspatch4" 42 | } 43 | 44 | fn diff(source: &[u8], target: &[u8]) -> Result> { 45 | let mut patch = Vec::new(); 46 | Bsdiff::new(source, target) 47 | .compare(Cursor::new(&mut patch)) 48 | .map_err(|e| UserFunctionError(e.into()))?; 49 | Ok(patch) 50 | } 51 | 52 | fn patch(source: &[u8], patch: &[u8]) -> Result> { 53 | let mut target = Vec::new(); 54 | Bspatch::new(patch) 55 | .and_then(|patch| patch.apply(source, Cursor::new(&mut target))) 56 | .map_err(|e| UserFunctionError(e.into()))?; 57 | Ok(target) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/bsdiffraw.rs: -------------------------------------------------------------------------------- 1 | use std::io::Cursor; 2 | 3 | use rusqlite::Error::UserFunctionError; 4 | 5 | use crate::common_diff::{register_differ, Differ}; 6 | use crate::rusqlite::{Connection, Result}; 7 | 8 | /// Register the `bsdiffraw` and `bspatchraw` SQL functions with the given `SQLite` connection. 9 | /// The `bsdiffraw` function takes two arguments, and returns the [BSDiff delta](https://github.com/mendsley/bsdiff#readme) (blob) of the binary difference. 10 | /// The arguments can be either a string or a blob. 11 | /// If any of the arguments are `NULL`, the result is `NULL`. 12 | /// 13 | /// # Example 14 | /// 15 | /// ``` 16 | /// # use sqlite_compressions::rusqlite::{Connection, Result}; 17 | /// # use sqlite_compressions::register_bsdiffraw_functions; 18 | /// # fn main() -> Result<()> { 19 | /// let db = Connection::open_in_memory()?; 20 | /// register_bsdiffraw_functions(&db)?; 21 | /// let result: String = db.query_row("SELECT hex(bsdiffraw('abc013479zz', 'abc23456789zzf'))", [], |r| r.get(0))?; 22 | /// assert_eq!(result.as_str(), "03000000000000000B00000000000000070000000000000000000032333435363738397A7A66"); 23 | /// let result: String = db.query_row("SELECT hex(bspatchraw('abc013479zz', bsdiffraw('abc013479zz', 'abc23456789zzf')))", [], |r| r.get(0))?; 24 | /// assert_eq!(result.as_str(), "61626332333435363738397A7A66"); 25 | /// let result: Vec = db.query_row("SELECT bspatchraw('013479', bsdiffraw('013479', '23456789'))", [], |r| r.get(0))?; 26 | /// let expected = b"23456789"; 27 | /// assert_eq!(result, expected); 28 | /// # Ok(()) 29 | /// # } 30 | /// ``` 31 | pub fn register_bsdiffraw_functions(conn: &Connection) -> Result<()> { 32 | register_differ::(conn) 33 | } 34 | 35 | pub struct BsdiffRawDiffer; 36 | 37 | impl Differ for BsdiffRawDiffer { 38 | fn diff_name() -> &'static str { 39 | "bsdiffraw" 40 | } 41 | 42 | fn patch_name() -> &'static str { 43 | "bspatchraw" 44 | } 45 | 46 | fn diff(source: &[u8], target: &[u8]) -> Result> { 47 | let mut patch = Vec::new(); 48 | bsdiff::diff(source, target, &mut patch).map_err(|e| UserFunctionError(e.into()))?; 49 | Ok(patch) 50 | } 51 | 52 | fn patch(source: &[u8], patch: &[u8]) -> Result> { 53 | let mut target = Vec::new(); 54 | bsdiff::patch(source, &mut Cursor::new(patch), &mut target) 55 | .map_err(|e| UserFunctionError(e.into()))?; 56 | Ok(target) 57 | } 58 | } 59 | 60 | #[cfg(test)] 61 | mod tests { 62 | use super::*; 63 | 64 | #[test] 65 | fn test_diff() { 66 | let source = b"abc013479zz"; 67 | let target = b"abc23456789zzf"; 68 | let patch = BsdiffRawDiffer::diff(source, target).unwrap(); 69 | let expected = b"\x03\x00\x00\x00\x00\x00\x00\x00\x0B\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x32\x33\x34\x35\x36\x37\x38\x39\x7A\x7A\x66"; 70 | assert_eq!(patch, expected); 71 | } 72 | 73 | #[test] 74 | fn test_patch() { 75 | let source = b"abc013479zz"; 76 | let patch = b"\x03\x00\x00\x00\x00\x00\x00\x00\x0B\x00\x00\x00\x00\x00\x00\x00\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x32\x33\x34\x35\x36\x37\x38\x39\x7A\x7A\x66"; 77 | let target = BsdiffRawDiffer::patch(source, patch).unwrap(); 78 | let expected = b"abc23456789zzf"; 79 | assert_eq!(target, expected); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/bzip2.rs: -------------------------------------------------------------------------------- 1 | use std::io::{Read, Write}; 2 | 3 | use bzip2::read::BzDecoder; 4 | use bzip2::write::BzEncoder; 5 | use bzip2::Compression; 6 | use rusqlite::Error::UserFunctionError; 7 | 8 | use crate::common::{register_compression, Encoder}; 9 | use crate::rusqlite::{Connection, Result}; 10 | 11 | /// Register the `bzip2` SQL functions with the given `SQLite` connection. 12 | /// The function takes a single argument and returns the [bzip2 compression](https://en.wikipedia.org/wiki/Bzip2) (blob) of that argument. 13 | /// The argument can be either a string or a blob. 14 | /// If the argument is `NULL`, the result is `NULL`. 15 | /// 16 | /// # Example 17 | /// 18 | /// ``` 19 | /// # use sqlite_compressions::rusqlite::{Connection, Result}; 20 | /// # use sqlite_compressions::register_bzip2_functions; 21 | /// # fn main() -> Result<()> { 22 | /// let db = Connection::open_in_memory()?; 23 | /// register_bzip2_functions(&db)?; 24 | /// let result: Vec = db.query_row("SELECT bzip2('hello')", [], |r| r.get(0))?; 25 | /// let expected = b"\x42\x5a\x68\x36\x31\x41\x59\x26\x53\x59\x19\x31\x65\x3d\x00\x00\x00\x81\x00\x02\x44\xa0\x00\x21\x9a\x68\x33\x4d\x07\x33\x8b\xb9\x22\x9c\x28\x48\x0c\x98\xb2\x9e\x80"; 26 | /// assert_eq!(result, expected); 27 | /// let result: String = db.query_row("SELECT CAST(bzip2_decode(bzip2('world')) AS TEXT)", [], |r| r.get(0))?; 28 | /// let expected = "world"; 29 | /// assert_eq!(result, expected); 30 | /// let result: bool = db.query_row("SELECT bzip2_test(bzip2('world'))", [], |r| r.get(0))?; 31 | /// let expected = true; 32 | /// assert_eq!(result, expected); 33 | /// # Ok(()) 34 | /// # } 35 | /// ``` 36 | pub fn register_bzip2_functions(conn: &Connection) -> Result<()> { 37 | register_compression::(conn) 38 | } 39 | 40 | pub struct Bzip2Encoder; 41 | 42 | impl Encoder for Bzip2Encoder { 43 | fn enc_name() -> &'static str { 44 | "bzip2" 45 | } 46 | fn dec_name() -> &'static str { 47 | "bzip2_decode" 48 | } 49 | fn test_name() -> &'static str { 50 | "bzip2_test" 51 | } 52 | 53 | fn encode(data: &[u8], quality: Option) -> Result> { 54 | let quality = if let Some(param) = quality { 55 | if param < Compression::fast().level() || param > Compression::best().level() { 56 | return Err(UserFunctionError( 57 | format!( 58 | "The optional second argument to bzip2() must be between {} and {}", 59 | Compression::fast().level(), 60 | Compression::best().level() 61 | ) 62 | .into(), 63 | )); 64 | } 65 | Compression::new(param) 66 | } else { 67 | Compression::default() 68 | }; 69 | 70 | let mut encoder = BzEncoder::new(Vec::new(), quality); 71 | encoder 72 | .write_all(data) 73 | .map_err(|e| UserFunctionError(e.into()))?; 74 | encoder.finish().map_err(|e| UserFunctionError(e.into())) 75 | } 76 | 77 | fn decode(data: &[u8]) -> Result> { 78 | let mut decompressed = Vec::new(); 79 | BzDecoder::new(data) 80 | .read_to_end(&mut decompressed) 81 | .map_err(|e| UserFunctionError(e.into()))?; 82 | Ok(decompressed) 83 | } 84 | 85 | fn test(data: &[u8]) -> bool { 86 | // reuse the same buffer when decompressing 87 | // ideally we should use some null buffer, but bzip2 doesn't seem to support that 88 | // note that buffer size does affect performance and depend on the input data size 89 | let mut buffer = [0u8; 1024]; 90 | let mut decoder = BzDecoder::new(data); 91 | while let Ok(len) = decoder.read(&mut buffer) { 92 | if len == 0 { 93 | return true; 94 | } 95 | } 96 | false 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/cdylib/cdylib.rs: -------------------------------------------------------------------------------- 1 | use std::os::raw::{c_char, c_int}; 2 | 3 | use rusqlite::ffi::SQLITE_NOTICE; 4 | use rusqlite::trace::log; 5 | use rusqlite::{ffi, Connection, Result}; 6 | 7 | #[allow(clippy::not_unsafe_ptr_arg_deref)] 8 | #[no_mangle] 9 | pub unsafe extern "C" fn sqlite3_extension_init( 10 | db: *mut ffi::sqlite3, 11 | pz_err_msg: *mut *mut c_char, 12 | p_api: *mut ffi::sqlite3_api_routines, 13 | ) -> c_int { 14 | Connection::extension_init2(db, pz_err_msg, p_api, extension_init) 15 | } 16 | 17 | fn extension_init(db: Connection) -> Result { 18 | sqlite_compressions::register_compression_functions(&db)?; 19 | log(SQLITE_NOTICE, "Loaded sqlite_compressions extension"); 20 | Ok(false) 21 | } 22 | -------------------------------------------------------------------------------- /src/common.rs: -------------------------------------------------------------------------------- 1 | use std::panic::{RefUnwindSafe, UnwindSafe}; 2 | 3 | #[cfg(feature = "trace")] 4 | use log::trace; 5 | use rusqlite::functions::Context; 6 | use rusqlite::types::{Type, ValueRef}; 7 | use rusqlite::Error::{InvalidFunctionParameterType, InvalidParameterCount}; 8 | 9 | use crate::rusqlite::functions::FunctionFlags; 10 | use crate::rusqlite::{Connection, Result}; 11 | 12 | #[cfg(not(feature = "trace"))] 13 | macro_rules! trace { 14 | ($($arg:tt)*) => {}; 15 | } 16 | 17 | pub trait Encoder { 18 | fn enc_name() -> &'static str; 19 | fn dec_name() -> &'static str; 20 | fn test_name() -> &'static str; 21 | fn encode(data: &[u8], quality: Option) -> Result>; 22 | fn decode(data: &[u8]) -> Result>; 23 | fn test(data: &[u8]) -> bool; 24 | } 25 | 26 | pub(crate) fn register_compression( 27 | conn: &Connection, 28 | ) -> Result<()> { 29 | // FunctionFlags derive Copy trait only in v0.31+, but we support v0.30+ 30 | macro_rules! flags { 31 | () => { 32 | FunctionFlags::SQLITE_UTF8 33 | | FunctionFlags::SQLITE_DETERMINISTIC 34 | | FunctionFlags::SQLITE_DIRECTONLY 35 | }; 36 | } 37 | 38 | trace!("Registering function {}", T::enc_name()); 39 | conn.create_scalar_function(T::enc_name(), -1, flags!(), encoder_fn::)?; 40 | 41 | trace!("Registering function {}", T::dec_name()); 42 | conn.create_scalar_function(T::dec_name(), -1, flags!(), decoder_fn::)?; 43 | 44 | trace!("Registering function {}", T::test_name()); 45 | conn.create_scalar_function(T::test_name(), -1, flags!(), testing_fn::) 46 | } 47 | 48 | fn encoder_fn( 49 | ctx: &Context, 50 | ) -> Result>> { 51 | let param_count = ctx.len(); 52 | if param_count == 0 || param_count > 2 { 53 | return Err(InvalidParameterCount(param_count, 1)); 54 | } 55 | let quality = if param_count == 2 { 56 | Some(ctx.get::(1)?) 57 | } else { 58 | None 59 | }; 60 | 61 | let value = ctx.get_raw(0); 62 | match value { 63 | ValueRef::Blob(val) => { 64 | trace!("{}: encoding blob {val:?}", T::enc_name()); 65 | Ok(Some(T::encode(val, quality)?)) 66 | } 67 | ValueRef::Text(val) => { 68 | trace!("{}: encoding text {val:?}", T::enc_name()); 69 | Ok(Some(T::encode(val, quality)?)) 70 | } 71 | ValueRef::Null => { 72 | trace!("{}: ignoring NULL", T::enc_name()); 73 | Ok(None) 74 | } 75 | #[allow(unused_variables)] 76 | ValueRef::Integer(val) => { 77 | trace!("{}: unsupported Integer {val:?}", T::enc_name()); 78 | Err(InvalidFunctionParameterType(0, Type::Integer)) 79 | } 80 | #[allow(unused_variables)] 81 | ValueRef::Real(val) => { 82 | trace!("{}: unsupported Real {val:?}", T::enc_name()); 83 | Err(InvalidFunctionParameterType(0, Type::Real)) 84 | } 85 | } 86 | } 87 | 88 | fn decoder_fn( 89 | ctx: &Context, 90 | ) -> Result>> { 91 | let param_count = ctx.len(); 92 | if param_count != 1 { 93 | return Err(InvalidParameterCount(param_count, 1)); 94 | } 95 | 96 | let value = ctx.get_raw(0); 97 | match value { 98 | ValueRef::Blob(val) => { 99 | trace!("{}: decoding blob {val:?}", T::dec_name()); 100 | Ok(Some(T::decode(val)?)) 101 | } 102 | ValueRef::Null => { 103 | trace!("{}: ignoring NULL", T::dec_name()); 104 | Ok(None) 105 | } 106 | #[allow(unused_variables)] 107 | ValueRef::Text(val) => { 108 | trace!("{}: unsupported Text {val:?}", T::dec_name()); 109 | Err(InvalidFunctionParameterType(0, Type::Text)) 110 | } 111 | #[allow(unused_variables)] 112 | ValueRef::Integer(val) => { 113 | trace!("{}: unsupported Integer {val:?}", T::dec_name()); 114 | Err(InvalidFunctionParameterType(0, Type::Integer)) 115 | } 116 | #[allow(unused_variables)] 117 | ValueRef::Real(val) => { 118 | trace!("{}: unsupported Real {val:?}", T::dec_name()); 119 | Err(InvalidFunctionParameterType(0, Type::Real)) 120 | } 121 | } 122 | } 123 | 124 | fn testing_fn( 125 | ctx: &Context, 126 | ) -> Result> { 127 | let param_count = ctx.len(); 128 | if param_count != 1 { 129 | return Err(InvalidParameterCount(param_count, 1)); 130 | } 131 | 132 | let value = ctx.get_raw(0); 133 | match value { 134 | ValueRef::Blob(val) => { 135 | trace!("{}: testing encoded blob {val:?}", T::test_name()); 136 | Ok(Some(T::test(val))) 137 | } 138 | ValueRef::Null => { 139 | trace!("{}: ignoring NULL", T::test_name()); 140 | Ok(None) 141 | } 142 | #[allow(unused_variables)] 143 | ValueRef::Text(val) => { 144 | trace!("{}: unsupported Text {val:?}", T::test_name()); 145 | Err(InvalidFunctionParameterType(0, Type::Text)) 146 | } 147 | #[allow(unused_variables)] 148 | ValueRef::Integer(val) => { 149 | trace!("{}: unsupported Integer {val:?}", T::test_name()); 150 | Err(InvalidFunctionParameterType(0, Type::Integer)) 151 | } 152 | #[allow(unused_variables)] 153 | ValueRef::Real(val) => { 154 | trace!("{}: unsupported Real {val:?}", T::test_name()); 155 | Err(InvalidFunctionParameterType(0, Type::Real)) 156 | } 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/common_diff.rs: -------------------------------------------------------------------------------- 1 | use std::panic::{RefUnwindSafe, UnwindSafe}; 2 | 3 | #[cfg(feature = "trace")] 4 | use log::trace; 5 | use rusqlite::functions::{Context, FunctionFlags}; 6 | use rusqlite::types::{Type, ValueRef}; 7 | use rusqlite::Connection; 8 | use rusqlite::Error::InvalidFunctionParameterType; 9 | 10 | #[cfg(not(feature = "trace"))] 11 | macro_rules! trace { 12 | ($($arg:tt)*) => {}; 13 | } 14 | 15 | use crate::rusqlite::Result; 16 | 17 | pub trait Differ { 18 | fn diff_name() -> &'static str; 19 | fn patch_name() -> &'static str; 20 | fn diff(source: &[u8], target: &[u8]) -> Result>; 21 | fn patch(source: &[u8], patch: &[u8]) -> Result>; 22 | } 23 | 24 | pub(crate) fn register_differ( 25 | conn: &Connection, 26 | ) -> Result<()> { 27 | // FunctionFlags derive Copy trait only in v0.31+, but we support v0.30+ 28 | macro_rules! flags { 29 | () => { 30 | FunctionFlags::SQLITE_UTF8 31 | | FunctionFlags::SQLITE_DETERMINISTIC 32 | | FunctionFlags::SQLITE_DIRECTONLY 33 | }; 34 | } 35 | 36 | trace!("Registering function {}", T::diff_name()); 37 | conn.create_scalar_function(T::diff_name(), 2, flags!(), diff_fn::)?; 38 | 39 | trace!("Registering function {}", T::patch_name()); 40 | conn.create_scalar_function(T::patch_name(), 2, flags!(), patch_fn::) 41 | } 42 | 43 | fn diff_fn( 44 | ctx: &Context, 45 | ) -> Result>> { 46 | let Some(source) = get_bytes(ctx, 0)? else { 47 | return Ok(None); 48 | }; 49 | let Some(target) = get_bytes(ctx, 1)? else { 50 | return Ok(None); 51 | }; 52 | Ok(Some(T::diff(source, target)?)) 53 | } 54 | 55 | fn patch_fn( 56 | ctx: &Context, 57 | ) -> Result>> { 58 | let Some(source) = get_bytes(ctx, 0)? else { 59 | return Ok(None); 60 | }; 61 | let Some(patch) = get_bytes(ctx, 1)? else { 62 | return Ok(None); 63 | }; 64 | Ok(Some(T::patch(source, patch)?)) 65 | } 66 | 67 | pub(crate) fn get_bytes<'a>(ctx: &'a Context, index: usize) -> Result> { 68 | match ctx.get_raw(index) { 69 | ValueRef::Blob(val) | ValueRef::Text(val) => Ok(Some(val)), 70 | ValueRef::Null => Ok(None), 71 | ValueRef::Integer(_) => Err(InvalidFunctionParameterType(index, Type::Integer)), 72 | ValueRef::Real(_) => Err(InvalidFunctionParameterType(index, Type::Real)), 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/gzip.rs: -------------------------------------------------------------------------------- 1 | use std::io::{Read, Write}; 2 | 3 | use flate2::read::GzDecoder; 4 | use flate2::write::GzEncoder; 5 | use flate2::Compression; 6 | use rusqlite::Error::UserFunctionError; 7 | 8 | use crate::common::{register_compression, Encoder}; 9 | use crate::rusqlite::{Connection, Result}; 10 | 11 | /// Register the `gzip` SQL functions with the given `SQLite` connection. 12 | /// The function takes a single argument and returns the [GZIP compression](https://en.wikipedia.org/wiki/Gzip) (blob) of that argument. 13 | /// The argument can be either a string or a blob. 14 | /// If the argument is `NULL`, the result is `NULL`. 15 | /// 16 | /// # Example 17 | /// 18 | /// ``` 19 | /// # use sqlite_compressions::rusqlite::{Connection, Result}; 20 | /// # use sqlite_compressions::register_gzip_functions; 21 | /// # fn main() -> Result<()> { 22 | /// let db = Connection::open_in_memory()?; 23 | /// register_gzip_functions(&db)?; 24 | /// let result: Vec = db.query_row("SELECT gzip('hello')", [], |r| r.get(0))?; 25 | /// let expected = b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcb\x48\xcd\xc9\xc9\x07\x00\x86\xa6\x10\x36\x05\x00\x00\x00"; 26 | /// assert_eq!(result, expected); 27 | /// let result: String = db.query_row("SELECT CAST(gzip_decode(gzip('world')) AS TEXT)", [], |r| r.get(0))?; 28 | /// let expected = "world"; 29 | /// assert_eq!(result, expected); 30 | /// let result: bool = db.query_row("SELECT gzip_test(gzip('world'))", [], |r| r.get(0))?; 31 | /// let expected = true; 32 | /// assert_eq!(result, expected); 33 | /// # Ok(()) 34 | /// # } 35 | /// ``` 36 | pub fn register_gzip_functions(conn: &Connection) -> Result<()> { 37 | register_compression::(conn) 38 | } 39 | 40 | pub struct GzipEncoder; 41 | 42 | impl Encoder for GzipEncoder { 43 | fn enc_name() -> &'static str { 44 | "gzip" 45 | } 46 | fn dec_name() -> &'static str { 47 | "gzip_decode" 48 | } 49 | fn test_name() -> &'static str { 50 | "gzip_test" 51 | } 52 | 53 | fn encode(data: &[u8], quality: Option) -> Result> { 54 | let quality = if let Some(param) = quality { 55 | if param > 9 { 56 | return Err(UserFunctionError( 57 | "The optional second argument to gzip() must be between 0 and 9".into(), 58 | )); 59 | } 60 | Compression::new(param) 61 | } else { 62 | Compression::default() 63 | }; 64 | 65 | let mut encoder = GzEncoder::new(Vec::new(), quality); 66 | encoder 67 | .write_all(data) 68 | .map_err(|e| UserFunctionError(e.into()))?; 69 | encoder.finish().map_err(|e| UserFunctionError(e.into())) 70 | } 71 | 72 | fn decode(data: &[u8]) -> Result> { 73 | let mut decompressed = Vec::new(); 74 | GzDecoder::new(data) 75 | .read_to_end(&mut decompressed) 76 | .map_err(|e| UserFunctionError(e.into()))?; 77 | Ok(decompressed) 78 | } 79 | 80 | fn test(data: &[u8]) -> bool { 81 | // reuse the same buffer when decompressing 82 | // ideally we should use some null buffer, but flate2 doesn't seem to support that 83 | // note that buffer size does affect performance and depend on the input data size 84 | let mut buffer = [0u8; 1024]; 85 | let mut decoder = GzDecoder::new(data); 86 | while let Ok(len) = decoder.read(&mut buffer) { 87 | if len == 0 { 88 | return true; 89 | } 90 | } 91 | false 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(feature = "default", doc = include_str!("../README.md"))] 2 | // 3 | // Unsafe code is required for cdylib, so only use it for this crate 4 | #![forbid(unsafe_code)] 5 | 6 | #[cfg(not(any( 7 | feature = "brotli", 8 | feature = "bsdiff4", 9 | feature = "bsdiffraw", 10 | feature = "bzip2", 11 | feature = "gzip", 12 | )))] 13 | compile_error!( 14 | "At least one of these features must be enabled: gzip, brotli, bzip2, bsdiff4, bsdiffraw" 15 | ); 16 | 17 | /// Re-export of the [`rusqlite`](https://crates.io/crates/rusqlite) crate to avoid version conflicts. 18 | pub use rusqlite; 19 | 20 | use crate::rusqlite::{Connection, Result}; 21 | 22 | #[cfg(any(feature = "bsdiff4", feature = "bsdiffraw"))] 23 | mod common_diff; 24 | #[cfg(any(feature = "bsdiff4", feature = "bsdiffraw"))] 25 | pub use crate::common_diff::Differ; 26 | 27 | #[cfg(any(feature = "brotli", feature = "bzip2", feature = "gzip"))] 28 | mod common; 29 | #[cfg(any(feature = "brotli", feature = "bzip2", feature = "gzip"))] 30 | pub use crate::common::Encoder; 31 | 32 | #[cfg(feature = "bsdiff4")] 33 | mod bsdiff4; 34 | #[cfg(feature = "bsdiff4")] 35 | pub use crate::bsdiff4::{register_bsdiff4_functions, Bsdiff4Differ}; 36 | 37 | #[cfg(feature = "bsdiffraw")] 38 | mod bsdiffraw; 39 | #[cfg(feature = "bsdiffraw")] 40 | pub use crate::bsdiffraw::{register_bsdiffraw_functions, BsdiffRawDiffer}; 41 | 42 | #[cfg(feature = "brotli")] 43 | mod brotli; 44 | #[cfg(feature = "brotli")] 45 | pub use crate::brotli::{register_brotli_functions, BrotliEncoder}; 46 | 47 | #[cfg(feature = "bzip2")] 48 | mod bzip2; 49 | #[cfg(feature = "bzip2")] 50 | pub use crate::bzip2::{register_bzip2_functions, Bzip2Encoder}; 51 | 52 | #[cfg(feature = "gzip")] 53 | mod gzip; 54 | #[cfg(feature = "gzip")] 55 | pub use crate::gzip::{register_gzip_functions, GzipEncoder}; 56 | 57 | /// Register all compression functions for the given `SQLite` connection. 58 | /// This is a convenience function that calls all the `register_*_functions` functions. 59 | /// Features must be enabled for the corresponding functions to be registered. 60 | /// 61 | /// # Example 62 | /// 63 | /// ``` 64 | /// # use sqlite_compressions::rusqlite::{Connection, Result}; 65 | /// # use sqlite_compressions::register_compression_functions; 66 | /// # fn main() -> Result<()> { 67 | /// let db = Connection::open_in_memory()?; 68 | /// register_compression_functions(&db)?; 69 | /// # if cfg!(feature = "gzip") { 70 | /// let result: String = db.query_row("SELECT hex(gzip('hello'))", [], |r| r.get(0))?; 71 | /// assert_eq!(&result, "1F8B08000000000000FFCB48CDC9C9070086A6103605000000"); 72 | /// let result: String = db.query_row("SELECT hex(gzip_decode(gzip(x'0123')))", [], |r| r.get(0))?; 73 | /// assert_eq!(&result, "0123"); 74 | /// let result: bool = db.query_row("SELECT gzip_test(gzip(x'0123'))", [], |r| r.get(0))?; 75 | /// assert_eq!(result, true); 76 | /// # } 77 | /// # if cfg!(feature = "brotli") { 78 | /// let result: String = db.query_row("SELECT hex(brotli('hello'))", [], |r| r.get(0))?; 79 | /// assert_eq!(&result, "0B028068656C6C6F03"); 80 | /// # } 81 | /// # Ok(()) 82 | /// # } 83 | /// ``` 84 | pub fn register_compression_functions(conn: &Connection) -> Result<()> { 85 | #[cfg(feature = "gzip")] 86 | register_gzip_functions(conn)?; 87 | #[cfg(feature = "brotli")] 88 | register_brotli_functions(conn)?; 89 | #[cfg(feature = "bzip2")] 90 | register_bzip2_functions(conn)?; 91 | #[cfg(feature = "bsdiff4")] 92 | register_bsdiff4_functions(conn)?; 93 | #[cfg(feature = "bsdiffraw")] 94 | register_bsdiffraw_functions(conn)?; 95 | 96 | Ok(()) 97 | } 98 | -------------------------------------------------------------------------------- /tests/test-ext.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | SQLITE3_BIN=${SQLITE3_BIN:-sqlite3} 5 | EXTENSION_FILE=${EXTENSION_FILE:-target/debug/examples/libsqlite_compressions} 6 | 7 | if [ ! -f "$EXTENSION_FILE" ] && [ ! -f "$EXTENSION_FILE.so" ] && [ ! -f "$EXTENSION_FILE.dylib" ] && [ ! -f "$EXTENSION_FILE.dll" ]; then 8 | echo "Extension file $EXTENSION_FILE [.so|.dylib|.dll] do not exist. Run 'just build-ext' first. Available files:" 9 | ls -l "$EXTENSION_FILE"* 10 | exit 1 11 | fi 12 | echo "Using extension file '$EXTENSION_FILE [.so|.dylib|.dll]'" 13 | 14 | if ! command -v "$SQLITE3_BIN" > /dev/null; then 15 | echo "$SQLITE3_BIN executable could not be found" 16 | exit 1 17 | fi 18 | echo "Found $SQLITE3_BIN executable $($SQLITE3_BIN --version)" 19 | 20 | test_one() { 21 | local sql=$1 22 | local expected=$2 23 | 24 | echo "Trying to get '$expected' from $sql" 25 | result=$($SQLITE3_BIN < Self { 14 | let db = Connection::open_in_memory().unwrap(); 15 | sqlite_compressions::register_compression_functions(&db).unwrap(); 16 | Self(db) 17 | } 18 | } 19 | 20 | impl Conn { 21 | pub fn sql(&self, query: &str) -> Result { 22 | self.0.query_row_and_then(query, [], |r| r.get(0)) 23 | } 24 | 25 | pub fn s(&self, func: &str, param: &str) -> String { 26 | self.q(¶m.replace('%', func)) 27 | } 28 | 29 | pub fn q(&self, query: &str) -> String { 30 | let query = format!("SELECT {query}"); 31 | match self.sql::>>(&query) { 32 | Ok(v) => match v { 33 | Some(v) => hex::encode(v), 34 | None => "NULL".into(), 35 | }, 36 | Err(e) => e.to_string(), 37 | } 38 | } 39 | 40 | pub fn bool(&self, func: &str, param: &str) -> String { 41 | let query = format!("SELECT {}", param.replace('%', func)); 42 | match self.sql::>(&query) { 43 | Ok(v) => match v { 44 | Some(v) => v.to_string(), 45 | None => "NULL".into(), 46 | }, 47 | Err(e) => e.to_string(), 48 | } 49 | } 50 | } 51 | 52 | #[rstest::rstest] 53 | #[cfg_attr(feature = "gzip", case("gzip"))] 54 | #[cfg_attr(feature = "brotli", case("brotli"))] 55 | #[cfg_attr(feature = "bzip2", case("bzip2"))] 56 | #[trace] 57 | #[test] 58 | #[cfg(any(feature = "brotli", feature = "bzip2", feature = "gzip"))] 59 | fn common(#[case] func: &str) { 60 | let c = Conn::default(); 61 | insta::allow_duplicates!( 62 | assert_snapshot!(c.s(func, "%(NULL)"), @"NULL"); 63 | assert_snapshot!(c.s(func, "%_decode(NULL)"), @"NULL"); 64 | assert_snapshot!(c.s(func, "%_test(NULL)"), @"NULL"); 65 | 66 | assert_snapshot!(c.s(func, "%()"), @"Wrong number of parameters passed to query. Got 0, needed 1"); 67 | assert_snapshot!(c.s(func, "%(1)"), @"Invalid function parameter type Integer at index 0"); 68 | assert_snapshot!(c.s(func, "%(0.42)"), @"Invalid function parameter type Real at index 0"); 69 | 70 | assert_snapshot!(c.s(func, "%_decode()"), @"Wrong number of parameters passed to query. Got 0, needed 1"); 71 | assert_snapshot!(c.s(func, "%_decode(NULL)"), @"NULL"); 72 | assert_snapshot!(c.s(func, "%_decode(1)"), @"Invalid function parameter type Integer at index 0"); 73 | assert_snapshot!(c.s(func, "%_decode(0.42)"), @"Invalid function parameter type Real at index 0"); 74 | 75 | assert_snapshot!(c.s(func, "%_decode(%(''))"), @""); 76 | assert_snapshot!(c.s(func, "%_decode(%(x''))"), @""); 77 | assert_snapshot!(c.s(func, "%_decode(%('a'))"), @"61"); 78 | assert_snapshot!(c.s(func, "%_decode(%(x'00'))"), @"00"); 79 | assert_snapshot!(c.s(func, "%_decode(%('123456789'))"), @"313233343536373839"); 80 | assert_snapshot!(c.s(func, "%_decode(%(x'0123456789abcdef'))"), @"0123456789abcdef"); 81 | 82 | assert_snapshot!(c.bool(func, "%_test(%(x'0123456789abcdef'))"), @"true"); 83 | assert_snapshot!(c.bool(func, "%_test(x'0123456789abcdef')"), @"false"); 84 | ); 85 | } 86 | 87 | #[test] 88 | #[cfg(feature = "gzip")] 89 | fn gzip() { 90 | let c = Conn::default(); 91 | assert_snapshot!(c.q("gzip('')"), @"1f8b08000000000000ff03000000000000000000"); 92 | assert_snapshot!(c.q("gzip(x'')"), @"1f8b08000000000000ff03000000000000000000"); 93 | assert_snapshot!(c.q("gzip('a')"), @"1f8b08000000000000ff4b040043beb7e801000000"); 94 | assert_snapshot!(c.q("gzip(x'00')"), @"1f8b08000000000000ff6300008def02d201000000"); 95 | assert_snapshot!(c.q("gzip('123456789')"), @"1f8b08000000000000ff33343236313533b7b004002639f4cb09000000"); 96 | assert_snapshot!(c.q("gzip(x'0123456789abcdef')"), @"1f8b08000000000000ff6354764def5c7df63d00aed1c72808000000"); 97 | 98 | assert_snapshot!(c.q("gzip(x'0123', 0)"), @"1f8b08000000000004ff010200fdff0123cc52a5fa02000000"); 99 | assert_snapshot!(c.q("gzip(x'0123', 5)"), @"1f8b08000000000000ff63540600cc52a5fa02000000"); 100 | assert_snapshot!(c.q("gzip(x'0123', 9)"), @"1f8b08000000000002ff63540600cc52a5fa02000000"); 101 | } 102 | 103 | #[test] 104 | #[cfg(feature = "brotli")] 105 | fn brotli() { 106 | let c = Conn::default(); 107 | assert_snapshot!(c.q("brotli('')"), @"3b"); 108 | assert_snapshot!(c.q("brotli(x'')"), @"3b"); 109 | assert_snapshot!(c.q("brotli('a')"), @"0b00806103"); 110 | assert_snapshot!(c.q("brotli(x'00')"), @"0b00800003"); 111 | assert_snapshot!(c.q("brotli('123456789')"), @"0b048031323334353637383903"); 112 | assert_snapshot!(c.q("brotli(x'0123456789abcdef')"), @"8b03800123456789abcdef03"); 113 | 114 | assert_snapshot!(c.q("brotli(x'0123', 0)"), @"8b0080012303"); 115 | assert_snapshot!(c.q("brotli(x'0123', 10)"), @"8b0080012303"); 116 | assert_snapshot!(c.q("brotli(x'0123', 99)"), @"8b0080012303"); 117 | } 118 | 119 | #[test] 120 | #[cfg(feature = "bzip2")] 121 | fn bzip2() { 122 | let c = Conn::default(); 123 | assert_snapshot!(c.q("bzip2('')"), @"425a683617724538509000000000"); 124 | assert_snapshot!(c.q("bzip2(x'')"), @"425a683617724538509000000000"); 125 | assert_snapshot!(c.q("bzip2('a')"), @"425a683631415926535919939b6b00000001002000200021184682ee48a70a120332736d60"); 126 | assert_snapshot!(c.q("bzip2(x'00')"), @"425a6836314159265359b1f7404b00000040004000200021184682ee48a70a12163ee80960"); 127 | assert_snapshot!(c.q("bzip2('123456789')"), @"425a6836314159265359fc89191800000008003fe02000220d0c0832621e0def29c177245385090fc8919180"); 128 | assert_snapshot!(c.q("bzip2(x'0123456789abcdef')"), @"425a6836314159265359f61121f9000000555520000800020000800020000800020000a000310c08191a69933573f945dc914e14243d84487e40"); 129 | 130 | // errors 131 | assert_snapshot!(c.q("bzip2(x'0123', 0)"), @"The optional second argument to bzip2() must be between 1 and 9"); 132 | assert_snapshot!(c.q("bzip2(x'0123', 10)"), @"The optional second argument to bzip2() must be between 1 and 9"); 133 | assert_snapshot!(c.q("bzip2(x'0123', 99)"), @"The optional second argument to bzip2() must be between 1 and 9"); 134 | } 135 | 136 | #[test] 137 | #[cfg(feature = "bsdiff4")] 138 | fn bsdiff4() { 139 | let c = Conn::default(); 140 | assert_snapshot!(c.q("bsdiff4('', '')"), @"42534449464634300e000000000000000e000000000000000000000000000000425a683617724538509000000000425a683617724538509000000000425a683617724538509000000000"); 141 | assert_snapshot!(c.q("bsdiff4(x'', x'')"), @"42534449464634300e000000000000000e000000000000000000000000000000425a683617724538509000000000425a683617724538509000000000425a683617724538509000000000"); 142 | assert_snapshot!(c.q("bsdiff4('a', '')"), @"42534449464634300e000000000000000e000000000000000000000000000000425a683617724538509000000000425a683617724538509000000000425a683617724538509000000000"); 143 | assert_snapshot!(c.q("bsdiff4(x'00', '')"), @"42534449464634300e000000000000000e000000000000000000000000000000425a683617724538509000000000425a683617724538509000000000425a683617724538509000000000"); 144 | assert_snapshot!(c.q("bsdiff4('123456789', '123456789')"), @"42534449464634302b0000000000000025000000000000000900000000000000425a6836314159265359439c5a03000000e00040200c00200030cd341268369327177245385090439c5a03425a6836314159265359752890670000004000420020002100828317724538509075289067425a683617724538509000000000"); 145 | assert_snapshot!(c.q("bsdiff4(x'0123456789abcdef', x'0123456789abcdef')"), @"42534449464634302b0000000000000025000000000000000800000000000000425a68363141592653591533c7b1000000e00040400c00200030cd3412683693271772453850901533c7b1425a683631415926535996fb44a60000004000440020002100828317724538509096fb44a6425a683617724538509000000000"); 146 | assert_snapshot!(c.q("bsdiff4('1234', '5678349A')"), @"42534449464634302a000000000000000e000000000000000800000000000000425a68363141592653591769093d00000140004c402000219a68334d32b6c078bb9229c28480bb4849e8425a683617724538509000000000425a683631415926535986c673c30000010c000fe0200020002190c210c08795392f8bb9229c2848436339e180"); 147 | assert_snapshot!(c.q("bsdiff4(x'1234', x'5678349A')"), @"42534449464634302c000000000000000e000000000000000400000000000000425a68363141592653590dcec6b900000140005c00200030cd34129ea7a357029ee1772453850900dcec6b90425a683617724538509000000000425a6836314159265359ff2387120000008aa004000100004000102000219a68334d32bc5dc914e14243fc8e1c48"); 148 | 149 | assert_snapshot!(c.q("bspatch4('', bsdiff4('', ''))"), @""); 150 | assert_snapshot!(c.q("bspatch4(x'', bsdiff4(x'', x''))"), @""); 151 | assert_snapshot!(c.q("bspatch4('a', bsdiff4('a', ''))"), @""); 152 | assert_snapshot!(c.q("bspatch4(x'00', bsdiff4(x'00', ''))"), @""); 153 | assert_snapshot!(c.q("bspatch4('123456789', bsdiff4('123456789', '123456789'))"), @"313233343536373839"); 154 | assert_snapshot!(c.q("bspatch4(x'0123456789abcdef', bsdiff4(x'0123456789abcdef', x'0123456789abcdef'))"), @"0123456789abcdef"); 155 | assert_snapshot!(c.q("bspatch4('1234', bsdiff4('1234', '5678349A'))"), @"3536373833343941"); 156 | assert_snapshot!(c.q("bspatch4(x'1234', bsdiff4(x'1234', x'5678349A'))"), @"5678349a"); 157 | 158 | // nulls 159 | assert_snapshot!(c.q("bsdiff4(NULL, NULL)"), @"NULL"); 160 | assert_snapshot!(c.q("bsdiff4('abc', NULL)"), @"NULL"); 161 | assert_snapshot!(c.q("bsdiff4(NULL, 'abc')"), @"NULL"); 162 | assert_snapshot!(c.q("bspatch4(NULL, NULL)"), @"NULL"); 163 | assert_snapshot!(c.q("bspatch4('abc', NULL)"), @"NULL"); 164 | assert_snapshot!(c.q("bspatch4(NULL, 'abc')"), @"NULL"); 165 | 166 | // errors 167 | assert_snapshot!(c.q("bsdiff4(x'0123')"), @"wrong number of arguments to function bsdiff4()"); 168 | assert_snapshot!(c.q("bsdiff4(x'0123', x'4567', x'89')"), @"wrong number of arguments to function bsdiff4()"); 169 | assert_snapshot!(c.q("bspatch4(x'0123')"), @"wrong number of arguments to function bspatch4()"); 170 | assert_snapshot!(c.q("bspatch4(x'0123', x'4567', x'89')"), @"wrong number of arguments to function bspatch4()"); 171 | } 172 | 173 | #[test] 174 | #[cfg(feature = "bsdiffraw")] 175 | fn bsdiffraw() { 176 | let c = Conn::default(); 177 | assert_snapshot!(c.q("bsdiffraw('', '')"), @""); 178 | assert_snapshot!(c.q("bsdiffraw(x'', x'')"), @""); 179 | assert_snapshot!(c.q("bsdiffraw('a', '')"), @""); 180 | assert_snapshot!(c.q("bsdiffraw(x'00', '')"), @""); 181 | assert_snapshot!(c.q("bsdiffraw('123456789', '123456789')"), @"090000000000000000000000000000000900000000000080000000000000000000"); 182 | assert_snapshot!(c.q("bsdiffraw(x'0123456789abcdef', x'0123456789abcdef')"), @"0800000000000000000000000000000008000000000000800000000000000000"); 183 | assert_snapshot!(c.q("bsdiffraw('1234', '5678349A')"), @"0000000000000000080000000000000003000000000000003536373833343941"); 184 | assert_snapshot!(c.q("bsdiffraw(x'1234', x'5678349A')"), @"0000000000000000040000000000000001000000000000005678349a"); 185 | 186 | assert_snapshot!(c.q("bspatchraw('', bsdiffraw('', ''))"), @""); 187 | assert_snapshot!(c.q("bspatchraw(x'', bsdiffraw(x'', x''))"), @""); 188 | assert_snapshot!(c.q("bspatchraw('a', bsdiffraw('a', ''))"), @""); 189 | assert_snapshot!(c.q("bspatchraw(x'00', bsdiffraw(x'00', ''))"), @""); 190 | assert_snapshot!(c.q("bspatchraw('123456789', bsdiffraw('123456789', '123456789'))"), @"313233343536373839"); 191 | assert_snapshot!(c.q("bspatchraw(x'0123456789abcdef', bsdiffraw(x'0123456789abcdef', x'0123456789abcdef'))"), @"0123456789abcdef"); 192 | assert_snapshot!(c.q("bspatchraw('1234', bsdiffraw('1234', '5678349A'))"), @"3536373833343941"); 193 | assert_snapshot!(c.q("bspatchraw(x'1234', bsdiffraw(x'1234', x'5678349A'))"), @"5678349a"); 194 | 195 | // nulls 196 | assert_snapshot!(c.q("bsdiffraw(NULL, NULL)"), @"NULL"); 197 | assert_snapshot!(c.q("bsdiffraw('abc', NULL)"), @"NULL"); 198 | assert_snapshot!(c.q("bsdiffraw(NULL, 'abc')"), @"NULL"); 199 | assert_snapshot!(c.q("bspatchraw(NULL, NULL)"), @"NULL"); 200 | assert_snapshot!(c.q("bspatchraw('abc', NULL)"), @"NULL"); 201 | assert_snapshot!(c.q("bspatchraw(NULL, 'abc')"), @"NULL"); 202 | 203 | // errors 204 | assert_snapshot!(c.q("bsdiffraw(x'0123')"), @"wrong number of arguments to function bsdiffraw()"); 205 | assert_snapshot!(c.q("bsdiffraw(x'0123', x'4567', x'89')"), @"wrong number of arguments to function bsdiffraw()"); 206 | assert_snapshot!(c.q("bspatchraw(x'0123')"), @"wrong number of arguments to function bspatchraw()"); 207 | assert_snapshot!(c.q("bspatchraw(x'0123', x'4567', x'89')"), @"wrong number of arguments to function bspatchraw()"); 208 | } 209 | --------------------------------------------------------------------------------