├── .env ├── .gitattributes ├── .github ├── FUNDING.yml └── workflows │ ├── c-tests.yaml │ ├── c-valgrind.yaml │ ├── publish.yaml │ ├── py-tests.yaml │ ├── rs-tests.yml │ └── rs-valgrind.yml ├── .gitignore ├── .gitmodules ├── .prettierignore ├── .prettierrc ├── LICENSE ├── Makefile ├── README.md ├── core ├── .clang-format ├── .gitignore ├── CHANGELOG.md ├── Makefile ├── README.md ├── all-ios-loadable.sh ├── all-ios-static.sh ├── nodejs-helper.d.ts ├── nodejs-helper.js ├── nodejs-install-helper.js ├── package.json ├── pnpm-lock.yaml ├── rs │ ├── bundle │ │ ├── Cargo.lock │ │ ├── Cargo.toml │ │ ├── Cargo.toml.integration-test │ │ ├── README.md │ │ ├── keep-stack-sizes.x │ │ ├── rust-toolchain.toml │ │ └── src │ │ │ └── lib.rs │ ├── bundle_static │ │ ├── Cargo.lock │ │ ├── Cargo.toml │ │ ├── rust-toolchain.toml │ │ └── src │ │ │ └── lib.rs │ ├── core │ │ ├── Cargo.lock │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── explore.sql │ │ ├── rust-toolchain.toml │ │ └── src │ │ │ ├── alter.rs │ │ │ ├── automigrate.rs │ │ │ ├── backfill.rs │ │ │ ├── bootstrap.rs │ │ │ ├── c.rs │ │ │ ├── changes_vtab.rs │ │ │ ├── changes_vtab_read.rs │ │ │ ├── changes_vtab_write.rs │ │ │ ├── compare_values.rs │ │ │ ├── config.rs │ │ │ ├── consts.rs │ │ │ ├── create_cl_set_vtab.rs │ │ │ ├── create_crr.rs │ │ │ ├── db_version.rs │ │ │ ├── ext_data.rs │ │ │ ├── is_crr.rs │ │ │ ├── lib.rs │ │ │ ├── local_writes │ │ │ ├── after_delete.rs │ │ │ ├── after_insert.rs │ │ │ ├── after_update.rs │ │ │ └── mod.rs │ │ │ ├── pack_columns.rs │ │ │ ├── sha.rs │ │ │ ├── stmt_cache.rs │ │ │ ├── tableinfo.rs │ │ │ ├── teardown.rs │ │ │ ├── test_exports.rs │ │ │ ├── triggers.rs │ │ │ ├── unpack_columns_vtab.rs │ │ │ └── util.rs │ ├── fractindex-core │ │ ├── Cargo.lock │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── explore.sql │ │ ├── rust-toolchain.toml │ │ └── src │ │ │ ├── as_ordered.rs │ │ │ ├── fractindex.rs │ │ │ ├── fractindex_view.rs │ │ │ ├── lib.rs │ │ │ └── util.rs │ └── integration_check │ │ ├── Cargo.lock │ │ ├── Cargo.toml │ │ ├── notes.md │ │ ├── rust-toolchain.toml │ │ └── src │ │ ├── lib.rs │ │ └── t │ │ ├── automigrate.rs │ │ ├── backfill.rs │ │ ├── fract.rs │ │ ├── mod.rs │ │ ├── pack_columns.rs │ │ ├── pk_only_tables.rs │ │ ├── pk_update.rs │ │ ├── sync_bit_honored.rs │ │ ├── tableinfo.rs │ │ ├── teardown.rs │ │ ├── test_cl_set_vtab.rs │ │ └── test_db_version.rs └── src │ ├── changes-vtab-rowid.test.c │ ├── changes-vtab.c │ ├── changes-vtab.h │ ├── changes-vtab.test.c │ ├── consts.h │ ├── core_init.c │ ├── crsqlite.c │ ├── crsqlite.h │ ├── crsqlite.test.c │ ├── ext-data.c │ ├── ext-data.h │ ├── ext-data.test.c │ ├── ext.h │ ├── fuzzer.cc │ ├── is-crr.test.c │ ├── rows-impacted.test.c │ ├── rs-fract.test.c │ ├── rust.h │ ├── sandbox.test.c │ ├── sqlite │ ├── Makefile │ ├── shell.c │ ├── sqlite3.c │ ├── sqlite3.h │ └── sqlite3ext.h │ ├── tests.c │ └── util.h ├── libsql-sync.sh ├── notes.md └── py ├── README.md ├── correctness ├── .gitignore ├── CHANGELOG.md ├── install-and-test.sh ├── notes.md ├── package.json ├── prior-dbs │ ├── v0.12.0.prior-db │ ├── v0.13.0.prior-db │ └── v0.15.0.prior-db ├── pyproject.toml ├── requirements.txt ├── spec.md ├── src │ ├── crsql_correctness.egg-info │ │ ├── PKG-INFO │ │ ├── SOURCES.txt │ │ ├── dependency_links.txt │ │ └── top_level.txt │ └── crsql_correctness │ │ └── __init__.py ├── test.sh └── tests │ ├── test_as_ordered.py │ ├── test_cl_merging.py │ ├── test_cl_triggers.py │ ├── test_commit_alter_perf.py │ ├── test_config.py │ ├── test_crsql_changes_filters.py │ ├── test_dbversion.py │ ├── test_insert_new_rows.py │ ├── test_lookaside_key_creation.py │ ├── test_prior_versions.py │ ├── test_sandbox.py │ ├── test_schema_modification.py │ ├── test_sentinel_omission.py │ ├── test_seq.py │ ├── test_site_id_lookaside.py │ ├── test_siteid.py │ ├── test_sync.py │ ├── test_sync_bit.py │ ├── test_sync_prop.py │ └── test_update_rows.py └── perf ├── .gitignore ├── notes.md └── perf.ipynb /.env: -------------------------------------------------------------------------------- 1 | PYTHONPATH=./py/correctness/src 2 | CRSQLITE_NOPREBUILD=1 3 | CRSQLITE_COMMIT_SHA=dev -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | core/src/sqlite/* linguist-vendored 2 | py/perf/* linguist-documentation 3 | 4 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [tantaman] 4 | -------------------------------------------------------------------------------- /.github/workflows/c-tests.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | name: "c-tests" 4 | jobs: 5 | build: 6 | name: Testing on ${{ matrix.os }} 7 | runs-on: ${{ matrix.os }} 8 | strategy: 9 | matrix: 10 | include: 11 | - os: ubuntu-latest 12 | - os: windows-2022 13 | - os: macos-latest 14 | steps: 15 | - uses: actions/checkout@v3 16 | with: 17 | submodules: true 18 | 19 | - name: Load .env file 20 | uses: xom9ikk/dotenv@v2 21 | with: 22 | path: ./ 23 | 24 | - name: Windows rust toolchain 25 | if: runner.os == 'Windows' 26 | run: | 27 | rm core/rs/integration_check/rust-toolchain.toml 28 | rustup component add rust-src --toolchain nightly-2023-10-05-x86_64-pc-windows-gnu 29 | rustup default nightly-2023-10-05-x86_64-pc-windows-gnu 30 | 31 | - name: Test 32 | run: | 33 | cd core 34 | make test 35 | -------------------------------------------------------------------------------- /.github/workflows/c-valgrind.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | name: "c-valgrind" 4 | jobs: 5 | build: 6 | name: Testing on ${{ matrix.os }} 7 | runs-on: ${{ matrix.os }} 8 | strategy: 9 | matrix: 10 | include: 11 | - os: ubuntu-latest 12 | #- os: windows-2022 13 | #- os: macos-latest 14 | steps: 15 | - uses: actions/checkout@v3 16 | with: 17 | submodules: true 18 | 19 | - name: Load .env file 20 | uses: xom9ikk/dotenv@v2 21 | with: 22 | path: ./ 23 | 24 | - name: Install valgrind 25 | run: sudo apt update && sudo apt install -y valgrind 26 | 27 | - name: Valgrind Test 28 | run: | 29 | cd core 30 | make test 31 | valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes -s dist/test 32 | -------------------------------------------------------------------------------- /.github/workflows/py-tests.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | name: "py-tests" 4 | jobs: 5 | build: 6 | name: Testing on ${{ matrix.os }} 7 | runs-on: ${{ matrix.os }} 8 | defaults: 9 | run: 10 | shell: bash -el {0} 11 | strategy: 12 | matrix: 13 | include: 14 | - os: ubuntu-latest 15 | - os: macos-latest 16 | steps: 17 | - uses: actions/checkout@v3 18 | with: 19 | submodules: true 20 | 21 | - name: Load .env file 22 | uses: xom9ikk/dotenv@v2 23 | with: 24 | path: ./ 25 | 26 | - name: Make loadable 27 | run: | 28 | cd core 29 | make loadable 30 | 31 | - uses: conda-incubator/setup-miniconda@v2 32 | with: 33 | auto-update-conda: true 34 | auto-activate-base: true 35 | activate-environment: anaconda-client-env 36 | python-version: "3.10" 37 | 38 | - name: Install SQLite 39 | run: conda install sqlite 40 | 41 | - name: Check SQLite Version 42 | run: echo "import sqlite3; print(sqlite3.sqlite_version)" | python 43 | 44 | # - name: Install Python 45 | # uses: actions/setup-python@v4 46 | # with: 47 | # python-version: "3.10" 48 | 49 | - name: Install pip 50 | run: | 51 | python -m pip install --upgrade pip 52 | 53 | - name: Test 54 | run: cd py/correctness && ./install-and-test.sh 55 | -------------------------------------------------------------------------------- /.github/workflows/rs-tests.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | name: "rs-tests" 4 | jobs: 5 | build: 6 | name: Testing on ${{ matrix.os }} 7 | runs-on: ${{ matrix.os }} 8 | strategy: 9 | matrix: 10 | include: 11 | - os: ubuntu-latest 12 | # - os: windows-2022 # Windows is complete nonsense. If someone wants to figure out how to make tests run there go ahead. 13 | - os: macos-latest 14 | steps: 15 | - uses: actions/checkout@v3 16 | with: 17 | submodules: true 18 | 19 | - name: Load .env file 20 | uses: xom9ikk/dotenv@v2 21 | with: 22 | path: ./ 23 | 24 | - name: Windows rust toolchain 25 | if: runner.os == 'Windows' 26 | run: | 27 | find . -name rust-toolchain.toml -exec echo rm {} \; 28 | rustup component add rust-src --toolchain nightly-2023-10-05-x86_64-pc-windows-gnu 29 | rustup default nightly-2023-10-05-x86_64-pc-windows-gnu 30 | 31 | - name: Test Fractindex 32 | run: | 33 | cd core/rs/fractindex-core 34 | cargo test --features=loadable_extension 35 | 36 | - name: Test Core 37 | run: | 38 | cd core/rs/core 39 | cargo test --features=loadable_extension 40 | -------------------------------------------------------------------------------- /.github/workflows/rs-valgrind.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | name: "rs-valgrind" 4 | jobs: 5 | build: 6 | name: Testing on ${{ matrix.os }} 7 | runs-on: ${{ matrix.os }} 8 | strategy: 9 | matrix: 10 | include: 11 | - os: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | with: 15 | submodules: true 16 | 17 | - name: Load .env file 18 | uses: xom9ikk/dotenv@v2 19 | with: 20 | path: ./ 21 | 22 | - name: Install valgrind 23 | run: sudo apt update && sudo apt install -y valgrind 24 | 25 | - name: Install Cargo Valgrind 26 | run: | 27 | cargo install cargo-valgrind 28 | 29 | - name: Test Fractindex 30 | run: | 31 | cd core/rs/fractindex-core 32 | cargo valgrind test --features=loadable_extension 33 | 34 | - name: Test Core 35 | run: | 36 | cd core/rs/core 37 | cargo valgrind test --features=loadable_extension 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | out.db 3 | .vscode 4 | a.out 5 | dist/ 6 | bench/ 7 | .ipynb_checkpoints/ 8 | *.db 9 | __pycache__/ 10 | node_modules/ 11 | build/ 12 | tsconfig.tsbuildinfo 13 | .turbo/ 14 | target/ 15 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "core/rs/sqlite-rs-embedded"] 2 | path = core/rs/sqlite-rs-embedded 3 | url = git@github.com:vlcn-io/sqlite-rs-embedded.git 4 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | LICENSE 2 | spec.md 3 | deps/ 4 | wa-sqlite/ 5 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5", 3 | "tabWidth": 2, 4 | "singleQuote": false 5 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 One Law LLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | git-deps = core/rs/sqlite-rs-embedded 2 | 3 | .EXPORT_ALL_VARIABLES: 4 | CRSQLITE_NOPREBUILD = 1 5 | 6 | all: crsqlite 7 | 8 | $(git-deps): 9 | git submodule update --init --recursive 10 | 11 | 12 | crsqlite: $(git-deps) 13 | cd core; \ 14 | make loadable 15 | 16 | clean: 17 | cd core && make clean 18 | 19 | .PHONY: crsqlite all clean 20 | -------------------------------------------------------------------------------- /core/.clang-format: -------------------------------------------------------------------------------- 1 | # Use the Google style in this project. 2 | BasedOnStyle: Google 3 | 4 | # Some folks prefer to write "int& foo" while others prefer "int &foo". The 5 | # Google Style Guide only asks for consistency within a project, we chose 6 | # "int &foo" for this project: 7 | DerivePointerAlignment: false 8 | PointerAlignment: Right 9 | -------------------------------------------------------------------------------- /core/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | target/ 3 | dist/ 4 | dbg/ 5 | dist-ios/ 6 | dist-ios-sim/ 7 | dist-macos/ 8 | test_leak_condition 9 | -------------------------------------------------------------------------------- /core/README.md: -------------------------------------------------------------------------------- 1 | See main readme: https://github.com/vlcn-io/cr-sqlite/blob/main/README.md 2 | -------------------------------------------------------------------------------- /core/all-ios-loadable.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # a hacky script to make all the various ios targets. 4 | # once we have something consistently working we'll streamline all of this. 5 | 6 | BUILD_DIR=./build 7 | DIST_PACKAGE_DIR=./dist 8 | 9 | function createXcframework() { 10 | plist=$(cat << EOF 11 | 12 | 13 | 14 | 15 | CFBundleDevelopmentRegion 16 | en 17 | CFBundleExecutable 18 | crsqlite 19 | CFBundleIdentifier 20 | io.vlcn.crsqlite 21 | CFBundleInfoDictionaryVersion 22 | 6.0 23 | CFBundlePackageType 24 | FMWK 25 | CFBundleSignature 26 | ???? 27 | CFBundleVersion 28 | 1.0.0 29 | CFBundleShortVersionString 30 | 1.0.0 31 | MinimumOSVersion 32 | 8.0 33 | 34 | 35 | EOF 36 | ) 37 | printf "\n\n\t\t===================== create ios device framework =====================\n\n" 38 | mkdir -p "${BUILD_DIR}/ios-arm64/crsqlite.framework" 39 | echo "${plist}" > "${BUILD_DIR}/ios-arm64/crsqlite.framework/Info.plist" 40 | cp -f "./dist-ios/crsqlite-aarch64-apple-ios.dylib" "${BUILD_DIR}/ios-arm64/crsqlite.framework/crsqlite" 41 | install_name_tool -id "@rpath/crsqlite.framework/crsqlite" "${BUILD_DIR}/ios-arm64/crsqlite.framework/crsqlite" 42 | 43 | printf "\n\n\t\t===================== create ios simulator framework =====================\n\n" 44 | mkdir -p "${BUILD_DIR}/ios-arm64_x86_64-simulator/crsqlite.framework" 45 | echo "${plist}" > "${BUILD_DIR}/ios-arm64_x86_64-simulator/crsqlite.framework/Info.plist" 46 | cp -p "./dist-ios-sim/crsqlite-universal-ios-sim.dylib" "${BUILD_DIR}/ios-arm64_x86_64-simulator/crsqlite.framework/crsqlite" 47 | install_name_tool -id "@rpath/crsqlite.framework/crsqlite" "${BUILD_DIR}/ios-arm64_x86_64-simulator/crsqlite.framework/crsqlite" 48 | 49 | printf "\n\n\t\t===================== create ios xcframework =====================\n\n" 50 | rm -rf "${BUILD_DIR}/crsqlite.xcframework" 51 | xcodebuild -create-xcframework -framework "${BUILD_DIR}/ios-arm64/crsqlite.framework" -framework "${BUILD_DIR}/ios-arm64_x86_64-simulator/crsqlite.framework" -output "${BUILD_DIR}/crsqlite.xcframework" 52 | 53 | mkdir -p ${DIST_PACKAGE_DIR} 54 | cp -Rf "${BUILD_DIR}/crsqlite.xcframework" "${DIST_PACKAGE_DIR}/crsqlite.xcframework" 55 | cd ${DIST_PACKAGE_DIR} 56 | tar -czvf crsqlite-ios-dylib.xcframework.tar.gz crsqlite.xcframework 57 | rm -rf ${BUILD_DIR} 58 | } 59 | 60 | # Make all the non-simulator libs 61 | # Package into a universal ios lib 62 | mkdir -p ./dist-ios 63 | 64 | # TODO: fix things up to not require a clean before each target. 65 | make clean 66 | export IOS_TARGET=aarch64-apple-ios; make loadable 67 | cp ./dist/crsqlite.dylib ./dist-ios/crsqlite-aarch64-apple-ios.dylib 68 | 69 | mkdir -p ./dist-ios-sim 70 | 71 | make clean 72 | export IOS_TARGET=aarch64-apple-ios-sim; make loadable 73 | cp ./dist/crsqlite.dylib ./dist-ios-sim/crsqlite-aarch64-apple-ios-sim.dylib 74 | 75 | make clean 76 | export IOS_TARGET=x86_64-apple-ios; make loadable 77 | cp ./dist/crsqlite.dylib ./dist-ios-sim/crsqlite-x86_64-apple-ios-sim.dylib 78 | 79 | cd ./dist-ios-sim 80 | lipo crsqlite-aarch64-apple-ios-sim.dylib crsqlite-x86_64-apple-ios-sim.dylib -create -output crsqlite-universal-ios-sim.dylib 81 | cd .. 82 | 83 | createXcframework 84 | -------------------------------------------------------------------------------- /core/all-ios-static.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # a hacky script to make all the various ios targets. 4 | # once we have something consistently working we'll streamline all of this. 5 | 6 | # Make all the non-simulator libs 7 | # Package into a universal ios lib 8 | mkdir -p ./dist-ios 9 | 10 | # TODO: fix things up to not require a clean before each target. 11 | make clean 12 | export IOS_TARGET=aarch64-apple-ios; make static 13 | cp ./dist/crsqlite-aarch64-apple-ios.a ./dist-ios 14 | 15 | make clean 16 | export IOS_TARGET=armv7-apple-ios; make static 17 | cp ./dist/crsqlite-armv7-apple-ios.a ./dist-ios 18 | 19 | make clean 20 | export IOS_TARGET=armv7s-apple-ios; make static 21 | cp ./dist/crsqlite-armv7s-apple-ios.a ./dist-ios 22 | 23 | cd ./dist-ios 24 | lipo crsqlite-aarch64-apple-ios.a crsqlite-armv7-apple-ios.a crsqlite-armv7s-apple-ios.a -create -output crsqlite-universal-ios.a 25 | 26 | cd .. 27 | # === 28 | 29 | # Make the simlator libs 30 | # Package into a universal ios sim lib 31 | mkdir -p ./dist-ios-sim 32 | 33 | make clean 34 | export IOS_TARGET=aarch64-apple-ios-sim; make static 35 | cp ./dist/crsqlite-aarch64-apple-ios-sim.a ./dist-ios-sim 36 | 37 | make clean 38 | export IOS_TARGET=x86_64-apple-ios; make static 39 | cp ./dist/crsqlite-x86_64-apple-ios.a ./dist-ios-sim 40 | 41 | cd ./dist-ios-sim 42 | lipo crsqlite-aarch64-apple-ios-sim.a crsqlite-x86_64-apple-ios.a -create -output crsqlite-universal-ios-sim.a 43 | 44 | cd .. 45 | # === 46 | 47 | # Make the macos static lib 48 | mkdir -p ./dist-macos 49 | make clean 50 | unset IOS_TARGET 51 | export CI_MAYBE_TARGET="aarch64-apple-darwin"; make static 52 | 53 | cp ./dist/crsqlite-aarch64-apple-darwin.a ./dist-macos 54 | 55 | -------------------------------------------------------------------------------- /core/nodejs-helper.d.ts: -------------------------------------------------------------------------------- 1 | export declare const extensionPath: string; 2 | -------------------------------------------------------------------------------- /core/nodejs-helper.js: -------------------------------------------------------------------------------- 1 | // Exports the path to the extension for those using 2 | // crsqlite in a Node.js environment. 3 | import * as url from "url"; 4 | import { join } from "path"; 5 | const __dirname = url.fileURLToPath(new URL(".", import.meta.url)); 6 | 7 | export const extensionPath = join(__dirname, "dist", "crsqlite"); 8 | -------------------------------------------------------------------------------- /core/nodejs-install-helper.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 1. Checks the current OS and CPU architecture 3 | * 2. Copies pre-built binaries from the `binaries` directory to the `dist` directory if one exists 4 | * 3. Otherwise, lets the standard install process via `make` take over 5 | */ 6 | import { join } from "path"; 7 | import fs from "fs"; 8 | import https from "https"; 9 | import pkg from "./package.json" with { type: "json" }; 10 | import { exec } from "child_process"; 11 | let { version } = pkg; 12 | 13 | let arch = process.arch; 14 | let os = process.platform; 15 | let ext = "unknown"; 16 | version = "v" + version; 17 | 18 | if (process.env.CRSQLITE_NOPREBUILD) { 19 | console.log("CRSQLITE_NOPREBUILD env variable is set. Building from source."); 20 | buildFromSource(); 21 | } else { 22 | // todo: check msys? 23 | if (["win32", "cygwin"].includes(process.platform)) { 24 | os = "win"; 25 | } 26 | 27 | // manual ovverides for testing 28 | // arch = "x86_64"; 29 | // os = "linux"; 30 | // version = "prebuild-test.11"; 31 | 32 | switch (os) { 33 | case "darwin": 34 | ext = "dylib"; 35 | break; 36 | case "linux": 37 | ext = "so"; 38 | break; 39 | case "win": 40 | ext = "dll"; 41 | break; 42 | } 43 | 44 | switch (arch) { 45 | case "x64": 46 | arch = "x86_64"; 47 | break; 48 | case "arm64": 49 | arch = "aarch64"; 50 | break; 51 | } 52 | 53 | const binaryUrl = `https://github.com/vlcn-io/cr-sqlite/releases/download/${version}/crsqlite-${os}-${arch}.zip`; 54 | console.log(`Look for prebuilt binary from ${binaryUrl}`); 55 | const distPath = join("dist", `crsqlite.${ext}`); 56 | 57 | if (!fs.existsSync(join(".", "dist"))) { 58 | fs.mkdirSync(join(".", "dist")); 59 | } 60 | 61 | if (fs.existsSync(distPath)) { 62 | console.log("Binary already present and installed."); 63 | process.exit(0); 64 | } 65 | 66 | // download the file at the url, if it exists 67 | let redirectCount = 0; 68 | function get(url, cb) { 69 | https.get(url, (res) => { 70 | if (res.statusCode === 302 || res.statusCode === 301) { 71 | ++redirectCount; 72 | if (redirectCount > 5) { 73 | throw new Error("Too many redirects"); 74 | } 75 | get(res.headers.location, cb); 76 | } else if (res.statusCode === 200) { 77 | cb(res); 78 | } else { 79 | cb(null); 80 | } 81 | }); 82 | } 83 | 84 | get(binaryUrl, (res) => { 85 | if (res == null) { 86 | console.log("No prebuilt binary available. Building from source."); 87 | buildFromSource(); 88 | return; 89 | } 90 | 91 | const file = fs.createWriteStream(join("dist", "crsqlite.zip")); 92 | res.pipe(file); 93 | file.on("finish", () => { 94 | file.close(); 95 | console.log("Prebuilt binary downloaded"); 96 | process.chdir(join(".", "dist")); 97 | exec("unzip crsqlite.zip", (err, stdout, stderr) => { 98 | if (err) { 99 | console.log("Error extracting"); 100 | console.log(err.message); 101 | process.exit(1); 102 | } 103 | if (stderr) { 104 | console.log(stderr); 105 | } 106 | console.log("Prebuilt binary extracted"); 107 | process.exit(0); 108 | }); 109 | }); 110 | // unzipper incorrectly unzips the file -- it becomes unloadable by sqlite. 111 | // res.pipe(unzipper.Extract({ path: join(".", "dist") })).on("close", () => { 112 | // console.log("Prebuilt binary downloaded"); 113 | // process.exit(0); 114 | // }); 115 | }); 116 | } 117 | 118 | function buildFromSource() { 119 | console.log("Building from source"); 120 | exec("make loadable", (err, stdout, stderr) => { 121 | if (err) { 122 | console.log("Error building from source"); 123 | console.log(err.message); 124 | process.exit(1); 125 | } 126 | if (stderr) { 127 | console.log(stderr); 128 | } 129 | console.log("Built from source"); 130 | console.log(stdout); 131 | process.exit(0); 132 | }); 133 | } 134 | -------------------------------------------------------------------------------- /core/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@vlcn.io/crsqlite", 3 | "version": "0.16.3", 4 | "description": "CR-SQLite loadable extension", 5 | "homepage": "https://vlcn.io", 6 | "repository": { 7 | "type": "git", 8 | "url": "git://github.com/vlcn-io/cr-sqlite" 9 | }, 10 | "files": [ 11 | "Makefile", 12 | "src/**", 13 | "rs/**/*.rs", 14 | "rs/**/*.toml", 15 | "rs/**/*.lock", 16 | "rs/**/*.h", 17 | "nodejs-helper.d.ts", 18 | "nodejs-helper.js", 19 | "nodejs-install-helper.js" 20 | ], 21 | "type": "module", 22 | "main": "nodejs-helper.js", 23 | "types": "nodejs-helper.d.ts", 24 | "scripts": { 25 | "install": "node ./nodejs-install-helper.js", 26 | "test": "make test", 27 | "deep-clean": "rm -rf ./dbg || true && rm -rf ./dist || true" 28 | }, 29 | "license": "Apache 2", 30 | "keywords": [ 31 | "sql", 32 | "sqlite", 33 | "sqlite3", 34 | "crdt" 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /core/rs/bundle/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "crsql_bundle" 3 | version = "0.1.0" 4 | edition = "2021" 5 | authors = ["Matt Wonlaw"] 6 | keywords = ["sqlite", "cr-sqlite"] 7 | license = "Apache 2" 8 | 9 | [lib] 10 | name = "crsql_bundle" 11 | crate-type = ["rlib"] 12 | 13 | # "cdylib" <-- only enable cdylib if loadable_extension is enabled 14 | 15 | [dependencies] 16 | crsql_fractindex_core = {path="../fractindex-core"} 17 | crsql_core = { path="../core" } 18 | sqlite_nostd = { path="../sqlite-rs-embedded/sqlite_nostd" } 19 | 20 | [profile.dev] 21 | panic = "abort" 22 | 23 | [profile.release] 24 | panic = "abort" 25 | 26 | [features] 27 | test = ["crsql_core/test"] 28 | libsql = ["crsql_core/libsql"] 29 | loadable_extension = [ 30 | "sqlite_nostd/loadable_extension", 31 | "crsql_fractindex_core/loadable_extension", 32 | "crsql_core/loadable_extension" 33 | ] 34 | static = [ 35 | "sqlite_nostd/static", 36 | "crsql_fractindex_core/static", 37 | "crsql_core/static" 38 | ] 39 | omit_load_extension = [ 40 | "sqlite_nostd/omit_load_extension", 41 | "crsql_fractindex_core/omit_load_extension", 42 | "crsql_core/omit_load_extension" 43 | ] 44 | -------------------------------------------------------------------------------- /core/rs/bundle/Cargo.toml.integration-test: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "crsql_bundle" 3 | version = "0.1.0" 4 | edition = "2021" 5 | authors = ["Matt Wonlaw"] 6 | keywords = ["sqlite", "cr-sqlite"] 7 | license = "Apache 2" 8 | 9 | [lib] 10 | name = "crsql_bundle" 11 | crate-type = ["rlib"] 12 | 13 | # "cdylib" <-- only enable cdylib if loadable_extension is enabled 14 | 15 | [dependencies] 16 | crsql_fractindex_core = {path="../fractindex-core"} 17 | crsql_core = { path="../core" } 18 | sqlite_nostd = { path="../sqlite-rs-embedded/sqlite_nostd" } 19 | 20 | [profile.dev] 21 | panic = "abort" 22 | 23 | [profile.release] 24 | panic = "abort" 25 | 26 | [features] 27 | test = [] 28 | loadable_extension = [ 29 | "sqlite_nostd/loadable_extension", 30 | "crsql_fractindex_core/loadable_extension", 31 | "crsql_core/loadable_extension" 32 | ] 33 | static = [ 34 | "sqlite_nostd/static", 35 | "crsql_fractindex_core/static", 36 | "crsql_core/static" 37 | ] 38 | omit_load_extension = [ 39 | "sqlite_nostd/omit_load_extension", 40 | "crsql_fractindex_core/omit_load_extension", 41 | "crsql_core/omit_load_extension" 42 | ] 43 | -------------------------------------------------------------------------------- /core/rs/bundle/README.md: -------------------------------------------------------------------------------- 1 | # bundle 2 | 3 | Using feature flags, bundles selected rust extensions into a single runtime loadable or statically linkable SQLite extension. 4 | 5 | - feature web : wasm support via web crate inclusion <-- just make this a target cfg 6 | - freature fract-index 7 | - feature auto-migrate 8 | -------------------------------------------------------------------------------- /core/rs/bundle/keep-stack-sizes.x: -------------------------------------------------------------------------------- 1 | /* file: keep-stack-sizes.x */ 2 | SECTIONS 3 | { 4 | /* `INFO` makes the section not allocatable so it won't be loaded into memory */ 5 | .stack_sizes (INFO) : 6 | { 7 | KEEP(*(.stack_sizes)); 8 | } 9 | } 10 | /* RUSTFLAGS="-Z emit-stack-sizes" cargo rustc --release -- -C link-arg=-Wl,-Tkeep-stack-sizes.x -C link-arg=-N */ 11 | /* https://doc.rust-lang.org/nightly/unstable-book/compiler-flags/emit-stack-sizes.html */ 12 | -------------------------------------------------------------------------------- /core/rs/bundle/rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "nightly-2023-10-05" 3 | components = [ "rust-src", "rustfmt", "clippy" ] 4 | -------------------------------------------------------------------------------- /core/rs/bundle/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | #![feature(core_intrinsics)] 3 | #![feature(lang_items)] 4 | 5 | extern crate alloc; 6 | 7 | use core::alloc::GlobalAlloc; 8 | use core::ffi::c_char; 9 | use core::panic::PanicInfo; 10 | use crsql_core; 11 | use crsql_core::sqlite3_crsqlcore_init; 12 | #[cfg(feature = "test")] 13 | pub use crsql_core::test_exports; 14 | use crsql_fractindex_core::sqlite3_crsqlfractionalindex_init; 15 | use sqlite_nostd as sqlite; 16 | use sqlite_nostd::SQLite3Allocator; 17 | 18 | // This must be our allocator so we can transfer ownership of memory to SQLite and have SQLite free that memory for us. 19 | // This drastically reduces copies when passing strings and blobs back and forth between Rust and C. 20 | #[global_allocator] 21 | static ALLOCATOR: SQLite3Allocator = SQLite3Allocator {}; 22 | 23 | // This must be our panic handler for WASM builds. For simplicity, we make it our panic handler for 24 | // all builds. Abort is also more portable than unwind, enabling us to go to more embedded use cases. 25 | #[panic_handler] 26 | fn panic(_info: &PanicInfo) -> ! { 27 | core::intrinsics::abort() 28 | } 29 | 30 | #[cfg(not(target_family = "wasm"))] 31 | #[lang = "eh_personality"] 32 | extern "C" fn eh_personality() {} 33 | 34 | #[cfg(target_family = "wasm")] 35 | #[no_mangle] 36 | pub fn __rust_alloc_error_handler(_: Layout) -> ! { 37 | core::intrinsics::abort() 38 | } 39 | 40 | #[no_mangle] 41 | pub extern "C" fn sqlite3_crsqlrustbundle_init( 42 | db: *mut sqlite::sqlite3, 43 | err_msg: *mut *mut c_char, 44 | api: *mut sqlite::api_routines, 45 | ) -> *mut ::core::ffi::c_void { 46 | sqlite::EXTENSION_INIT2(api); 47 | 48 | let rc = sqlite3_crsqlfractionalindex_init(db, err_msg, api); 49 | if rc != 0 { 50 | return core::ptr::null_mut(); 51 | } 52 | 53 | sqlite3_crsqlcore_init(db, err_msg, api) 54 | } 55 | -------------------------------------------------------------------------------- /core/rs/bundle_static/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "crsql_bundle_static" 3 | version = "0.1.0" 4 | edition = "2021" 5 | authors = ["Matt Wonlaw"] 6 | keywords = ["sqlite", "cr-sqlite"] 7 | license = "Apache 2" 8 | 9 | [lib] 10 | name = "crsql_bundle_static" 11 | crate-type = ["staticlib"] 12 | 13 | [dependencies] 14 | crsql_bundle = {path="../bundle"} 15 | 16 | [profile.dev] 17 | panic = "abort" 18 | 19 | [profile.release] 20 | panic = "abort" 21 | 22 | [features] 23 | libsql = ["crsql_bundle/libsql"] 24 | test = [ 25 | "crsql_bundle/test" 26 | ] 27 | loadable_extension = [ 28 | "crsql_bundle/loadable_extension" 29 | ] 30 | static = [ 31 | "crsql_bundle/static" 32 | ] 33 | omit_load_extension = [ 34 | "crsql_bundle/omit_load_extension" 35 | ] 36 | -------------------------------------------------------------------------------- /core/rs/bundle_static/rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "nightly-2023-10-05" 3 | 4 | -------------------------------------------------------------------------------- /core/rs/bundle_static/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | 3 | pub use crsql_bundle; 4 | -------------------------------------------------------------------------------- /core/rs/core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "crsql_core" 3 | version = "0.1.0" 4 | edition = "2021" 5 | authors = ["Matt Wonlaw"] 6 | keywords = ["sqlite", "cr-sqlite", "crdt"] 7 | license = "Apache 2" 8 | 9 | [lib] 10 | name = "crsql_core" 11 | crate-type = ["rlib"] 12 | 13 | [dependencies] 14 | sqlite_nostd = { path="../sqlite-rs-embedded/sqlite_nostd" } 15 | bytes = { version = "1.5", default-features = false } 16 | num-traits = { version = "0.2.17", default-features = false } 17 | num-derive = "0.4.1" 18 | 19 | [dev-dependencies] 20 | 21 | [profile.dev] 22 | panic = "abort" 23 | 24 | [profile.release] 25 | panic = "abort" 26 | 27 | [features] 28 | test = [] 29 | libsql = [] 30 | loadable_extension = ["sqlite_nostd/loadable_extension"] 31 | static = ["sqlite_nostd/static"] 32 | omit_load_extension = ["sqlite_nostd/omit_load_extension"] 33 | -------------------------------------------------------------------------------- /core/rs/core/README.md: -------------------------------------------------------------------------------- 1 | # core 2 | 3 | Eventually all the `c` code will be re-written and Rust and said functionality will be present here. 4 | 5 | Until then, `core` will slowly gobble different bits. -------------------------------------------------------------------------------- /core/rs/core/explore.sql: -------------------------------------------------------------------------------- 1 | -- backfill 2 | 3 | INSERT INTO x__crsql_clock VALUES (SELECT * FROM x); 4 | 5 | -- ^-- you need to unroll the columns -------------------------------------------------------------------------------- /core/rs/core/rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "nightly-2023-10-05" -------------------------------------------------------------------------------- /core/rs/core/src/alter.rs: -------------------------------------------------------------------------------- 1 | // Not yet fully migrated from `crsqlite.c` 2 | 3 | use alloc::boxed::Box; 4 | use alloc::format; 5 | use alloc::string::String; 6 | use alloc::vec::Vec; 7 | use core::ffi::{c_char, c_int, CStr}; 8 | use core::mem; 9 | #[cfg(not(feature = "std"))] 10 | use num_traits::FromPrimitive; 11 | use sqlite_nostd::{sqlite3, Connection, ResultCode, StrRef}; 12 | 13 | use crate::c::crsql_ExtData; 14 | use crate::db_version::fill_db_version_if_needed; 15 | use crate::tableinfo::{crsql_ensure_table_infos_are_up_to_date, TableInfo}; 16 | 17 | #[no_mangle] 18 | pub unsafe extern "C" fn crsql_compact_post_alter( 19 | db: *mut sqlite3, 20 | tbl_name: *const c_char, 21 | ext_data: *mut crsql_ExtData, 22 | errmsg: *mut *mut c_char, 23 | ) -> c_int { 24 | match compact_post_alter(db, tbl_name, ext_data, errmsg) { 25 | Ok(rc) | Err(rc) => rc as c_int, 26 | } 27 | } 28 | 29 | unsafe fn compact_post_alter( 30 | db: *mut sqlite3, 31 | tbl_name: *const c_char, 32 | ext_data: *mut crsql_ExtData, 33 | errmsg: *mut *mut c_char, 34 | ) -> Result { 35 | let tbl_name_str = CStr::from_ptr(tbl_name).to_str()?; 36 | fill_db_version_if_needed(db, ext_data).or_else(|msg| { 37 | errmsg.set(&msg); 38 | Err(ResultCode::ERROR) 39 | })?; 40 | let current_db_version = (*ext_data).dbVersion; 41 | 42 | // If primary key columns change (in the schema) 43 | // We need to drop, re-create and backfill 44 | // the clock table. 45 | // A change in pk columns means a change in all identities 46 | // of all rows. 47 | // We can determine this by comparing unique index on lookaside table vs 48 | // pks on source table 49 | let stmt = db.prepare_v2(&format!( 50 | "SELECT count(name) FROM ( 51 | SELECT name FROM pragma_table_info('{table_name}') 52 | WHERE pk > 0 AND name NOT IN 53 | (SELECT name FROM pragma_index_info('{table_name}__crsql_pks_pks')) 54 | UNION SELECT name FROM pragma_index_info('{table_name}__crsql_pks_pks') WHERE name NOT IN 55 | (SELECT name FROM pragma_table_info('{table_name}') WHERE pk > 0) AND name != 'col_name' 56 | );", 57 | table_name = crate::util::escape_ident_as_value(tbl_name_str), 58 | ))?; 59 | stmt.step()?; 60 | 61 | let pk_diff = stmt.column_int(0); 62 | // immediately drop stmt, otherwise clock table is considered locked. 63 | drop(stmt); 64 | 65 | if pk_diff > 0 { 66 | // drop the clock table so we can re-create it 67 | db.exec_safe(&format!( 68 | "DROP TABLE \"{table_name}__crsql_clock\"; 69 | DROP TABLE \"{table_name}__crsql_pks\";", 70 | table_name = crate::util::escape_ident(tbl_name_str), 71 | ))?; 72 | } else { 73 | // clock table is still relevant but needs compacting 74 | // in case columns were removed during the migration 75 | 76 | // First delete entries that no longer have a column 77 | let sql = format!( 78 | "DELETE FROM \"{tbl_name_ident}__crsql_clock\" WHERE \"col_name\" NOT IN ( 79 | SELECT name FROM pragma_table_info('{tbl_name_val}') UNION SELECT '{cl_sentinel}' 80 | )", 81 | tbl_name_ident = crate::util::escape_ident(tbl_name_str), 82 | tbl_name_val = crate::util::escape_ident_as_value(tbl_name_str), 83 | cl_sentinel = crate::c::DELETE_SENTINEL, 84 | ); 85 | db.exec_safe(&sql)?; 86 | 87 | // Next delete entries that no longer have a row but keeping tombstones 88 | // TODO: if we move the sentinel metadata to the lookaside this becomes much simpler 89 | let mut sql = String::from( 90 | format!( 91 | "DELETE FROM \"{tbl_name}__crsql_clock\" WHERE (col_name != '-1' OR (col_name = '-1' AND col_version % 2 != 0)) 92 | AND NOT EXISTS (SELECT 1 FROM \"{tbl_name}\" JOIN \"{tbl_name}__crsql_pks\" ON ", 93 | tbl_name = crate::util::escape_ident(tbl_name_str), 94 | ), 95 | ); 96 | let c_rc = crsql_ensure_table_infos_are_up_to_date(db, ext_data, errmsg); 97 | if c_rc != ResultCode::OK as c_int { 98 | if let Some(rc) = ResultCode::from_i32(c_rc) { 99 | return Err(rc); 100 | } 101 | return Err(ResultCode::ERROR); 102 | } 103 | let table_infos = 104 | mem::ManuallyDrop::new(Box::from_raw((*ext_data).tableInfos as *mut Vec)); 105 | let table_info = table_infos.iter().find(|x| x.tbl_name == tbl_name_str); 106 | if table_info.is_none() { 107 | return Err(ResultCode::ERROR); 108 | } 109 | // TODO: safe since we checked above but make more idiomatic 110 | let table_info = table_info.unwrap(); 111 | 112 | // for each pk col, append \"%w\".\"%w\" = \"%w__crsql_pks\".\"%w\" 113 | // to the where clause then close the statement. 114 | for (i, col) in table_info.pks.iter().enumerate() { 115 | if i > 0 { 116 | sql.push_str(" AND "); 117 | } 118 | 119 | sql.push_str(&format!( 120 | "\"{tbl_name}\".\"{col_name}\" = \"{tbl_name}__crsql_pks\".\"{col_name}\"", 121 | tbl_name = crate::util::escape_ident(tbl_name_str), 122 | col_name = &col.name, 123 | )); 124 | } 125 | sql.push_str( 126 | &format!( 127 | " WHERE \"{tbl_name}__crsql_clock\".key = \"{tbl_name}__crsql_pks\".__crsql_key LIMIT 1)", 128 | tbl_name = crate::util::escape_ident(tbl_name_str) 129 | ) 130 | ); 131 | db.exec_safe(&sql)?; 132 | 133 | // now delete pk lookasides that no longer map to anything in the clock tables 134 | let sql = format!( 135 | "DELETE FROM \"{tbl_name}__crsql_pks\" WHERE __crsql_key NOT IN ( 136 | SELECT key FROM \"{tbl_name}__crsql_clock\" 137 | )", 138 | tbl_name = crate::util::escape_ident(tbl_name_str), 139 | ); 140 | db.exec_safe(&sql)?; 141 | } 142 | 143 | let stmt = db.prepare_v2( 144 | "INSERT OR REPLACE INTO crsql_master (key, value) VALUES ('pre_compact_dbversion', ?)", 145 | )?; 146 | stmt.bind_int64(1, current_db_version)?; 147 | stmt.step()?; 148 | Ok(ResultCode::OK) 149 | } 150 | -------------------------------------------------------------------------------- /core/rs/core/src/changes_vtab_read.rs: -------------------------------------------------------------------------------- 1 | extern crate alloc; 2 | use crate::tableinfo::TableInfo; 3 | use alloc::format; 4 | use alloc::string::String; 5 | use alloc::vec; 6 | use alloc::vec::Vec; 7 | use sqlite::ResultCode; 8 | 9 | use sqlite_nostd as sqlite; 10 | 11 | fn crsql_changes_query_for_table(table_info: &TableInfo) -> Result { 12 | if table_info.pks.len() == 0 { 13 | // no primary keys? We can't get changes for a table w/o primary keys... 14 | // this should be an impossible case. 15 | return Err(ResultCode::ABORT); 16 | } 17 | 18 | let pk_list = crate::util::as_identifier_list(&table_info.pks, Some("pk_tbl."))?; 19 | // TODO: we can remove the self join if we put causal length in the primary key table 20 | 21 | // We LEFT JOIN and COALESCE the causal length 22 | // since we incorporated an optimization to not store causal length records 23 | // until they're required. I.e., do not store them until a delete 24 | // is actually issued. This cuts data weight quite a bit for 25 | // rows that never get removed. 26 | Ok(format!( 27 | "SELECT 28 | '{table_name_val}' as tbl, 29 | crsql_pack_columns({pk_list}) as pks, 30 | t1.col_name as cid, 31 | t1.col_version as col_vrsn, 32 | t1.db_version as db_vrsn, 33 | site_tbl.site_id as site_id, 34 | t1.key, 35 | t1.seq as seq, 36 | COALESCE(t2.col_version, 1) as cl 37 | FROM \"{table_name_ident}__crsql_clock\" AS t1 38 | JOIN \"{table_name_ident}__crsql_pks\" AS pk_tbl ON t1.key = pk_tbl.__crsql_key 39 | LEFT JOIN crsql_site_id AS site_tbl ON t1.site_id = site_tbl.ordinal 40 | LEFT JOIN \"{table_name_ident}__crsql_clock\" AS t2 ON 41 | t1.key = t2.key AND t2.col_name = '{sentinel}'", 42 | table_name_val = crate::util::escape_ident_as_value(&table_info.tbl_name), 43 | pk_list = pk_list, 44 | table_name_ident = crate::util::escape_ident(&table_info.tbl_name), 45 | sentinel = crate::c::INSERT_SENTINEL 46 | )) 47 | } 48 | 49 | pub fn changes_union_query( 50 | table_infos: &Vec, 51 | idx_str: &str, 52 | ) -> Result { 53 | let mut sub_queries = vec![]; 54 | 55 | for table_info in table_infos { 56 | let query_part = crsql_changes_query_for_table(&table_info)?; 57 | sub_queries.push(query_part); 58 | } 59 | 60 | // Manually null-terminate the string so we don't have to copy it to create a CString. 61 | // We can just extract the raw bytes of the Rust string. 62 | return Ok(format!( 63 | "SELECT tbl, pks, cid, col_vrsn, db_vrsn, site_id, key, seq, cl FROM ({unions}) {idx_str}\0", 64 | unions = sub_queries.join(" UNION ALL "), 65 | idx_str = idx_str, 66 | )); 67 | } 68 | -------------------------------------------------------------------------------- /core/rs/core/src/compare_values.rs: -------------------------------------------------------------------------------- 1 | use alloc::format; 2 | use alloc::string::String; 3 | use core::ffi::c_int; 4 | use sqlite::value; 5 | use sqlite::Value; 6 | use sqlite_nostd as sqlite; 7 | 8 | // TODO: add an integration test that ensures NULL == NULL! 9 | pub fn crsql_compare_sqlite_values(l: *mut sqlite::value, r: *mut sqlite::value) -> c_int { 10 | let l_type = l.value_type(); 11 | let r_type = r.value_type(); 12 | 13 | if l_type != r_type { 14 | // We swap the compare since we want null to be _less than_ all things 15 | // and null is assigned to ordinal 5 (greatest thing). 16 | return (r_type as i32) - (l_type as i32); 17 | } 18 | 19 | match l_type { 20 | sqlite::ColumnType::Blob => l.blob().cmp(r.blob()) as c_int, 21 | sqlite::ColumnType::Float => { 22 | let l_double = l.double(); 23 | let r_double = r.double(); 24 | if l_double < r_double { 25 | return -1; 26 | } else if l_double > r_double { 27 | return 1; 28 | } 29 | return 0; 30 | } 31 | sqlite::ColumnType::Integer => { 32 | let l_int = l.int64(); 33 | let r_int = r.int64(); 34 | // no subtraction since that could overflow the c_int return type 35 | if l_int < r_int { 36 | return -1; 37 | } else if l_int > r_int { 38 | return 1; 39 | } 40 | return 0; 41 | } 42 | sqlite::ColumnType::Null => 0, 43 | sqlite::ColumnType::Text => l.text().cmp(r.text()) as c_int, 44 | } 45 | } 46 | 47 | pub fn any_value_changed(left: &[*mut value], right: &[*mut value]) -> Result { 48 | if left.len() != right.len() { 49 | return Err(format!( 50 | "left and right values must have the same length: {} != {}", 51 | left.len(), 52 | right.len() 53 | )); 54 | } 55 | 56 | for (l, r) in left.iter().zip(right.iter()) { 57 | if crsql_compare_sqlite_values(*l, *r) != 0 { 58 | return Ok(true); 59 | } 60 | } 61 | 62 | Ok(false) 63 | } 64 | -------------------------------------------------------------------------------- /core/rs/core/src/config.rs: -------------------------------------------------------------------------------- 1 | use alloc::format; 2 | 3 | use sqlite::{Connection, Context}; 4 | use sqlite_nostd as sqlite; 5 | use sqlite_nostd::{ResultCode, Value}; 6 | 7 | use crate::c::crsql_ExtData; 8 | 9 | pub const MERGE_EQUAL_VALUES: &str = "merge-equal-values"; 10 | 11 | pub extern "C" fn crsql_config_set( 12 | ctx: *mut sqlite::context, 13 | argc: i32, 14 | argv: *mut *mut sqlite::value, 15 | ) { 16 | let args = sqlite::args!(argc, argv); 17 | 18 | let name = args[0].text(); 19 | 20 | let value = match name { 21 | MERGE_EQUAL_VALUES => { 22 | let value = args[1]; 23 | let ext_data = ctx.user_data() as *mut crsql_ExtData; 24 | unsafe { (*ext_data).mergeEqualValues = value.int() }; 25 | value 26 | } 27 | _ => { 28 | ctx.result_error("Unknown setting name"); 29 | ctx.result_error_code(ResultCode::ERROR); 30 | return; 31 | } 32 | }; 33 | 34 | let db = ctx.db_handle(); 35 | match insert_config_setting(db, name, value) { 36 | Ok(value) => { 37 | ctx.result_value(value); 38 | } 39 | Err(rc) => { 40 | ctx.result_error("Could not persist config in database"); 41 | ctx.result_error_code(rc); 42 | return; 43 | } 44 | } 45 | } 46 | 47 | fn insert_config_setting( 48 | db: *mut sqlite_nostd::sqlite3, 49 | name: &str, 50 | value: *mut sqlite::value, 51 | ) -> Result<*mut sqlite::value, ResultCode> { 52 | let stmt = 53 | db.prepare_v2("INSERT OR REPLACE INTO crsql_master VALUES (?, ?) RETURNING value")?; 54 | 55 | stmt.bind_text(1, &format!("config.{name}"), sqlite::Destructor::TRANSIENT)?; 56 | stmt.bind_value(2, value)?; 57 | 58 | if let ResultCode::ROW = stmt.step()? { 59 | stmt.column_value(0) 60 | } else { 61 | Err(ResultCode::ERROR) 62 | } 63 | } 64 | 65 | pub extern "C" fn crsql_config_get( 66 | ctx: *mut sqlite::context, 67 | argc: i32, 68 | argv: *mut *mut sqlite::value, 69 | ) { 70 | let args = sqlite::args!(argc, argv); 71 | 72 | let name = args[0].text(); 73 | 74 | match name { 75 | MERGE_EQUAL_VALUES => { 76 | let ext_data = ctx.user_data() as *mut crsql_ExtData; 77 | ctx.result_int(unsafe { (*ext_data).mergeEqualValues }); 78 | } 79 | _ => { 80 | ctx.result_error("Unknown setting name"); 81 | ctx.result_error_code(ResultCode::ERROR); 82 | return; 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /core/rs/core/src/consts.rs: -------------------------------------------------------------------------------- 1 | pub const TBL_SITE_ID: &'static str = "crsql_site_id"; 2 | pub const TBL_SCHEMA: &'static str = "crsql_master"; 3 | // pub const CRSQLITE_VERSION_0_15_0: i32 = 15_00_00; 4 | // pub const CRSQLITE_VERSION_0_13_0: i32 = 13_00_00; 5 | // MM_mm_pp_xx 6 | // so a 1.0.0 release is: 7 | // 01_00_00_00 -> 1000000 8 | // a 0.5 release is: 9 | // 00_05_00_00 -> 50000 10 | // a 0.5.1 is: 11 | // 00_05_01_00 12 | // and, if we ever need it, we can track individual builds of a patch release 13 | // 00_05_01_01 14 | pub const CRSQLITE_VERSION: i32 = 16_03_00; 15 | pub const CRSQLITE_VERSION_STR: &'static str = "0.16.3"; 16 | pub const CRSQLITE_VERSION_0_15_0: i32 = 15_00_00; 17 | 18 | pub const SITE_ID_LEN: i32 = 16; 19 | pub const ROWID_SLAB_SIZE: i64 = 10000000000000; 20 | // db version is a signed 64bit int since sqlite doesn't support saving and 21 | // retrieving unsigned 64bit ints. (2^64 / 2) is a big enough number to write 1 22 | // million entries per second for 3,000 centuries. 23 | pub const MIN_POSSIBLE_DB_VERSION: i64 = 0; 24 | pub const MAX_TBL_NAME_LEN: i32 = 2048; 25 | -------------------------------------------------------------------------------- /core/rs/core/src/create_crr.rs: -------------------------------------------------------------------------------- 1 | use core::ffi::c_char; 2 | use sqlite_nostd as sqlite; 3 | use sqlite_nostd::ResultCode; 4 | 5 | use crate::bootstrap::create_clock_table; 6 | use crate::tableinfo::{is_table_compatible, pull_table_info}; 7 | use crate::triggers::create_triggers; 8 | use crate::{backfill_table, is_crr, remove_crr_triggers_if_exist}; 9 | 10 | /** 11 | * Create a new crr -- 12 | * all triggers, views, tables 13 | */ 14 | pub fn create_crr( 15 | db: *mut sqlite::sqlite3, 16 | _schema: &str, 17 | table: &str, 18 | is_commit_alter: bool, 19 | no_tx: bool, 20 | err: *mut *mut c_char, 21 | ) -> Result { 22 | if !is_table_compatible(db, table, err)? { 23 | return Err(ResultCode::ERROR); 24 | } 25 | if is_crr(db, table)? { 26 | return Ok(ResultCode::OK); 27 | } 28 | 29 | // We do not / can not pull this from the cached set of table infos 30 | // since nothing would exist in it for a table not yet made into a crr. 31 | // TODO: Note: we can optimize out our `ensureTableInfosAreUpToDate` by mutating our ext data 32 | // when upgrading stuff to CRRs 33 | let table_info = pull_table_info(db, table, err)?; 34 | 35 | create_clock_table(db, &table_info, err)?; 36 | remove_crr_triggers_if_exist(db, table)?; 37 | create_triggers(db, &table_info, err)?; 38 | 39 | backfill_table( 40 | db, 41 | table, 42 | &table_info.pks, 43 | &table_info.non_pks, 44 | is_commit_alter, 45 | no_tx, 46 | )?; 47 | 48 | Ok(ResultCode::OK) 49 | } 50 | -------------------------------------------------------------------------------- /core/rs/core/src/db_version.rs: -------------------------------------------------------------------------------- 1 | use core::ptr; 2 | 3 | use crate::alloc::string::ToString; 4 | use alloc::format; 5 | use alloc::string::String; 6 | use core::ffi::{c_char, c_int}; 7 | use sqlite::ResultCode; 8 | use sqlite::StrRef; 9 | use sqlite::{sqlite3, Stmt}; 10 | use sqlite_nostd as sqlite; 11 | 12 | use crate::c::crsql_ExtData; 13 | use crate::c::crsql_fetchPragmaDataVersion; 14 | use crate::c::crsql_fetchPragmaSchemaVersion; 15 | use crate::c::DB_VERSION_SCHEMA_VERSION; 16 | use crate::consts::MIN_POSSIBLE_DB_VERSION; 17 | use crate::ext_data::recreate_db_version_stmt; 18 | 19 | #[no_mangle] 20 | pub extern "C" fn crsql_fill_db_version_if_needed( 21 | db: *mut sqlite3, 22 | ext_data: *mut crsql_ExtData, 23 | errmsg: *mut *mut c_char, 24 | ) -> c_int { 25 | match fill_db_version_if_needed(db, ext_data) { 26 | Ok(rc) => rc as c_int, 27 | Err(msg) => { 28 | errmsg.set(&msg); 29 | ResultCode::ERROR as c_int 30 | } 31 | } 32 | } 33 | 34 | #[no_mangle] 35 | pub extern "C" fn crsql_next_db_version( 36 | db: *mut sqlite3, 37 | ext_data: *mut crsql_ExtData, 38 | merging_version: sqlite::int64, 39 | errmsg: *mut *mut c_char, 40 | ) -> sqlite::int64 { 41 | match next_db_version(db, ext_data, Some(merging_version)) { 42 | Ok(version) => version, 43 | Err(msg) => { 44 | errmsg.set(&msg); 45 | -1 46 | } 47 | } 48 | } 49 | 50 | /** 51 | * Given this needs to do a pragma check, invoke it as little as possible. 52 | * TODO: We could optimize to only do a pragma check once per transaction. 53 | * Need to save some bit that states we checked the pragma already and reset on tx commit or rollback. 54 | */ 55 | pub fn next_db_version( 56 | db: *mut sqlite3, 57 | ext_data: *mut crsql_ExtData, 58 | merging_version: Option, 59 | ) -> Result { 60 | fill_db_version_if_needed(db, ext_data)?; 61 | 62 | let mut ret = unsafe { (*ext_data).dbVersion + 1 }; 63 | if ret < unsafe { (*ext_data).pendingDbVersion } { 64 | ret = unsafe { (*ext_data).pendingDbVersion }; 65 | } 66 | if let Some(merging_version) = merging_version { 67 | if ret < merging_version { 68 | ret = merging_version; 69 | } 70 | } 71 | unsafe { 72 | (*ext_data).pendingDbVersion = ret; 73 | } 74 | Ok(ret) 75 | } 76 | 77 | pub fn fill_db_version_if_needed( 78 | db: *mut sqlite3, 79 | ext_data: *mut crsql_ExtData, 80 | ) -> Result { 81 | unsafe { 82 | let rc = crsql_fetchPragmaDataVersion(db, ext_data); 83 | if rc == -1 { 84 | return Err("failed to fetch PRAGMA data_version".to_string()); 85 | } 86 | if (*ext_data).dbVersion != -1 && rc == 0 { 87 | return Ok(ResultCode::OK); 88 | } 89 | fetch_db_version_from_storage(db, ext_data) 90 | } 91 | } 92 | 93 | pub fn fetch_db_version_from_storage( 94 | db: *mut sqlite3, 95 | ext_data: *mut crsql_ExtData, 96 | ) -> Result { 97 | unsafe { 98 | let schema_changed = if (*ext_data).pDbVersionStmt == ptr::null_mut() { 99 | 1 as c_int 100 | } else { 101 | crsql_fetchPragmaSchemaVersion(db, ext_data, DB_VERSION_SCHEMA_VERSION) 102 | }; 103 | 104 | if schema_changed < 0 { 105 | return Err("failed to fetch the pragma schema version".to_string()); 106 | } 107 | 108 | if schema_changed > 0 { 109 | match recreate_db_version_stmt(db, ext_data) { 110 | Ok(ResultCode::DONE) => { 111 | // this means there are no clock tables / this is a clean db 112 | (*ext_data).dbVersion = 0; 113 | return Ok(ResultCode::OK); 114 | } 115 | Ok(_) => {} 116 | Err(rc) => return Err(format!("failed to recreate db version stmt: {}", rc)), 117 | } 118 | } 119 | 120 | let db_version_stmt = (*ext_data).pDbVersionStmt; 121 | let rc = db_version_stmt.step(); 122 | match rc { 123 | // no rows? We're a fresh db with the min starting version 124 | Ok(ResultCode::DONE) => { 125 | db_version_stmt.reset().or_else(|rc| { 126 | Err(format!( 127 | "failed to reset db version stmt after DONE: {}", 128 | rc 129 | )) 130 | })?; 131 | (*ext_data).dbVersion = MIN_POSSIBLE_DB_VERSION; 132 | Ok(ResultCode::OK) 133 | } 134 | // got a row? It is our db version. 135 | Ok(ResultCode::ROW) => { 136 | (*ext_data).dbVersion = db_version_stmt.column_int64(0); 137 | db_version_stmt 138 | .reset() 139 | .or_else(|rc| Err(format!("failed to reset db version stmt after ROW: {}", rc))) 140 | } 141 | // Not row or done? Something went wrong. 142 | Ok(rc) | Err(rc) => { 143 | db_version_stmt.reset().or_else(|rc| { 144 | Err(format!("failed to reset db version stmt after ROW: {}", rc)) 145 | })?; 146 | Err(format!("failed to step db version stmt: {}", rc)) 147 | } 148 | } 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /core/rs/core/src/ext_data.rs: -------------------------------------------------------------------------------- 1 | extern crate alloc; 2 | use crate::alloc::string::ToString; 3 | use alloc::vec; 4 | use core::ffi::c_int; 5 | use core::ptr::null_mut; 6 | 7 | use sqlite::{sqlite3, Connection, ResultCode, Stmt}; 8 | use sqlite_nostd as sqlite; 9 | 10 | use crate::{c::crsql_ExtData, util::get_db_version_union_query}; 11 | 12 | #[no_mangle] 13 | pub extern "C" fn crsql_recreate_db_version_stmt( 14 | db: *mut sqlite3, 15 | ext_data: *mut crsql_ExtData, 16 | ) -> c_int { 17 | match recreate_db_version_stmt(db, ext_data) { 18 | Ok(ResultCode::DONE) => -1, // negative 1 means no clock tables exist and there is nothing to fetch 19 | Ok(rc) | Err(rc) => rc as c_int, 20 | } 21 | } 22 | 23 | pub fn recreate_db_version_stmt( 24 | db: *mut sqlite3, 25 | ext_data: *mut crsql_ExtData, 26 | ) -> Result { 27 | let clock_tables_stmt = unsafe { (*ext_data).pSelectClockTablesStmt }; 28 | let db_version_stmt = unsafe { (*ext_data).pDbVersionStmt }; 29 | 30 | db_version_stmt.finalize()?; 31 | unsafe { 32 | (*ext_data).pDbVersionStmt = null_mut(); 33 | } 34 | 35 | let mut clock_tbl_names = vec![]; 36 | loop { 37 | match clock_tables_stmt.step() { 38 | Ok(ResultCode::DONE) => { 39 | clock_tables_stmt.reset()?; 40 | if clock_tbl_names.len() == 0 { 41 | return Ok(ResultCode::DONE); 42 | } 43 | break; 44 | } 45 | Ok(ResultCode::ROW) => { 46 | clock_tbl_names.push(clock_tables_stmt.column_text(0).to_string()); 47 | } 48 | Ok(rc) | Err(rc) => { 49 | clock_tables_stmt.reset()?; 50 | return Err(rc); 51 | } 52 | } 53 | } 54 | 55 | let union = get_db_version_union_query(&clock_tbl_names); 56 | 57 | let db_version_stmt = db.prepare_v3(&union, sqlite::PREPARE_PERSISTENT)?; 58 | unsafe { 59 | (*ext_data).pDbVersionStmt = db_version_stmt.into_raw(); 60 | } 61 | 62 | Ok(ResultCode::OK) 63 | } 64 | -------------------------------------------------------------------------------- /core/rs/core/src/is_crr.rs: -------------------------------------------------------------------------------- 1 | use alloc::format; 2 | use sqlite::Connection; 3 | use sqlite_nostd as sqlite; 4 | use sqlite_nostd::ResultCode; 5 | 6 | /** 7 | * Given a table name, returns whether or not it has already 8 | * been upgraded to a CRR. 9 | */ 10 | pub fn is_crr(db: *mut sqlite::sqlite3, table: &str) -> Result { 11 | let stmt = 12 | db.prepare_v2("SELECT count(*) FROM sqlite_master WHERE type = 'trigger' AND name = ?")?; 13 | stmt.bind_text( 14 | 1, 15 | &format!("{}__crsql_itrig", table), 16 | sqlite::Destructor::TRANSIENT, 17 | )?; 18 | stmt.step()?; 19 | let count = stmt.column_int(0); 20 | 21 | if count == 0 { 22 | Ok(false) 23 | } else { 24 | Ok(true) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /core/rs/core/src/local_writes/after_delete.rs: -------------------------------------------------------------------------------- 1 | use alloc::string::String; 2 | use core::ffi::c_int; 3 | use sqlite::sqlite3; 4 | use sqlite::value; 5 | use sqlite::Context; 6 | use sqlite::ResultCode; 7 | use sqlite_nostd as sqlite; 8 | 9 | use crate::{c::crsql_ExtData, tableinfo::TableInfo}; 10 | 11 | use super::bump_seq; 12 | use super::trigger_fn_preamble; 13 | 14 | /** 15 | * crsql_after_delete("table", old_pk_values...) 16 | */ 17 | pub unsafe extern "C" fn x_crsql_after_delete( 18 | ctx: *mut sqlite::context, 19 | argc: c_int, 20 | argv: *mut *mut sqlite::value, 21 | ) { 22 | let result = trigger_fn_preamble(ctx, argc, argv, |table_info, values, ext_data| { 23 | after_delete(ctx.db_handle(), ext_data, table_info, &values[1..]) 24 | }); 25 | 26 | match result { 27 | Ok(_) => { 28 | ctx.result_int64(0); 29 | } 30 | Err(msg) => { 31 | ctx.result_error(&msg); 32 | } 33 | } 34 | } 35 | 36 | fn after_delete( 37 | db: *mut sqlite3, 38 | ext_data: *mut crsql_ExtData, 39 | tbl_info: &TableInfo, 40 | pks_old: &[*mut value], 41 | ) -> Result { 42 | let db_version = crate::db_version::next_db_version(db, ext_data, None)?; 43 | let seq = bump_seq(ext_data); 44 | let key = tbl_info 45 | .get_or_create_key_via_raw_values(db, pks_old) 46 | .or_else(|_| Err("failed geteting or creating lookaside key"))?; 47 | 48 | let mark_locally_deleted_stmt_ref = tbl_info 49 | .get_mark_locally_deleted_stmt(db) 50 | .or_else(|_e| Err("failed to get mark_locally_deleted_stmt"))?; 51 | let mark_locally_deleted_stmt = mark_locally_deleted_stmt_ref 52 | .as_ref() 53 | .ok_or("Failed to deref sentinel stmt")?; 54 | mark_locally_deleted_stmt 55 | .bind_int64(1, key) 56 | .and_then(|_| mark_locally_deleted_stmt.bind_int64(2, db_version)) 57 | .and_then(|_| mark_locally_deleted_stmt.bind_int(3, seq)) 58 | .and_then(|_| mark_locally_deleted_stmt.bind_int64(4, db_version)) 59 | .and_then(|_| mark_locally_deleted_stmt.bind_int(5, seq)) 60 | .or_else(|_| Err("failed binding to mark locally deleted stmt"))?; 61 | super::step_trigger_stmt(mark_locally_deleted_stmt)?; 62 | 63 | // now actually delete the row metadata 64 | let drop_clocks_stmt_ref = tbl_info 65 | .get_merge_delete_drop_clocks_stmt(db) 66 | .or_else(|_e| Err("failed to get mark_locally_deleted_stmt"))?; 67 | let drop_clocks_stmt = drop_clocks_stmt_ref 68 | .as_ref() 69 | .ok_or("Failed to deref sentinel stmt")?; 70 | 71 | drop_clocks_stmt 72 | .bind_int64(1, key) 73 | .or_else(|_e| Err("failed to bind pks to drop_clocks_stmt"))?; 74 | super::step_trigger_stmt(drop_clocks_stmt) 75 | } 76 | -------------------------------------------------------------------------------- /core/rs/core/src/local_writes/after_insert.rs: -------------------------------------------------------------------------------- 1 | use alloc::string::String; 2 | use core::ffi::c_int; 3 | use sqlite::sqlite3; 4 | use sqlite::value; 5 | use sqlite::Context; 6 | use sqlite::ResultCode; 7 | use sqlite_nostd as sqlite; 8 | 9 | use crate::{c::crsql_ExtData, tableinfo::TableInfo}; 10 | 11 | use super::bump_seq; 12 | use super::trigger_fn_preamble; 13 | 14 | /** 15 | * crsql_after_insert("table", pk_values...) 16 | */ 17 | pub unsafe extern "C" fn x_crsql_after_insert( 18 | ctx: *mut sqlite::context, 19 | argc: c_int, 20 | argv: *mut *mut sqlite::value, 21 | ) { 22 | let result = trigger_fn_preamble(ctx, argc, argv, |table_info, values, ext_data| { 23 | after_insert(ctx.db_handle(), ext_data, table_info, &values[1..]) 24 | }); 25 | 26 | match result { 27 | Ok(_) => { 28 | ctx.result_int64(0); 29 | } 30 | Err(msg) => { 31 | ctx.result_error(&msg); 32 | } 33 | } 34 | } 35 | 36 | fn after_insert( 37 | db: *mut sqlite3, 38 | ext_data: *mut crsql_ExtData, 39 | tbl_info: &TableInfo, 40 | pks_new: &[*mut value], 41 | ) -> Result { 42 | let db_version = crate::db_version::next_db_version(db, ext_data, None)?; 43 | let (create_record_existed, key_new) = tbl_info 44 | .get_or_create_key_for_insert(db, pks_new) 45 | .or_else(|_| Err("failed geteting or creating lookaside key"))?; 46 | if tbl_info.non_pks.len() == 0 { 47 | let seq = bump_seq(ext_data); 48 | // just a sentinel record 49 | return super::mark_new_pk_row_created(db, tbl_info, key_new, db_version, seq); 50 | } else if create_record_existed { 51 | // update the create record since it already exists. 52 | let seq = bump_seq(ext_data); 53 | update_create_record(db, tbl_info, key_new, db_version, seq)?; 54 | } 55 | 56 | // now for each non-pk column, create or update the column record 57 | for col in tbl_info.non_pks.iter() { 58 | let seq = bump_seq(ext_data); 59 | super::mark_locally_updated(db, tbl_info, key_new, col, db_version, seq)?; 60 | } 61 | Ok(ResultCode::OK) 62 | } 63 | 64 | fn update_create_record( 65 | db: *mut sqlite3, 66 | tbl_info: &TableInfo, 67 | new_key: sqlite::int64, 68 | db_version: sqlite::int64, 69 | seq: i32, 70 | ) -> Result { 71 | let update_create_record_stmt_ref = tbl_info 72 | .get_maybe_mark_locally_reinserted_stmt(db) 73 | .or_else(|_e| Err("failed to get update_create_record_stmt"))?; 74 | let update_create_record_stmt = update_create_record_stmt_ref 75 | .as_ref() 76 | .ok_or("Failed to deref update_create_record_stmt")?; 77 | 78 | update_create_record_stmt 79 | .bind_int64(1, db_version) 80 | .and_then(|_| update_create_record_stmt.bind_int(2, seq)) 81 | .and_then(|_| update_create_record_stmt.bind_int64(3, new_key)) 82 | .and_then(|_| { 83 | update_create_record_stmt.bind_text( 84 | 4, 85 | crate::c::INSERT_SENTINEL, 86 | sqlite::Destructor::STATIC, 87 | ) 88 | }) 89 | .or_else(|_e| Err("failed binding to update_create_record_stmt"))?; 90 | 91 | super::step_trigger_stmt(update_create_record_stmt) 92 | } 93 | -------------------------------------------------------------------------------- /core/rs/core/src/local_writes/mod.rs: -------------------------------------------------------------------------------- 1 | use core::ffi::{c_char, c_int}; 2 | use core::mem::ManuallyDrop; 3 | 4 | use crate::alloc::string::ToString; 5 | use crate::c::crsql_ExtData; 6 | use crate::stmt_cache::reset_cached_stmt; 7 | use alloc::boxed::Box; 8 | use alloc::format; 9 | use alloc::string::String; 10 | use alloc::vec::Vec; 11 | use sqlite::sqlite3; 12 | use sqlite::{Context, ManagedStmt, Value}; 13 | use sqlite_nostd as sqlite; 14 | use sqlite_nostd::ResultCode; 15 | 16 | use crate::tableinfo::{crsql_ensure_table_infos_are_up_to_date, ColumnInfo, TableInfo}; 17 | 18 | pub mod after_delete; 19 | pub mod after_insert; 20 | pub mod after_update; 21 | 22 | fn trigger_fn_preamble( 23 | ctx: *mut sqlite::context, 24 | argc: c_int, 25 | argv: *mut *mut sqlite::value, 26 | f: F, 27 | ) -> Result 28 | where 29 | F: Fn(&TableInfo, &[*mut sqlite::value], *mut crsql_ExtData) -> Result, 30 | { 31 | if argc < 1 { 32 | return Err("expected at least 1 argument".to_string()); 33 | } 34 | 35 | let values = sqlite::args!(argc, argv); 36 | let ext_data = sqlite::user_data(ctx) as *mut crsql_ExtData; 37 | let mut inner_err: *mut c_char = core::ptr::null_mut(); 38 | let outer_err: *mut *mut c_char = &mut inner_err; 39 | 40 | let rc = crsql_ensure_table_infos_are_up_to_date(ctx.db_handle(), ext_data, outer_err); 41 | if rc != ResultCode::OK as c_int { 42 | return Err(format!( 43 | "failed to ensure table infos are up to date: {}", 44 | rc 45 | )); 46 | } 47 | 48 | let table_infos = 49 | unsafe { ManuallyDrop::new(Box::from_raw((*ext_data).tableInfos as *mut Vec)) }; 50 | let table_name = values[0].text(); 51 | let table_info = match table_infos.iter().find(|t| &(t.tbl_name) == table_name) { 52 | Some(t) => t, 53 | None => { 54 | return Err(format!("table {} not found", table_name)); 55 | } 56 | }; 57 | 58 | f(table_info, &values, ext_data) 59 | } 60 | 61 | fn step_trigger_stmt(stmt: &ManagedStmt) -> Result { 62 | match stmt.step() { 63 | Ok(ResultCode::DONE) => { 64 | reset_cached_stmt(stmt.stmt) 65 | .or_else(|_e| Err("done -- unable to reset cached trigger stmt"))?; 66 | Ok(ResultCode::OK) 67 | } 68 | Ok(code) | Err(code) => { 69 | reset_cached_stmt(stmt.stmt) 70 | .or_else(|_e| Err("error -- unable to reset cached trigger stmt"))?; 71 | Err(format!( 72 | "unexpected result code from tigger_stmt.step: {}", 73 | code 74 | )) 75 | } 76 | } 77 | } 78 | 79 | fn mark_new_pk_row_created( 80 | db: *mut sqlite3, 81 | tbl_info: &TableInfo, 82 | key_new: sqlite::int64, 83 | db_version: i64, 84 | seq: i32, 85 | ) -> Result { 86 | let mark_locally_created_stmt_ref = tbl_info 87 | .get_mark_locally_created_stmt(db) 88 | .or_else(|_e| Err("failed to get mark_locally_created_stmt"))?; 89 | let mark_locally_created_stmt = mark_locally_created_stmt_ref 90 | .as_ref() 91 | .ok_or("Failed to deref sentinel stmt")?; 92 | 93 | mark_locally_created_stmt 94 | .bind_int64(1, key_new) 95 | .and_then(|_| mark_locally_created_stmt.bind_int64(2, db_version)) 96 | .and_then(|_| mark_locally_created_stmt.bind_int(3, seq)) 97 | .and_then(|_| mark_locally_created_stmt.bind_int64(4, db_version)) 98 | .and_then(|_| mark_locally_created_stmt.bind_int(5, seq)) 99 | .or_else(|_| Err("failed binding to mark_locally_created_stmt"))?; 100 | step_trigger_stmt(mark_locally_created_stmt) 101 | } 102 | 103 | fn bump_seq(ext_data: *mut crsql_ExtData) -> c_int { 104 | unsafe { 105 | (*ext_data).seq += 1; 106 | (*ext_data).seq - 1 107 | } 108 | } 109 | 110 | #[allow(non_snake_case)] 111 | fn mark_locally_updated( 112 | db: *mut sqlite3, 113 | tbl_info: &TableInfo, 114 | new_key: sqlite::int64, 115 | col_info: &ColumnInfo, 116 | db_version: sqlite::int64, 117 | seq: i32, 118 | ) -> Result { 119 | let mark_locally_updated_stmt_ref = tbl_info 120 | .get_mark_locally_updated_stmt(db) 121 | .or_else(|_e| Err("failed to get mark_locally_updated_stmt"))?; 122 | let mark_locally_updated_stmt = mark_locally_updated_stmt_ref 123 | .as_ref() 124 | .ok_or("Failed to deref sentinel stmt")?; 125 | 126 | mark_locally_updated_stmt 127 | .bind_int64(1, new_key) 128 | .and_then(|_| { 129 | mark_locally_updated_stmt.bind_text(2, &col_info.name, sqlite::Destructor::STATIC) 130 | }) 131 | .and_then(|_| mark_locally_updated_stmt.bind_int64(3, db_version)) 132 | .and_then(|_| mark_locally_updated_stmt.bind_int(4, seq)) 133 | .and_then(|_| mark_locally_updated_stmt.bind_int64(5, db_version)) 134 | .and_then(|_| mark_locally_updated_stmt.bind_int(6, seq)) 135 | .or_else(|_| Err("failed binding to mark_locally_updated_stmt"))?; 136 | step_trigger_stmt(mark_locally_updated_stmt) 137 | } 138 | -------------------------------------------------------------------------------- /core/rs/core/src/sha.rs: -------------------------------------------------------------------------------- 1 | // The sha of the commit that this version of crsqlite was built from. 2 | pub const SHA: &'static str = core::env!("CRSQLITE_COMMIT_SHA"); 3 | -------------------------------------------------------------------------------- /core/rs/core/src/stmt_cache.rs: -------------------------------------------------------------------------------- 1 | extern crate alloc; 2 | use alloc::vec::Vec; 3 | use core::mem::ManuallyDrop; 4 | 5 | use alloc::boxed::Box; 6 | use sqlite::Stmt; 7 | use sqlite_nostd as sqlite; 8 | use sqlite_nostd::ResultCode; 9 | 10 | use crate::c::crsql_ExtData; 11 | use crate::tableinfo::TableInfo; 12 | 13 | // Finalize prepared statements attached to table infos. 14 | // Do not drop the table infos. 15 | // We do this explicitly since `drop` cannot return an error and we want to 16 | // return the error / not panic. 17 | #[no_mangle] 18 | pub extern "C" fn crsql_clear_stmt_cache(ext_data: *mut crsql_ExtData) { 19 | let tbl_infos = 20 | unsafe { ManuallyDrop::new(Box::from_raw((*ext_data).tableInfos as *mut Vec)) }; 21 | for tbl_info in tbl_infos.iter() { 22 | // TODO: return an error. 23 | let _ = tbl_info.clear_stmts(); 24 | } 25 | } 26 | 27 | pub fn reset_cached_stmt(stmt: *mut sqlite::stmt) -> Result { 28 | if stmt.is_null() { 29 | return Ok(ResultCode::OK); 30 | } 31 | stmt.clear_bindings()?; 32 | stmt.reset() 33 | } 34 | -------------------------------------------------------------------------------- /core/rs/core/src/teardown.rs: -------------------------------------------------------------------------------- 1 | use sqlite_nostd as sqlite; 2 | use sqlite_nostd::{Connection, ResultCode}; 3 | extern crate alloc; 4 | use alloc::format; 5 | 6 | pub fn remove_crr_clock_table_if_exists( 7 | db: *mut sqlite::sqlite3, 8 | table: &str, 9 | ) -> Result { 10 | let escaped_table = crate::util::escape_ident(table); 11 | db.exec_safe(&format!( 12 | "DROP TABLE IF EXISTS \"{table}__crsql_clock\"", 13 | table = escaped_table 14 | ))?; 15 | db.exec_safe(&format!( 16 | "DROP TABLE IF EXISTS \"{table}__crsql_pks\"", 17 | table = escaped_table 18 | )) 19 | } 20 | 21 | pub fn remove_crr_triggers_if_exist( 22 | db: *mut sqlite::sqlite3, 23 | table: &str, 24 | ) -> Result { 25 | let escaped_table = crate::util::escape_ident(table); 26 | 27 | db.exec_safe(&format!( 28 | "DROP TRIGGER IF EXISTS \"{table}__crsql_itrig\"", 29 | table = escaped_table 30 | ))?; 31 | 32 | db.exec_safe(&format!( 33 | "DROP TRIGGER IF EXISTS \"{table}__crsql_utrig\"", 34 | table = escaped_table 35 | ))?; 36 | 37 | // get all columns of table 38 | // iterate pk cols 39 | // drop triggers against those pk cols 40 | let stmt = db.prepare_v2("SELECT name FROM pragma_table_info(?) WHERE pk > 0")?; 41 | stmt.bind_text(1, table, sqlite::Destructor::STATIC)?; 42 | while stmt.step()? == ResultCode::ROW { 43 | let col_name = stmt.column_text(0)?; 44 | db.exec_safe(&format!( 45 | "DROP TRIGGER IF EXISTS \"{tbl_name}_{col_name}__crsql_utrig\"", 46 | tbl_name = crate::util::escape_ident(table), 47 | col_name = crate::util::escape_ident(col_name), 48 | ))?; 49 | } 50 | 51 | db.exec_safe(&format!( 52 | "DROP TRIGGER IF EXISTS \"{table}__crsql_dtrig\"", 53 | table = escaped_table 54 | )) 55 | } 56 | -------------------------------------------------------------------------------- /core/rs/core/src/test_exports.rs: -------------------------------------------------------------------------------- 1 | pub use crate::bootstrap; 2 | pub use crate::c; 3 | pub use crate::db_version; 4 | pub use crate::pack_columns; 5 | pub use crate::tableinfo; 6 | -------------------------------------------------------------------------------- /core/rs/core/src/triggers.rs: -------------------------------------------------------------------------------- 1 | extern crate alloc; 2 | use alloc::format; 3 | use sqlite::Connection; 4 | 5 | use core::ffi::c_char; 6 | 7 | use sqlite::{sqlite3, ResultCode}; 8 | use sqlite_nostd as sqlite; 9 | 10 | use crate::tableinfo::TableInfo; 11 | 12 | pub fn create_triggers( 13 | db: *mut sqlite3, 14 | table_info: &TableInfo, 15 | err: *mut *mut c_char, 16 | ) -> Result { 17 | create_insert_trigger(db, table_info, err)?; 18 | create_update_trigger(db, table_info, err)?; 19 | create_delete_trigger(db, table_info, err) 20 | } 21 | 22 | fn create_insert_trigger( 23 | db: *mut sqlite3, 24 | table_info: &TableInfo, 25 | _err: *mut *mut c_char, 26 | ) -> Result { 27 | let create_trigger_sql = format!( 28 | "CREATE TRIGGER IF NOT EXISTS \"{table_name}__crsql_itrig\" 29 | AFTER INSERT ON \"{table_name}\" WHEN crsql_internal_sync_bit() = 0 30 | BEGIN 31 | VALUES (crsql_after_insert('{table_name}', {pk_new_list})); 32 | END;", 33 | table_name = crate::util::escape_ident_as_value(&table_info.tbl_name), 34 | pk_new_list = crate::util::as_identifier_list(&table_info.pks, Some("NEW."))? 35 | ); 36 | 37 | db.exec_safe(&create_trigger_sql) 38 | } 39 | 40 | fn create_update_trigger( 41 | db: *mut sqlite3, 42 | table_info: &TableInfo, 43 | _err: *mut *mut c_char, 44 | ) -> Result { 45 | let table_name = &table_info.tbl_name; 46 | let pk_columns = &table_info.pks; 47 | let non_pk_columns = &table_info.non_pks; 48 | let pk_new_list = crate::util::as_identifier_list(pk_columns, Some("NEW."))?; 49 | let pk_old_list = crate::util::as_identifier_list(pk_columns, Some("OLD."))?; 50 | 51 | let trigger_body = if non_pk_columns.is_empty() { 52 | format!( 53 | "VALUES (crsql_after_update('{table_name}', {pk_new_list}, {pk_old_list}))", 54 | table_name = crate::util::escape_ident_as_value(table_name), 55 | pk_new_list = pk_new_list, 56 | pk_old_list = pk_old_list, 57 | ) 58 | } else { 59 | format!( 60 | "VALUES (crsql_after_update('{table_name}', {pk_new_list}, {pk_old_list}, {non_pk_new_list}, {non_pk_old_list}))", 61 | table_name = crate::util::escape_ident_as_value(table_name), 62 | pk_new_list = pk_new_list, 63 | pk_old_list = pk_old_list, 64 | non_pk_new_list = crate::util::as_identifier_list(non_pk_columns, Some("NEW."))?, 65 | non_pk_old_list = crate::util::as_identifier_list(non_pk_columns, Some("OLD."))? 66 | ) 67 | }; 68 | db.exec_safe(&format!( 69 | "CREATE TRIGGER IF NOT EXISTS \"{table_name}__crsql_utrig\" 70 | AFTER UPDATE ON \"{table_name}\" WHEN crsql_internal_sync_bit() = 0 71 | BEGIN 72 | {trigger_body}; 73 | END;", 74 | table_name = crate::util::escape_ident(table_name), 75 | )) 76 | } 77 | 78 | fn create_delete_trigger( 79 | db: *mut sqlite3, 80 | table_info: &TableInfo, 81 | _err: *mut *mut c_char, 82 | ) -> Result { 83 | let table_name = &table_info.tbl_name; 84 | let pk_columns = &table_info.pks; 85 | let pk_old_list = crate::util::as_identifier_list(pk_columns, Some("OLD."))?; 86 | 87 | let create_trigger_sql = format!( 88 | "CREATE TRIGGER IF NOT EXISTS \"{table_name}__crsql_dtrig\" 89 | AFTER DELETE ON \"{table_name}\" WHEN crsql_internal_sync_bit() = 0 90 | BEGIN 91 | VALUES (crsql_after_delete('{table_name}', {pk_old_list})); 92 | END;", 93 | table_name = crate::util::escape_ident(table_name), 94 | pk_old_list = pk_old_list 95 | ); 96 | 97 | db.exec_safe(&create_trigger_sql) 98 | } 99 | -------------------------------------------------------------------------------- /core/rs/core/src/util.rs: -------------------------------------------------------------------------------- 1 | extern crate alloc; 2 | 3 | use crate::{alloc::string::ToString, tableinfo::ColumnInfo}; 4 | use alloc::format; 5 | use alloc::string::String; 6 | use alloc::vec; 7 | use alloc::vec::Vec; 8 | use core::str::Utf8Error; 9 | use sqlite::{sqlite3, ColumnType, Connection, ResultCode}; 10 | use sqlite_nostd as sqlite; 11 | 12 | pub fn get_dflt_value( 13 | db: *mut sqlite3, 14 | table: &str, 15 | col: &str, 16 | ) -> Result, ResultCode> { 17 | let sql = "SELECT [dflt_value], [notnull] FROM pragma_table_info(?) WHERE name = ?"; 18 | let stmt = db.prepare_v2(sql)?; 19 | stmt.bind_text(1, table, sqlite_nostd::Destructor::STATIC)?; 20 | stmt.bind_text(2, col, sqlite_nostd::Destructor::STATIC)?; 21 | let rc = stmt.step()?; 22 | if rc == ResultCode::DONE { 23 | // There should always be a row for a column in pragma_table_info 24 | return Err(ResultCode::DONE); 25 | } 26 | 27 | let notnull = stmt.column_int(1); 28 | let dflt_column_type = stmt.column_type(0)?; 29 | 30 | // if the column is nullable and no default value is specified 31 | // then the default value is null. 32 | if notnull == 0 && dflt_column_type == ColumnType::Null { 33 | return Ok(Some(String::from("NULL"))); 34 | } 35 | 36 | if dflt_column_type == ColumnType::Null { 37 | // no default value specified 38 | // and the column is not nullable 39 | return Ok(None); 40 | } 41 | 42 | return Ok(Some(String::from(stmt.column_text(0)?))); 43 | } 44 | 45 | pub fn get_db_version_union_query(tbl_names: &Vec) -> String { 46 | let unions_str = tbl_names 47 | .iter() 48 | .map(|tbl_name| { 49 | format!( 50 | "SELECT max(db_version) as version FROM \"{}\"", 51 | escape_ident(tbl_name), 52 | ) 53 | }) 54 | .collect::>() 55 | .join(" UNION ALL "); 56 | 57 | return format!( 58 | "SELECT max(version) as version FROM ({} UNION SELECT value as 59 | version FROM crsql_master WHERE key = 'pre_compact_dbversion')", 60 | unions_str 61 | ); 62 | } 63 | 64 | pub fn slab_rowid(idx: i32, rowid: sqlite::int64) -> sqlite::int64 { 65 | if idx < 0 { 66 | return -1; 67 | } 68 | 69 | let modulo = rowid % crate::consts::ROWID_SLAB_SIZE; 70 | return (idx as i64) * crate::consts::ROWID_SLAB_SIZE + modulo; 71 | } 72 | 73 | pub fn where_list(columns: &Vec, prefix: Option<&str>) -> Result { 74 | let mut result = vec![]; 75 | for c in columns { 76 | let name = &c.name; 77 | if let Some(prefix) = prefix { 78 | result.push(format!( 79 | "{prefix}\"{col_name}\" IS ?", 80 | prefix = prefix, 81 | col_name = crate::util::escape_ident(name) 82 | )); 83 | } else { 84 | result.push(format!( 85 | "\"{col_name}\" IS ?", 86 | col_name = crate::util::escape_ident(name) 87 | )); 88 | } 89 | } 90 | 91 | Ok(result.join(" AND ")) 92 | } 93 | 94 | pub fn binding_list(num_slots: usize) -> String { 95 | core::iter::repeat('?') 96 | .take(num_slots) 97 | .map(|c| c.to_string()) 98 | .collect::>() 99 | .join(", ") 100 | } 101 | 102 | pub fn as_identifier_list( 103 | columns: &Vec, 104 | prefix: Option<&str>, 105 | ) -> Result { 106 | let mut result = vec![]; 107 | for c in columns { 108 | result.push(if let Some(prefix) = prefix { 109 | format!("{}\"{}\"", prefix, crate::util::escape_ident(&c.name)) 110 | } else { 111 | format!("\"{}\"", crate::util::escape_ident(&c.name)) 112 | }) 113 | } 114 | Ok(result.join(",")) 115 | } 116 | 117 | pub fn escape_ident(ident: &str) -> String { 118 | return ident.replace("\"", "\"\""); 119 | } 120 | 121 | pub fn escape_ident_as_value(ident: &str) -> String { 122 | return ident.replace("'", "''"); 123 | } 124 | 125 | pub trait Countable { 126 | fn count(self, sql: &str) -> Result; 127 | } 128 | 129 | impl Countable for *mut sqlite::sqlite3 { 130 | fn count(self, sql: &str) -> Result { 131 | let stmt = self.prepare_v2(sql)?; 132 | stmt.step()?; 133 | Ok(stmt.column_int(0)) 134 | } 135 | } 136 | 137 | #[cfg(test)] 138 | mod tests { 139 | use super::*; 140 | 141 | #[test] 142 | fn test_slab_rowid() { 143 | let foo_slab = slab_rowid(0, 1); 144 | let bar_slab = slab_rowid(1, 2); 145 | let baz_slab = slab_rowid(2, 3); 146 | 147 | assert_eq!(foo_slab, 1); 148 | assert_eq!(bar_slab, 2 + crate::consts::ROWID_SLAB_SIZE); 149 | assert_eq!(baz_slab, 3 + crate::consts::ROWID_SLAB_SIZE * 2); 150 | assert_eq!(slab_rowid(0, crate::consts::ROWID_SLAB_SIZE), 0); 151 | assert_eq!(slab_rowid(0, crate::consts::ROWID_SLAB_SIZE + 1), 1); 152 | 153 | let foo_slab = slab_rowid(0, crate::consts::ROWID_SLAB_SIZE + 1); 154 | let bar_slab = slab_rowid(1, crate::consts::ROWID_SLAB_SIZE + 2); 155 | let baz_slab = slab_rowid(2, crate::consts::ROWID_SLAB_SIZE * 2 + 3); 156 | 157 | assert_eq!(foo_slab, 1); 158 | assert_eq!(bar_slab, 2 + crate::consts::ROWID_SLAB_SIZE); 159 | assert_eq!(baz_slab, 3 + crate::consts::ROWID_SLAB_SIZE * 2); 160 | } 161 | 162 | #[test] 163 | fn test_get_db_version_union_query() { 164 | let tbl_names = vec!["foo".to_string(), "bar".to_string(), "baz".to_string()]; 165 | let union = get_db_version_union_query(&tbl_names); 166 | assert_eq!( 167 | union, 168 | "SELECT max(version) as version FROM (SELECT max(db_version) as version FROM \"foo\" UNION ALL SELECT max(db_version) as version FROM \"bar\" UNION ALL SELECT max(db_version) as version FROM \"baz\" UNION SELECT value as\n version FROM crsql_master WHERE key = 'pre_compact_dbversion')" 169 | ); 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /core/rs/fractindex-core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "crsql_fractindex_core" 3 | version = "0.1.0" 4 | edition = "2021" 5 | authors = ["Matt Wonlaw"] 6 | keywords = ["sqlite", "cr-sqlite", "fractional indexing"] 7 | license = "Apache 2" 8 | 9 | [lib] 10 | name = "crsql_fractindex_core" 11 | crate-type = ["rlib"] 12 | 13 | [dependencies] 14 | sqlite_nostd = { path="../sqlite-rs-embedded/sqlite_nostd" } 15 | 16 | [dev-dependencies] 17 | rand = "0.8.5" 18 | 19 | [profile.dev] 20 | panic = "abort" 21 | 22 | [profile.release] 23 | panic = "abort" 24 | 25 | [build] 26 | target = "wasm32-unknown-emscripten" 27 | 28 | [features] 29 | loadable_extension = ["sqlite_nostd/loadable_extension"] 30 | static = ["sqlite_nostd/static"] 31 | omit_load_extension = ["sqlite_nostd/omit_load_extension"] 32 | -------------------------------------------------------------------------------- /core/rs/fractindex-core/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vlcn-io/cr-sqlite/891fe9e0190dd20917f807d739c809e1bc32f6a3/core/rs/fractindex-core/README.md -------------------------------------------------------------------------------- /core/rs/fractindex-core/explore.sql: -------------------------------------------------------------------------------- 1 | SELECT crsql_orderings(after_primary_key, collection_column, collection_id, table, ordering_column); 2 | 3 | WITH "cte" AS ( 4 | SELECT "id", ${order_column}, row_number() OVER (ORDER BY ${order_column}) as "rn" FROM ${table} WHERE ${collection_id} = ${collection} 5 | ), "current" AS ( 6 | SELECT "rn" FROM "cte" 7 | WHERE "id" = ${after_id} 8 | ) 9 | SELECT "cte"."id", "cte".${order_column} FROM "cte", "current" 10 | WHERE ABS("cte"."rn" - "current"."rn") <= 1 11 | ORDER BY "cte"."rn" 12 | 13 | 14 | -- that'll get us the rows we need 15 | -- then we need to go down on before until we hit a distinct before 16 | 17 | -- if we collide on before too, run this to find before before. 18 | SELECT "${order_column}" FROM ${table} WHERE ${collection_id} = ${collection} AND "${order_column}" < ${before} ORDER BY "${order_column}" DESC LIMIT 1 19 | 20 | 21 | -- https://gist.github.com/Azarattum/0071f6dea0d2813c0b164b8d34ac2a1f 22 | 23 | 24 | -- below would return the new assignments 25 | -- where order is applied based on the selected row in the where statement 26 | -- we know pks from schema. 27 | -- we know order column if user defines it. 28 | SELECT id, order FROM foo_order WHERE collection_id = 1 AND item_id = 1 ORDER BY order ASC; 29 | /** 30 | 31 | 32 | UPDATE foo_order SET 33 | */ 34 | 35 | UPDATE todo 36 | SET order = orderings.order 37 | FROM (SELECT crsql_orderings(...)) AS orderings 38 | WHERE todo.id = orderings.id; 39 | 40 | CREATE VIRTUAL TABLE foo_fract USING crsql_fractional_index (order_column_name); 41 | 42 | ^-- from here we create the vtab based on the existing table schema. 43 | ^-- - make all columns available 44 | ^-- - 45 | 46 | -- interesting but no good given they scan the entire sub-query 47 | select prev_row, target_row, next_row 48 | from ( 49 | select 50 | lag(_rowid_) over (order by {ordering}) as prev_row, 51 | _rowid_ as target_row, 52 | lead(_rowid_) over (order by {ordering}) as next_row 53 | from {tbl} 54 | where {collection_id} = {collection} 55 | ) as t 56 | where target_pk = 'target_pk'; 57 | 58 | 59 | select crsql_fract_key_between(target_row_order, next_row_order) 60 | from ( 61 | select 62 | _rowid_ as target_row, 63 | lead(_rowid_) over (order by {ordering}) as next_row 64 | from {tbl} 65 | where {collection_id} = {collection} 66 | ) as t 67 | where target_pk = 'target_pk'; 68 | 69 | 70 | -- ^-- extend key_between to return NULL if provided with colliding keys 71 | 72 | 73 | select prev_row, target_row, next_row 74 | from ( 75 | select 76 | id, 77 | lag(_rowid_) over (order by id) as prev_row, 78 | _rowid_ as target_row, 79 | lead(_rowid_) over (order by id) as next_row 80 | from todo 81 | where list_id = 1 82 | ) as t 83 | where t.id = 1; 84 | 85 | -- as point queries: 86 | 87 | -- Count the number of rows in the table with the same ordering value as the target row. 88 | -- case when that. 89 | 90 | SELECT crsql_fract_key_between( 91 | (SELECT ordering FROM todo WHERE id = 1 AND list_id = 2 ORDER BY ordering DESC LIMIT 1, 1), 92 | (SELECT ordering FROM todo WHERE id = 1 AND list_id = 2 ORDER BY ordering ASC LIMIT 1, 1) 93 | ); 94 | 95 | SELECT max( 96 | (SELECT ordering FROM todo WHERE id = 1 AND list_id = 2 ORDER BY ordering DESC LIMIT 1, 1), 97 | (SELECT ordering FROM todo WHERE id = 1 AND list_id = 2 ORDER BY ordering ASC LIMIT 1, 1) 98 | ); 99 | 100 | -- ^- simple. Two point queries. Common case of no collisions we'll just get what we need back. 101 | -- If we collide, things get ineresting. We need to find the next distinct ordering value. 102 | 103 | -- on collision we do this 104 | SELECT ordering FROM todo WHERE list_id = y AND ordering < (SELECT ordering FROM todo WHERE id = x) ORDER BY ordering LIMIT 1; 105 | 106 | -- and bump the target row down to a slot between the returned ordering and its current value. 107 | -- down because we're inserting after. 108 | -- the new insert receives the value that the old thing had 109 | 110 | -- Find the row before the target row 111 | -- we will need to move that row 112 | SELECT _rowid_, ordering FROM todo 113 | JOIN (SELECT list_id FROM todo WHERE id = 1) as t ON todo.list_id = t.list_id ORDER BY ordering DESC LIMIT 1, 1; 114 | 115 | 116 | -- alt: 117 | 118 | -- find row b4 target row 119 | 120 | after_ordering = SELECT ordering FROM todo WHERE {after_id_predicates}; 121 | 122 | UPDATE todo SET ordering = crsql_fract_key_between( 123 | SELECT ordering FROM todo WHERE {list_predicates} AND ordering < {after_ordering}, 124 | {after_ordering} 125 | ) WHERE {after_id_predicates} 126 | 127 | return after_ordering; 128 | 129 | --- 130 | 131 | CREATE TRIGGER IF NOT EXISTS \"{table}_fractindex_update_trig\" 132 | INSTEAD OF UPDATE ON \"{table}_fractindex\" 133 | BEGIN 134 | UPDATE \"{table}\" SET 135 | {base_sets_ex_order}, 136 | \"{order_col}\" = CASE ( 137 | SELECT count(*) FROM \"{table}\" WHERE {list_predicates} AND \"{order_col}\" = ( 138 | SELECT \"{order_col}\" FROM \"{table}\" WHERE {after_predicates} 139 | ) 140 | ) 141 | WHEN 0 THEN crsql_fract_key_between( 142 | (SELECT \"{order_col}\" FROM \"{table}\" WHERE {after_predicates}), 143 | (SELECT \"{order_col}\" FROM \"{table}\" WHERE {list_predicates} AND \"{order_col}\" > ( 144 | SELECT \"{order_col}\" FROM \"{table}\" WHERE {after_predicates} 145 | ) LIMIT 1) 146 | ) 147 | ELSE crsql_fract_fix_conflict_return_old_key( 148 | ?, ?, {list_bind_slots}{maybe_comma}, -1, ?, {after_pk_values} 149 | ); 150 | END; 151 | -------------------------------------------------------------------------------- /core/rs/fractindex-core/rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "nightly-2023-10-05" -------------------------------------------------------------------------------- /core/rs/fractindex-core/src/as_ordered.rs: -------------------------------------------------------------------------------- 1 | use sqlite_nostd::{context, sqlite3, Connection, Context, Destructor, ResultCode, Value}; 2 | extern crate alloc; 3 | use alloc::format; 4 | use alloc::vec::Vec; 5 | 6 | use crate::{ 7 | fractindex_view::create_fract_view_and_view_triggers, 8 | util::{collection_max_select, collection_min_select, escape_ident}, 9 | }; 10 | 11 | // TODO: do validation and suggest indices? collection and order should be indexed as compound index 12 | // with col columns first. 13 | pub fn as_ordered( 14 | context: *mut context, 15 | db: *mut sqlite3, 16 | table: &str, 17 | order_by_column: *mut sqlite_nostd::value, 18 | collection_columns: &[*mut sqlite_nostd::value], 19 | ) { 20 | // 0. we should drop all triggers and views if they already exist 21 | // or be fancy and track schema versions to know if this is needed. 22 | let rc = db.exec_safe(&format!( 23 | "DROP TRIGGER IF EXISTS \"__crsql_{table}_fractindex_pend_trig\";", 24 | table = escape_ident(table) 25 | )); 26 | if rc.is_err() { 27 | context.result_error("Failed dropping prior incarnation of fractindex triggers"); 28 | } 29 | 30 | let rc = db.exec_safe(&format!( 31 | "DROP VIEW IF EXISTS \"{table}_fractindex\";", 32 | table = escape_ident(table) 33 | )); 34 | if rc.is_err() { 35 | context.result_error("Failed dropping prior incarnation of fractindex views"); 36 | } 37 | 38 | // 1. ensure that all columns exist in the target table 39 | let mut provided_cols = collection_columns.to_vec(); 40 | provided_cols.push(order_by_column); 41 | let rc = table_has_all_columns(db, table, &provided_cols); 42 | 43 | if rc.is_err() { 44 | context.result_error("Failed determining if all columns are present in the base table"); 45 | return; 46 | } 47 | if let Ok(false) = rc { 48 | context.result_error("all columns are not present in the base table"); 49 | return; 50 | } 51 | 52 | if let Err(_) = db.exec_safe("SAVEPOINT as_ordered;") { 53 | return; 54 | } 55 | 56 | let collection_column_names = collection_columns 57 | .iter() 58 | .map(|c| c.text()) 59 | .collect::>(); 60 | // 2. set up triggers to allow for append and pre-pend insertions 61 | if let Err(_) = create_pend_trigger(db, table, order_by_column, &collection_column_names) { 62 | let _ = db.exec_safe("ROLLBACK;"); 63 | context.result_error("Failed creating triggers for the base table"); 64 | return; 65 | } 66 | 67 | if let Err(_) = create_simple_move_trigger(db, table, order_by_column, &collection_column_names) 68 | { 69 | let _ = db.exec_safe("ROLLBACK;"); 70 | context.result_error("Failed creating simple move trigger"); 71 | return; 72 | } 73 | 74 | // 4. create fract view for insert after and move operations 75 | if let Err(_) = 76 | create_fract_view_and_view_triggers(db, table, order_by_column, &collection_column_names) 77 | { 78 | let _ = db.exec_safe("ROLLBACK;"); 79 | context.result_error("Failed creating view for the base table"); 80 | return; 81 | } 82 | 83 | let _ = db.exec_safe("RELEASE as_ordered;"); 84 | } 85 | 86 | fn table_has_all_columns( 87 | db: *mut sqlite3, 88 | table: &str, 89 | columns: &Vec<*mut sqlite_nostd::value>, 90 | ) -> Result { 91 | let bindings = columns.iter().map(|_| "?").collect::>().join(", "); 92 | let sql = format!( 93 | "SELECT count(*) FROM pragma_table_info(?) WHERE \"name\" IN ({})", 94 | bindings 95 | ); 96 | let stmt = db.prepare_v2(&sql)?; 97 | stmt.bind_text(1, table, Destructor::STATIC)?; 98 | for (i, col) in columns.iter().enumerate() { 99 | stmt.bind_value((i + 2) as i32, *col)?; 100 | } 101 | 102 | let step_code = stmt.step()?; 103 | if step_code == ResultCode::ROW { 104 | let count = stmt.column_int(0); 105 | if count != columns.len() as i32 { 106 | return Ok(false); 107 | } 108 | } 109 | 110 | Ok(true) 111 | } 112 | 113 | fn create_pend_trigger( 114 | db: *mut sqlite3, 115 | table: &str, 116 | order_by_column: *mut sqlite_nostd::value, 117 | collection_columns: &Vec<&str>, 118 | ) -> Result { 119 | let trigger = format!( 120 | "CREATE TRIGGER IF NOT EXISTS \"__crsql_{table}_fractindex_pend_trig\" AFTER INSERT ON \"{table}\" 121 | WHEN CAST(NEW.\"{order_by_column}\" AS INTEGER) = -1 OR CAST(NEW.\"{order_by_column}\" AS INTEGER) = 1 BEGIN 122 | UPDATE \"{table}\" SET \"{order_by_column}\" = CASE CAST(NEW.\"{order_by_column}\" AS INTEGER) 123 | WHEN -1 THEN crsql_fract_key_between(NULL, ({min_select})) 124 | WHEN 1 THEN crsql_fract_key_between(({max_select}), NULL) 125 | END 126 | WHERE _rowid_ = NEW._rowid_; 127 | END;", 128 | table = escape_ident(table), 129 | order_by_column = escape_ident(order_by_column.text()), 130 | min_select = collection_min_select(table, order_by_column, collection_columns)?, 131 | max_select = collection_max_select(table, order_by_column, collection_columns)? 132 | ); 133 | db.exec_safe(&trigger) 134 | } 135 | 136 | fn create_simple_move_trigger( 137 | db: *mut sqlite3, 138 | table: &str, 139 | order_by_column: *mut sqlite_nostd::value, 140 | collection_columns: &Vec<&str>, 141 | ) -> Result { 142 | // simple move allows moving a thing to the start or end of the list 143 | let trigger = format!( 144 | "CREATE TRIGGER IF NOT EXISTS \"__crsql_{table}_fractindex_ezmove\" AFTER UPDATE OF \"{order_col}\" ON \"{table}\" 145 | WHEN NEW.\"{order_col}\" = -1 OR NEW.\"{order_col}\" = 1 BEGIN 146 | UPDATE \"{table}\" SET \"{order_col}\" = CASE NEW.\"{order_col}\" 147 | WHEN -1 THEN crsql_fract_key_between(NULL, ({min_select})) 148 | WHEN 1 THEN crsql_fract_key_between(({max_select}), NULL) 149 | END 150 | WHERE _rowid_ = NEW._rowid_; 151 | END; 152 | ", 153 | table = escape_ident(table), 154 | order_col = escape_ident(order_by_column.text()), 155 | min_select = collection_min_select(table, order_by_column, collection_columns)?, 156 | max_select = collection_max_select(table, order_by_column, collection_columns)? 157 | ); 158 | db.exec_safe(&trigger) 159 | } 160 | -------------------------------------------------------------------------------- /core/rs/fractindex-core/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(not(test), no_std)] 2 | #![allow(non_upper_case_globals)] 3 | #![feature(core_intrinsics)] 4 | 5 | mod as_ordered; 6 | mod fractindex; 7 | mod fractindex_view; 8 | mod util; 9 | 10 | use core::ffi::{c_char, c_int}; 11 | pub use fractindex::*; 12 | use fractindex_view::fix_conflict_return_old_key; 13 | use sqlite::args; 14 | use sqlite::ColumnType; 15 | use sqlite::Connection; 16 | use sqlite::ResultCode; 17 | use sqlite::{Context, Value}; 18 | use sqlite_nostd as sqlite; 19 | extern crate alloc; 20 | 21 | pub extern "C" fn crsql_fract_as_ordered( 22 | ctx: *mut sqlite::context, 23 | argc: i32, 24 | argv: *mut *mut sqlite::value, 25 | ) { 26 | let args = args!(argc, argv); 27 | // decode the args, call as_ordered 28 | if args.len() < 2 { 29 | ctx.result_error( 30 | "Must provide at least 2 arguments -- the table name and the column to order by", 31 | ); 32 | return; 33 | } 34 | 35 | let db = ctx.db_handle(); 36 | let table = args[0].text(); 37 | let collection_columns = &args[2..]; 38 | as_ordered::as_ordered(ctx, db, table, args[1], collection_columns); 39 | } 40 | 41 | pub extern "C" fn crsql_fract_key_between( 42 | ctx: *mut sqlite::context, 43 | argc: i32, 44 | argv: *mut *mut sqlite::value, 45 | ) { 46 | let args = args!(argc, argv); 47 | 48 | let left = args[0]; 49 | let right = args[1]; 50 | 51 | let left = if left.value_type() == ColumnType::Null { 52 | None 53 | } else { 54 | Some(left.text()) 55 | }; 56 | 57 | let right = if right.value_type() == ColumnType::Null { 58 | None 59 | } else { 60 | Some(right.text()) 61 | }; 62 | 63 | let result = key_between(left, right); 64 | 65 | match result { 66 | Ok(Some(r)) => ctx.result_text_transient(&r), 67 | Ok(None) => ctx.result_null(), 68 | Err(r) => ctx.result_error(r), 69 | } 70 | } 71 | 72 | pub extern "C" fn crsql_fract_fix_conflict_return_old_key( 73 | ctx: *mut sqlite::context, 74 | argc: i32, 75 | argv: *mut *mut sqlite::value, 76 | ) { 77 | let args = args!(argc, argv); 78 | 79 | // process args 80 | // fix_conflict_return_old_key(); 81 | if args.len() < 4 { 82 | ctx.result_error("Too few arguments to fix_conflict_return_old_key"); 83 | return; 84 | } 85 | let table = args[0]; 86 | let order_col = args[1]; 87 | 88 | let collection_columns: &[*mut sqlite_nostd::value] = pull_collection_column_names(2, args); 89 | // 2 is where we started, + how many collection columns + 1 for the separator (-1) 90 | let next_index = 2 + collection_columns.len() + 1; 91 | // from next_index we'll read in primary key names and values 92 | 93 | let primary_key_and_value_count = args.len() - next_index; 94 | if primary_key_and_value_count <= 0 || primary_key_and_value_count % 2 != 0 { 95 | ctx.result_error("Incorrect number of primary keys and values provided. Must have at least 1 primary key."); 96 | return; 97 | } 98 | 99 | let primary_key_count = primary_key_and_value_count / 2; 100 | let pk_names = &args[next_index..next_index + primary_key_count]; 101 | let pk_values = 102 | &args[next_index + primary_key_count..next_index + primary_key_count + primary_key_count]; 103 | 104 | if let Err(_) = fix_conflict_return_old_key( 105 | ctx, 106 | table.text(), 107 | order_col, 108 | collection_columns, 109 | pk_names, 110 | pk_values, 111 | ) { 112 | ctx.result_error("Failed fixing up ordering conflicts on insert"); 113 | } 114 | 115 | return; 116 | } 117 | 118 | fn pull_collection_column_names( 119 | from: usize, 120 | args: &[*mut sqlite_nostd::value], 121 | ) -> &[*mut sqlite_nostd::value] { 122 | let mut i = from; 123 | while i < args.len() { 124 | let next = args[i]; 125 | if next.value_type() == ColumnType::Integer { 126 | break; 127 | } 128 | i += 1; 129 | } 130 | 131 | return &args[from..i]; 132 | } 133 | 134 | #[no_mangle] 135 | pub extern "C" fn sqlite3_crsqlfractionalindex_init( 136 | db: *mut sqlite::sqlite3, 137 | _err_msg: *mut *mut c_char, 138 | api: *mut sqlite::api_routines, 139 | ) -> c_int { 140 | sqlite::EXTENSION_INIT2(api); 141 | 142 | if let Err(rc) = db.create_function_v2( 143 | "crsql_fract_as_ordered", 144 | -1, 145 | sqlite::UTF8 | sqlite::DIRECTONLY, 146 | None, 147 | Some(crsql_fract_as_ordered), 148 | None, 149 | None, 150 | None, 151 | ) { 152 | return rc as c_int; 153 | } 154 | 155 | if let Err(rc) = db.create_function_v2( 156 | "crsql_fract_key_between", 157 | 2, 158 | sqlite::UTF8 | sqlite::DETERMINISTIC | sqlite::INNOCUOUS, 159 | None, 160 | Some(crsql_fract_key_between), 161 | None, 162 | None, 163 | None, 164 | ) { 165 | return rc as c_int; 166 | } 167 | 168 | if let Err(rc) = db.create_function_v2( 169 | "crsql_fract_fix_conflict_return_old_key", 170 | -1, 171 | sqlite::UTF8 | sqlite::INNOCUOUS, 172 | None, 173 | Some(crsql_fract_fix_conflict_return_old_key), 174 | None, 175 | None, 176 | None, 177 | ) { 178 | return rc as c_int; 179 | } 180 | 181 | ResultCode::OK as c_int 182 | } 183 | -------------------------------------------------------------------------------- /core/rs/fractindex-core/src/util.rs: -------------------------------------------------------------------------------- 1 | extern crate alloc; 2 | use alloc::format; 3 | use alloc::string::String; 4 | use alloc::vec::Vec; 5 | use sqlite_nostd::Value; 6 | use sqlite_nostd::{self, Connection, Destructor, ResultCode}; 7 | 8 | pub fn where_predicates>(columns: &[T]) -> Result { 9 | let mut predicates = String::new(); 10 | for (i, column_name) in columns.iter().enumerate() { 11 | predicates.push_str(&format!( 12 | "\"{}\" = NEW.\"{}\"", 13 | column_name.as_ref(), 14 | column_name.as_ref() 15 | )); 16 | if i < columns.len() - 1 { 17 | predicates.push_str(" AND "); 18 | } 19 | } 20 | if columns.len() == 0 { 21 | predicates.push_str("1"); 22 | } 23 | Ok(predicates) 24 | } 25 | 26 | pub fn collection_min_select( 27 | table: &str, 28 | order_by_column: *mut sqlite_nostd::value, 29 | collection_columns: &Vec<&str>, 30 | ) -> Result { 31 | Ok(format!( 32 | "SELECT MIN(\"{order_col}\") FROM \"{table}\" WHERE {list_preds} AND \"{order_col}\" != -1 AND \"{order_col}\" != 1", 33 | order_col = escape_ident(order_by_column.text()), 34 | table = escape_ident(table), 35 | list_preds = where_predicates(collection_columns)? 36 | )) 37 | } 38 | 39 | pub fn collection_max_select( 40 | table: &str, 41 | order_by_column: *mut sqlite_nostd::value, 42 | collection_columns: &Vec<&str>, 43 | ) -> Result { 44 | Ok(format!( 45 | "SELECT MAX(\"{order_col}\") FROM \"{table}\" WHERE {list_preds} AND \"{order_col}\" != -1 AND \"{order_col}\" != 1", 46 | order_col = escape_ident(order_by_column.text()), 47 | table = escape_ident(table), 48 | list_preds = where_predicates(collection_columns)? 49 | )) 50 | } 51 | 52 | /// Stmt is returned to the caller since all values become invalid as soon as the 53 | /// statement is dropped. 54 | pub fn extract_pk_columns( 55 | db: *mut sqlite_nostd::sqlite3, 56 | table: &str, 57 | ) -> Result, ResultCode> { 58 | let sql = "SELECT \"name\" FROM pragma_table_info(?) WHERE \"pk\" > 0 ORDER BY \"pk\" ASC"; 59 | let stmt = db.prepare_v2(&sql)?; 60 | stmt.bind_text(1, table, Destructor::STATIC)?; 61 | let mut columns = Vec::new(); 62 | while stmt.step()? == ResultCode::ROW { 63 | columns.push(String::from(stmt.column_text(0)?)); 64 | } 65 | Ok(columns) 66 | } 67 | 68 | /// Stmt is returned to the caller since all values become invalid as soon as the 69 | /// statement is dropped. 70 | pub fn extract_columns( 71 | db: *mut sqlite_nostd::sqlite3, 72 | table: &str, 73 | ) -> Result, ResultCode> { 74 | let sql = "SELECT \"name\" FROM pragma_table_info(?)"; 75 | let stmt = db.prepare_v2(&sql)?; 76 | stmt.bind_text(1, table, Destructor::STATIC)?; 77 | let mut columns = Vec::new(); 78 | while stmt.step()? == ResultCode::ROW { 79 | columns.push(String::from(stmt.column_text(0)?)); 80 | } 81 | Ok(columns) 82 | } 83 | 84 | pub fn escape_ident(ident: &str) -> String { 85 | return ident.replace("\"", "\"\""); 86 | } 87 | 88 | /// You should not use this for anything except defining triggers. 89 | pub fn escape_arg(arg: &str) -> String { 90 | return arg.replace("'", "''"); 91 | } 92 | -------------------------------------------------------------------------------- /core/rs/integration_check/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "crsql_integration_check" 3 | version = "0.0.1" 4 | edition = "2021" 5 | authors = ["Matt Wonlaw"] 6 | description = "rs integration check for crsqlite" 7 | keywords = ["sqlite"] 8 | license = "Apache-2.0" 9 | 10 | [lib] 11 | name = "crsql_integration_check" 12 | crate-type = ["staticlib"] 13 | 14 | [dependencies] 15 | sqlite_nostd = { path="../sqlite-rs-embedded/sqlite_nostd", features=["static", "omit_load_extension"] } 16 | crsql_bundle = { path="../bundle", features=["static", "omit_load_extension", "test"] } 17 | cargo-valgrind = "2.1.0" 18 | libc-print = "0.1.22" 19 | 20 | [build-dependencies] 21 | cc = "1.0" 22 | 23 | [features] 24 | libsql = ["crsql_bundle/libsql"] 25 | static = [] 26 | omit_load_extension = [] 27 | -------------------------------------------------------------------------------- /core/rs/integration_check/notes.md: -------------------------------------------------------------------------------- 1 | cargo test -- --nocapture 2 | cargo watch "test -- --nocapture" 3 | export RUST_BACKTRACE=0/1 4 | 5 | 6 | cargo test --test tableinfo 7 | 8 | // why is it `static` feature? -------------------------------------------------------------------------------- /core/rs/integration_check/rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "nightly-2023-10-05" -------------------------------------------------------------------------------- /core/rs/integration_check/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![no_std] 2 | extern crate alloc; 3 | mod t; 4 | use alloc::ffi::CString; 5 | pub use crsql_bundle; 6 | use libc_print::std_name::println; 7 | 8 | use core::ffi::c_char; 9 | use sqlite::{Connection, ManagedConnection, ResultCode}; 10 | use sqlite_nostd as sqlite; 11 | 12 | /** 13 | * Tests in a main crate because ubuntu is seriously fucked 14 | * and can't find `sqlite3_malloc` when compiling it as integration tests. 15 | */ 16 | #[no_mangle] 17 | pub extern "C" fn crsql_integration_check() { 18 | println!("Running automigrate"); 19 | t::automigrate::run_suite().expect("automigrate suite"); 20 | println!("Running backfill"); 21 | t::backfill::run_suite().expect("backfill suite"); 22 | println!("Running fract"); 23 | t::fract::run_suite(); 24 | println!("Running pack_columns"); 25 | t::pack_columns::run_suite().expect("pack columns suite"); 26 | println!("Running pk_only_tables"); 27 | t::pk_only_tables::run_suite(); 28 | println!("Running sync_bit_honored"); 29 | t::sync_bit_honored::run_suite().expect("sync bit honored suite"); 30 | println!("Running tableinfo"); 31 | t::tableinfo::run_suite(); 32 | println!("Running tear_down"); 33 | t::teardown::run_suite().expect("tear down suite"); 34 | println!("Running cl_set_vtab"); 35 | t::test_cl_set_vtab::run_suite().expect("test cl set vtab suite"); 36 | println!("Running db_version"); 37 | t::test_db_version::run_suite().expect("test db version suite"); 38 | } 39 | 40 | pub fn opendb() -> Result { 41 | let connection = sqlite::open(sqlite::strlit!(":memory:"))?; 42 | // connection.enable_load_extension(true)?; 43 | // connection.load_extension("../../dbg/crsqlite", None)?; 44 | Ok(CRConnection { db: connection }) 45 | } 46 | 47 | pub fn opendb_file(f: &str) -> Result { 48 | let f = CString::new(f)?; 49 | let connection = sqlite::open(f.as_ptr())?; 50 | // connection.enable_load_extension(true)?; 51 | // connection.load_extension("../../dbg/crsqlite", None)?; 52 | Ok(CRConnection { db: connection }) 53 | } 54 | 55 | pub struct CRConnection { 56 | pub db: ManagedConnection, 57 | } 58 | 59 | impl Drop for CRConnection { 60 | fn drop(&mut self) { 61 | if let Err(_) = self.db.exec_safe("SELECT crsql_finalize()") { 62 | panic!("Failed to finalize cr sql statements"); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /core/rs/integration_check/src/t/backfill.rs: -------------------------------------------------------------------------------- 1 | extern crate crsql_bundle; 2 | // Test that we can backfill old tables 3 | // the bulk of these tests have been moved to the python code 4 | // given integration tests are much more easily written in python 5 | use sqlite::{Connection, ResultCode}; 6 | use sqlite_nostd as sqlite; 7 | 8 | fn new_empty_table() -> Result<(), ResultCode> { 9 | let db = crate::opendb()?; 10 | // Just testing that we can execute these statements without error 11 | db.db 12 | .exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name);")?; 13 | db.db.exec_safe("SELECT crsql_as_crr('foo');")?; 14 | db.db.exec_safe("SELECT * FROM foo__crsql_clock;")?; 15 | Ok(()) 16 | } 17 | 18 | fn new_nonempty_table(apply_twice: bool) -> Result<(), ResultCode> { 19 | let db = crate::opendb()?; 20 | db.db 21 | .exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name);")?; 22 | db.db 23 | .exec_safe("INSERT INTO foo VALUES (1, 'one'), (2, 'two');")?; 24 | db.db.exec_safe("SELECT crsql_as_crr('foo');")?; 25 | let stmt = db.db.prepare_v2("SELECT * FROM foo__crsql_clock;")?; 26 | if apply_twice { 27 | db.db.exec_safe("SELECT crsql_as_crr('foo');")?; 28 | } 29 | 30 | let mut cnt = 0; 31 | while stmt.step()? == ResultCode::ROW { 32 | cnt = cnt + 1; 33 | assert_eq!(stmt.column_int64(0), cnt); // pk 34 | assert_eq!(stmt.column_text(1)?, "name"); // col name 35 | assert_eq!(stmt.column_int64(2), 1); // col version 36 | assert_eq!(stmt.column_int64(3), 1); // db version 37 | } 38 | assert_eq!(cnt, 2); 39 | 40 | // select from crsql_changes too 41 | let stmt = db.db.prepare_v2( 42 | "SELECT [table], [pk], [cid], [val], [col_version], [db_version] FROM crsql_changes;", 43 | )?; 44 | let mut cnt = 0; 45 | while stmt.step().unwrap() == ResultCode::ROW { 46 | cnt = cnt + 1; 47 | if cnt == 1 { 48 | assert_eq!(stmt.column_blob(1)?, [1, 9, 1]); // pk 49 | assert_eq!(stmt.column_text(3)?, "one"); // col value 50 | } else { 51 | assert_eq!(stmt.column_blob(1)?, [1, 9, 2]); // pk 52 | assert_eq!(stmt.column_text(3)?, "two"); // col value 53 | } 54 | assert_eq!(stmt.column_text(0)?, "foo"); // table name 55 | assert_eq!(stmt.column_text(2)?, "name"); // col name 56 | assert_eq!(stmt.column_int64(4), 1); // col version 57 | assert_eq!(stmt.column_int64(5), 1); // db version 58 | } 59 | assert_eq!(cnt, 2); 60 | Ok(()) 61 | } 62 | 63 | fn reapplied_empty_table() -> Result<(), ResultCode> { 64 | let db = crate::opendb()?; 65 | // Just testing that we can execute these statements without error 66 | db.db 67 | .exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name);")?; 68 | db.db.exec_safe("SELECT crsql_as_crr('foo');")?; 69 | db.db.exec_safe("SELECT * FROM foo__crsql_clock;")?; 70 | db.db.exec_safe("SELECT crsql_as_crr('foo');")?; 71 | db.db.exec_safe("SELECT * FROM foo__crsql_clock;")?; 72 | Ok(()) 73 | } 74 | 75 | pub fn run_suite() -> Result<(), ResultCode> { 76 | new_empty_table()?; 77 | new_nonempty_table(false)?; 78 | reapplied_empty_table()?; 79 | new_nonempty_table(true)?; 80 | Ok(()) 81 | } 82 | -------------------------------------------------------------------------------- /core/rs/integration_check/src/t/fract.rs: -------------------------------------------------------------------------------- 1 | extern crate crsql_bundle; 2 | use sqlite::Connection; 3 | use sqlite_nostd as sqlite; 4 | 5 | fn sort_no_list_col() { 6 | let w = crate::opendb().expect("db opened"); 7 | let db = &w.db; 8 | 9 | db.exec_safe("CREATE TABLE todo (id primary key, position)") 10 | .expect("table created"); 11 | db.exec_safe("SELECT crsql_fract_as_ordered('todo', 'position')") 12 | .expect("as ordered"); 13 | db.exec_safe( 14 | "INSERT INTO todo VALUES (1, 'Zm'), (2, 'ZmG'), (3, 'ZmG'), (4, 'ZmV'), (5, 'Zn')", 15 | ) 16 | .expect("inserted initial values"); 17 | db.exec_safe("UPDATE todo_fractindex SET after_id = 2 WHERE id = 5") 18 | .expect("repositioned id 5"); 19 | } 20 | 21 | pub fn run_suite() { 22 | sort_no_list_col(); 23 | } 24 | -------------------------------------------------------------------------------- /core/rs/integration_check/src/t/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod automigrate; 2 | pub mod backfill; 3 | pub mod fract; 4 | pub mod pack_columns; 5 | pub mod pk_only_tables; 6 | pub mod pk_update; 7 | pub mod sync_bit_honored; 8 | pub mod tableinfo; 9 | pub mod teardown; 10 | pub mod test_cl_set_vtab; 11 | pub mod test_db_version; 12 | -------------------------------------------------------------------------------- /core/rs/integration_check/src/t/pack_columns.rs: -------------------------------------------------------------------------------- 1 | use crsql_bundle::test_exports::pack_columns::unpack_columns; 2 | use crsql_bundle::test_exports::pack_columns::ColumnValue; 3 | use sqlite::{Connection, ResultCode}; 4 | use sqlite_nostd as sqlite; 5 | 6 | // The rust test is mainly to check with valgrind and ensure we're correctly 7 | // freeing data as we do some passing of destructors from rust to SQLite. 8 | // Complete property based tests for encode & decode exist in python. 9 | fn test_pack_columns() -> Result<(), ResultCode> { 10 | let db = crate::opendb()?; 11 | db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY, x, y)")?; 12 | let insert_stmt = db.db.prepare_v2("INSERT INTO foo VALUES (?, ?, ?)")?; 13 | let blob: [u8; 3] = [1, 2, 3]; 14 | 15 | insert_stmt.bind_int(1, 12)?; 16 | insert_stmt.bind_text(2, "str", sqlite::Destructor::STATIC)?; 17 | insert_stmt.bind_blob(3, &blob, sqlite::Destructor::STATIC)?; 18 | insert_stmt.step()?; 19 | 20 | let select_stmt = db 21 | .db 22 | .prepare_v2("SELECT quote(crsql_pack_columns(id, x, y)) FROM foo")?; 23 | select_stmt.step()?; 24 | let result = select_stmt.column_text(0)?; 25 | assert!(result == "X'03090C0B037374720C03010203'"); 26 | // 03 09 0C 0B 03 73 74 72 0C 03 01 02 03 27 | // cols: 03 28 | // type & intlen: 09 -> 0b00001001 -> 01 type & 01 intlen 29 | // value: 0C -> 12 30 | // type & intlen: 0B -> 0b00001011 -> 03 type & 01 intlen 31 | // 03 -> len 32 | // 73 74 72 -> str 33 | // type & intlen: 0C -> 0b00001100 -> 04 type & 01 intlen 34 | // len: 03 35 | // bytes: 01 02 3 36 | // voila, done in 13 bytes! < 18 byte string < 26 byte binary w/o varints 37 | 38 | // Before variable length encoding: 39 | // 03 01 00 00 00 00 00 00 00 0C 03 00 00 00 03 73 74 72 04 00 00 00 03 01 02 03 40 | // cols:03 41 | // type: 01 (integer) 42 | // value: 00 00 00 00 00 00 00 0C (12) TODO: encode as variable length integers to save space? 43 | // type: 03 (text) 44 | // len: 00 00 00 03 (3) 45 | // byes: 73 (s) 74 (t) 72 (r) 46 | // type: 04 (blob) 47 | // len: 00 00 00 03 (3) 48 | // bytes: 01 02 03 49 | // vs string: 50 | // 12|'str'|x'010203' 51 | // ^ 18 bytes via string 52 | // vs 53 | // 26 bytes via binary 54 | // 13 bytes are wasted due to not using variable length encoding for integers 55 | // So.. do variable length ints? 56 | 57 | let select_stmt = db 58 | .db 59 | .prepare_v2("SELECT crsql_pack_columns(id, x, y) FROM foo")?; 60 | select_stmt.step()?; 61 | let result = select_stmt.column_blob(0)?; 62 | assert!(result.len() == 13); 63 | let unpacked = unpack_columns(result)?; 64 | assert!(unpacked.len() == 3); 65 | 66 | if let ColumnValue::Integer(i) = unpacked[0] { 67 | assert!(i == 12); 68 | } else { 69 | assert!("unexpected type" == ""); 70 | } 71 | if let ColumnValue::Text(s) = &unpacked[1] { 72 | assert!(s == "str") 73 | } else { 74 | assert!("unexpected type" == ""); 75 | } 76 | if let ColumnValue::Blob(b) = &unpacked[2] { 77 | assert!(b[..] == blob); 78 | } else { 79 | assert!("unexpected type" == ""); 80 | } 81 | 82 | db.db.exec_safe("DELETE FROM foo")?; 83 | let insert_stmt = db.db.prepare_v2("INSERT INTO foo VALUES (?, ?, ?)")?; 84 | 85 | insert_stmt.bind_int(1, 0)?; 86 | insert_stmt.bind_int(2, 10000000)?; 87 | insert_stmt.bind_int(3, -2500000)?; 88 | insert_stmt.step()?; 89 | 90 | let select_stmt = db 91 | .db 92 | .prepare_v2("SELECT crsql_pack_columns(id, x, y) FROM foo")?; 93 | select_stmt.step()?; 94 | let result = select_stmt.column_blob(0)?; 95 | let unpacked = unpack_columns(result)?; 96 | assert!(unpacked.len() == 3); 97 | 98 | if let ColumnValue::Integer(i) = unpacked[0] { 99 | assert!(i == 0); 100 | } else { 101 | assert!("unexpected type" == ""); 102 | } 103 | if let ColumnValue::Integer(i) = unpacked[1] { 104 | assert!(i == 10000000) 105 | } else { 106 | assert!("unexpected type" == ""); 107 | } 108 | if let ColumnValue::Integer(i) = unpacked[2] { 109 | assert!(i == -2500000); 110 | } else { 111 | assert!("unexpected type" == ""); 112 | } 113 | 114 | Ok(()) 115 | } 116 | 117 | fn test_unpack_columns() -> Result<(), ResultCode> { 118 | let db = crate::opendb().unwrap(); 119 | db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY, x, y)")?; 120 | let insert_stmt = db.db.prepare_v2("INSERT INTO foo VALUES (?, ?, ?)")?; 121 | let blob: [u8; 3] = [1, 2, 3]; 122 | 123 | insert_stmt.bind_int(1, 12)?; 124 | insert_stmt.bind_text(2, "str", sqlite::Destructor::STATIC)?; 125 | insert_stmt.bind_blob(3, &blob, sqlite::Destructor::STATIC)?; 126 | insert_stmt.step()?; 127 | 128 | let select_stmt = db 129 | .db 130 | .prepare_v2("SELECT cell FROM crsql_unpack_columns WHERE package = (SELECT crsql_pack_columns(id, x, y) FROM foo)")?; 131 | select_stmt.step()?; 132 | assert!(select_stmt.column_int(0) == 12); 133 | select_stmt.step()?; 134 | assert!(select_stmt.column_text(0)? == "str"); 135 | select_stmt.step()?; 136 | assert!(select_stmt.column_blob(0)? == blob); 137 | 138 | Ok(()) 139 | } 140 | 141 | pub fn run_suite() -> Result<(), ResultCode> { 142 | test_pack_columns()?; 143 | test_unpack_columns() 144 | } 145 | -------------------------------------------------------------------------------- /core/rs/integration_check/src/t/pk_update.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * These tests are very similar to `pk_only_tables` tests. 3 | * What we want to test here is that rows whose primary keys get changed get 4 | * replicated correctly. 5 | * 6 | * Example: 7 | * ``` 8 | * CREATE TABLE foo (id primary key, value); 9 | * ``` 10 | * 11 | * | id | value | 12 | * | -- | ----- | 13 | * | 1 | abc | 14 | * 15 | * Now we: 16 | * ``` 17 | * UPDATE foo SET id = 2 WHERE id = 1; 18 | * ``` 19 | * 20 | * This should be a _delete_ of row id 1 and a _create_ of 21 | * row id 2, bringing all the values from row 1 to row 2. 22 | * 23 | * pk_only_tables.rs tested this for table that _only_ 24 | * had primary key columns but not for tables that have 25 | * primary key columns + other columns. 26 | */ 27 | -------------------------------------------------------------------------------- /core/rs/integration_check/src/t/sync_bit_honored.rs: -------------------------------------------------------------------------------- 1 | extern crate crsql_bundle; 2 | use sqlite::{Connection, ResultCode}; 3 | use sqlite_nostd as sqlite; 4 | 5 | // If sync bit is on, nothing gets written to clock tables for that connection. 6 | fn sync_bit_honored() -> Result<(), ResultCode> { 7 | let db = crate::opendb()?; 8 | let conn = &db.db; 9 | conn.exec_safe("CREATE TABLE foo (a primary key not null, b);")?; 10 | conn.exec_safe("SELECT crsql_as_crr('foo');")?; 11 | conn.exec_safe("SELECT crsql_internal_sync_bit(1)")?; 12 | conn.exec_safe("INSERT INTO foo VALUES (1, 2);")?; 13 | conn.exec_safe("UPDATE foo SET b = 5 WHERE a = 1;")?; 14 | conn.exec_safe("INSERT INTO foo VALUES (2, 2);")?; 15 | conn.exec_safe("DELETE FROM foo WHERE a = 2;")?; 16 | 17 | let stmt = conn.prepare_v2("SELECT 1 FROM foo__crsql_clock")?; 18 | let result = stmt.step()?; 19 | assert!(result == ResultCode::DONE); 20 | 21 | Ok(()) 22 | } 23 | 24 | pub fn run_suite() -> Result<(), ResultCode> { 25 | sync_bit_honored() 26 | } 27 | -------------------------------------------------------------------------------- /core/rs/integration_check/src/t/teardown.rs: -------------------------------------------------------------------------------- 1 | extern crate crsql_bundle; 2 | use sqlite::{Connection, ResultCode}; 3 | use sqlite_nostd as sqlite; 4 | 5 | fn tear_down() -> Result<(), ResultCode> { 6 | let db = crate::opendb()?; 7 | db.db 8 | .exec_safe("CREATE TABLE foo (a primary key not null, b);")?; 9 | db.db.exec_safe("SELECT crsql_as_crr('foo');")?; 10 | db.db.exec_safe("SELECT crsql_as_table('foo');")?; 11 | let stmt = db 12 | .db 13 | .prepare_v2("SELECT count(*) FROM sqlite_master WHERE name LIKE 'foo__%'")?; 14 | stmt.step()?; 15 | let count = stmt.column_int(0); 16 | assert!(count == 0); 17 | Ok(()) 18 | } 19 | 20 | pub fn run_suite() -> Result<(), ResultCode> { 21 | tear_down() 22 | } 23 | -------------------------------------------------------------------------------- /core/rs/integration_check/src/t/test_cl_set_vtab.rs: -------------------------------------------------------------------------------- 1 | extern crate crsql_bundle; 2 | use sqlite::{Connection, ResultCode}; 3 | use sqlite_nostd as sqlite; 4 | 5 | /* 6 | Test: 7 | - create crr 8 | - destroy crr 9 | - use crr that was created 10 | - create if not exist vtab 11 | - 12 | */ 13 | 14 | fn create_crr_via_vtab() -> Result<(), ResultCode> { 15 | let db = crate::opendb()?; 16 | let conn = &db.db; 17 | 18 | conn.exec_safe("CREATE VIRTUAL TABLE foo_schema USING CLSet (a primary key not null, b);")?; 19 | conn.exec_safe("INSERT INTO foo VALUES (1, 2);")?; 20 | let stmt = conn.prepare_v2("SELECT count(*) FROM crsql_changes")?; 21 | stmt.step()?; 22 | let count = stmt.column_int(0); 23 | assert_eq!(count, 1); 24 | Ok(()) 25 | } 26 | 27 | fn destroy_crr_via_vtab() -> Result<(), ResultCode> { 28 | let db = crate::opendb()?; 29 | let conn = &db.db; 30 | 31 | conn.exec_safe("CREATE VIRTUAL TABLE foo_schema USING CLSet (a primary key not null, b);")?; 32 | conn.exec_safe("DROP TABLE foo_schema")?; 33 | let stmt = conn.prepare_v2("SELECT count(*) FROM sqlite_master WHERE name LIKE '%foo%'")?; 34 | stmt.step()?; 35 | let count = stmt.column_int(0); 36 | assert_eq!(count, 0); 37 | Ok(()) 38 | } 39 | 40 | fn create_invalid_crr() -> Result<(), ResultCode> { 41 | let db = crate::opendb()?; 42 | let conn = &db.db; 43 | 44 | let result = conn.exec_safe("CREATE VIRTUAL TABLE foo_schema USING CLSet (a, b);"); 45 | assert_eq!(result, Err(ResultCode::ERROR)); 46 | let msg = conn.errmsg().unwrap(); 47 | assert_eq!( 48 | msg, 49 | "Table foo has no primary key or primary key is nullable. CRRs must have a non nullable primary key" 50 | ); 51 | Ok(()) 52 | } 53 | 54 | fn create_if_not_exists() -> Result<(), ResultCode> { 55 | let db = crate::opendb()?; 56 | let conn = &db.db; 57 | 58 | conn.exec_safe( 59 | "CREATE VIRTUAL TABLE IF NOT EXISTS foo_schema USING CLSet (a primary key not null, b);", 60 | )?; 61 | conn.exec_safe("INSERT INTO foo VALUES (1, 2);")?; 62 | let stmt = conn.prepare_v2("SELECT count(*) FROM crsql_changes")?; 63 | stmt.step()?; 64 | let count = stmt.column_int(0); 65 | assert_eq!(count, 1); 66 | drop(stmt); 67 | // second create is a no-op 68 | conn.exec_safe( 69 | "CREATE VIRTUAL TABLE IF NOT EXISTS foo_schema USING CLSet (a primary key not null, b);", 70 | )?; 71 | let stmt = conn.prepare_v2("SELECT count(*) FROM crsql_changes")?; 72 | stmt.step()?; 73 | let count = stmt.column_int(0); 74 | assert_eq!(count, 1); 75 | Ok(()) 76 | } 77 | 78 | // and later migration tests 79 | // UPDATE foo SET schema = '...'; 80 | // INSERT INTO foo (alter) VALUES ('...'); 81 | // and auto-migrate tests for whole schema. 82 | // auto-migrate would... 83 | // - re-write `create vtab` things as `update foo set schema = ...` where those vtabs did not exist. 84 | 85 | pub fn run_suite() -> Result<(), ResultCode> { 86 | create_crr_via_vtab()?; 87 | destroy_crr_via_vtab()?; 88 | create_invalid_crr()?; 89 | create_if_not_exists() 90 | } 91 | -------------------------------------------------------------------------------- /core/rs/integration_check/src/t/test_db_version.rs: -------------------------------------------------------------------------------- 1 | extern crate alloc; 2 | use alloc::{ffi::CString, string::String}; 3 | use core::ffi::c_char; 4 | use crsql_bundle::test_exports; 5 | use sqlite::{Connection, ResultCode}; 6 | use sqlite_nostd as sqlite; 7 | 8 | fn make_site() -> *mut c_char { 9 | let inner_ptr: *mut c_char = CString::new("0000000000000000").unwrap().into_raw(); 10 | inner_ptr 11 | } 12 | 13 | fn test_fetch_db_version_from_storage() -> Result { 14 | let c = crate::opendb().expect("db opened"); 15 | let db = &c.db; 16 | let raw_db = db.db; 17 | let ext_data = unsafe { test_exports::c::crsql_newExtData(raw_db, make_site()) }; 18 | 19 | test_exports::db_version::fetch_db_version_from_storage(raw_db, ext_data)?; 20 | // no clock tables, no version. 21 | assert_eq!(0, unsafe { (*ext_data).dbVersion }); 22 | 23 | // this was a bug where calling twice on a fresh db would fail the second 24 | // time. 25 | test_exports::db_version::fetch_db_version_from_storage(raw_db, ext_data)?; 26 | // should still return same data on a subsequent call with no schema 27 | assert_eq!(0, unsafe { (*ext_data).dbVersion }); 28 | 29 | // create some schemas 30 | db.exec_safe("CREATE TABLE foo (a primary key not null, b);") 31 | .expect("made foo"); 32 | db.exec_safe("SELECT crsql_as_crr('foo');") 33 | .expect("made foo crr"); 34 | test_exports::db_version::fetch_db_version_from_storage(raw_db, ext_data)?; 35 | // still v0 since no rows are inserted 36 | assert_eq!(0, unsafe { (*ext_data).dbVersion }); 37 | 38 | // version is bumped due to insert 39 | db.exec_safe("INSERT INTO foo (a, b) VALUES (1, 2);") 40 | .expect("inserted"); 41 | test_exports::db_version::fetch_db_version_from_storage(raw_db, ext_data)?; 42 | assert_eq!(1, unsafe { (*ext_data).dbVersion }); 43 | 44 | db.exec_safe("CREATE TABLE bar (a primary key not null, b);") 45 | .expect("created bar"); 46 | db.exec_safe("SELECT crsql_as_crr('bar');") 47 | .expect("bar as crr"); 48 | db.exec_safe("INSERT INTO bar VALUES (1, 2)") 49 | .expect("inserted into bar"); 50 | test_exports::db_version::fetch_db_version_from_storage(raw_db, ext_data)?; 51 | assert_eq!(2, unsafe { (*ext_data).dbVersion }); 52 | 53 | test_exports::db_version::fetch_db_version_from_storage(raw_db, ext_data)?; 54 | assert_eq!(2, unsafe { (*ext_data).dbVersion }); 55 | 56 | unsafe { 57 | test_exports::c::crsql_freeExtData(ext_data); 58 | }; 59 | 60 | Ok(ResultCode::OK) 61 | } 62 | 63 | fn test_next_db_version() -> Result<(), String> { 64 | let c = crate::opendb().expect("db opened"); 65 | let db = &c.db; 66 | let raw_db = db.db; 67 | let ext_data = unsafe { test_exports::c::crsql_newExtData(raw_db, make_site()) }; 68 | 69 | // is current + 1 70 | // doesn't bump forward on successive calls 71 | assert_eq!( 72 | 1, 73 | test_exports::db_version::next_db_version(raw_db, ext_data, None)? 74 | ); 75 | assert_eq!( 76 | 1, 77 | test_exports::db_version::next_db_version(raw_db, ext_data, None)? 78 | ); 79 | // doesn't roll back with new provideds 80 | assert_eq!( 81 | 1, 82 | test_exports::db_version::next_db_version(raw_db, ext_data, Some(-1))? 83 | ); 84 | assert_eq!( 85 | 1, 86 | test_exports::db_version::next_db_version(raw_db, ext_data, Some(0))? 87 | ); 88 | // sets to max of current and provided 89 | assert_eq!( 90 | 3, 91 | test_exports::db_version::next_db_version(raw_db, ext_data, Some(3))? 92 | ); 93 | assert_eq!( 94 | 3, 95 | test_exports::db_version::next_db_version(raw_db, ext_data, Some(2))? 96 | ); 97 | 98 | // existing db version not touched 99 | assert_eq!(0, unsafe { (*ext_data).dbVersion }); 100 | 101 | unsafe { 102 | test_exports::c::crsql_freeExtData(ext_data); 103 | }; 104 | Ok(()) 105 | } 106 | 107 | pub fn run_suite() -> Result<(), String> { 108 | test_fetch_db_version_from_storage()?; 109 | test_next_db_version()?; 110 | Ok(()) 111 | } 112 | -------------------------------------------------------------------------------- /core/src/changes-vtab-rowid.test.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Test that: 3 | * 1. The rowid we return for a row on insert matches the rowid we get for it on 4 | * read 5 | * 2. That we can query the vtab by rowid?? 6 | * 3. The returned rowid matches the rowid used in a point query by rowid 7 | * 4. 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "consts.h" 16 | #include "crsqlite.h" 17 | 18 | int crsql_close(sqlite3 *db); 19 | 20 | // static void testRowidForInsert() { 21 | // printf("RowidForInsert\n"); 22 | 23 | // sqlite3 *db; 24 | // int rc; 25 | // rc = sqlite3_open(":memory:", &db); 26 | 27 | // rc = sqlite3_exec(db, "CREATE TABLE foo (a primary key not null, b);", 0, 28 | // 0, 0); rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo');", 0, 0, 0); 29 | // assert(rc == SQLITE_OK); 30 | 31 | // char *zSql = 32 | // "INSERT INTO crsql_changes ([table], pk, cid, val, col_version, " 33 | // "db_version) " 34 | // "VALUES " 35 | // "('foo', '1', 'b', '1', 1, 1) RETURNING _rowid_;"; 36 | // sqlite3_stmt *pStmt; 37 | // rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); 38 | // assert(rc == SQLITE_OK); 39 | // assert(sqlite3_step(pStmt) == SQLITE_ROW); 40 | // printf("rowid: %d\n", sqlite3_column_int64(pStmt, 0)); 41 | // assert(sqlite3_column_int64(pStmt, 0) == 1); 42 | // sqlite3_finalize(pStmt); 43 | 44 | // // TODO: make extra crr tables and check their slab allotments and returned 45 | // // rowids 46 | 47 | // crsql_close(db); 48 | // printf("\t\e[0;32mSuccess\e[0m\n"); 49 | // } 50 | 51 | static void testRowidsForReads() { 52 | printf("RowidForReads\n"); 53 | 54 | sqlite3 *db; 55 | int rc; 56 | rc = sqlite3_open(":memory:", &db); 57 | 58 | rc = sqlite3_exec(db, "CREATE TABLE foo (a primary key not null, b);", 0, 0, 59 | 0); 60 | rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo');", 0, 0, 0); 61 | assert(rc == SQLITE_OK); 62 | sqlite3_exec(db, "INSERT INTO foo VALUES (1,2);", 0, 0, 0); 63 | sqlite3_exec(db, "INSERT INTO foo VALUES (2,3);", 0, 0, 0); 64 | 65 | char *zSql = "SELECT _rowid_ FROM crsql_changes"; 66 | sqlite3_stmt *pStmt; 67 | rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); 68 | assert(rc == SQLITE_OK); 69 | assert(sqlite3_step(pStmt) == SQLITE_ROW); 70 | assert(sqlite3_column_int64(pStmt, 0) == 1); 71 | assert(sqlite3_step(pStmt) == SQLITE_ROW); 72 | assert(sqlite3_column_int64(pStmt, 0) == 2); 73 | sqlite3_finalize(pStmt); 74 | 75 | rc = 76 | sqlite3_exec(db, "CREATE TABLE bar (a primary key not null, b)", 0, 0, 0); 77 | rc += sqlite3_exec(db, "SELECT crsql_as_crr('bar');", 0, 0, 0); 78 | rc += sqlite3_exec(db, "INSERT INTO bar VALUES (1,2);", 0, 0, 0); 79 | rc += sqlite3_exec(db, "INSERT INTO bar VALUES (2,3);", 0, 0, 0); 80 | 81 | rc += 82 | sqlite3_exec(db, "CREATE TABLE baz (a primary key not null, b)", 0, 0, 0); 83 | rc += sqlite3_exec(db, "SELECT crsql_as_crr('baz');", 0, 0, 0); 84 | rc += sqlite3_exec(db, "INSERT INTO baz VALUES (1,2);", 0, 0, 0); 85 | rc += sqlite3_exec(db, "INSERT INTO baz VALUES (2,3);", 0, 0, 0); 86 | 87 | assert(rc == SQLITE_OK); 88 | 89 | sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); 90 | sqlite3_step(pStmt); 91 | assert(sqlite3_column_int64(pStmt, 0) == 1); 92 | sqlite3_step(pStmt); 93 | assert(sqlite3_column_int64(pStmt, 0) == 2); 94 | sqlite3_step(pStmt); 95 | assert(sqlite3_column_int64(pStmt, 0) == 1 * ROWID_SLAB_SIZE + 1); 96 | sqlite3_step(pStmt); 97 | assert(sqlite3_column_int64(pStmt, 0) == 1 * ROWID_SLAB_SIZE + 2); 98 | sqlite3_step(pStmt); 99 | assert(sqlite3_column_int64(pStmt, 0) == 2 * ROWID_SLAB_SIZE + 1); 100 | sqlite3_step(pStmt); 101 | assert(sqlite3_column_int64(pStmt, 0) == 2 * ROWID_SLAB_SIZE + 2); 102 | sqlite3_finalize(pStmt); 103 | 104 | crsql_close(db); 105 | printf("\t\e[0;32mSuccess\e[0m\n"); 106 | } 107 | 108 | // static void testInsertRowidMatchesReadRowid() { 109 | // printf("RowidForInsertMatchesRowidForRead\n"); 110 | // printf("\t\e[0;32mSuccess\e[0m\n"); 111 | // } 112 | 113 | void crsqlChangesVtabRowidTestSuite() { 114 | printf("\e[47m\e[1;30mSuite: crsql_changesVtabRowid\e[0m\n"); 115 | // testRowidForInsert(); 116 | testRowidsForReads(); 117 | // testInsertRowidMatchesReadRowid(); 118 | } -------------------------------------------------------------------------------- /core/src/changes-vtab.c: -------------------------------------------------------------------------------- 1 | #include "changes-vtab.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "consts.h" 8 | #include "crsqlite.h" 9 | #include "ext-data.h" 10 | #include "rust.h" 11 | 12 | int crsql_changes_next(sqlite3_vtab_cursor *cur); 13 | 14 | /** 15 | * Created when the virtual table is initialized. 16 | * This happens when the vtab is first used in a given connection. 17 | * The method allocated the crsql_Changes_vtab for use for the duration 18 | * of the connection. 19 | */ 20 | static int changesConnect(sqlite3 *db, void *pAux, int argc, 21 | const char *const *argv, sqlite3_vtab **ppVtab, 22 | char **pzErr) { 23 | crsql_Changes_vtab *pNew; 24 | int rc; 25 | 26 | rc = sqlite3_declare_vtab( 27 | db, 28 | "CREATE TABLE x([table] TEXT NOT NULL, [pk] BLOB NOT NULL, [cid] TEXT " 29 | "NOT NULL, [val] ANY, [col_version] INTEGER NOT NULL, [db_version] " 30 | "INTEGER NOT NULL, [site_id] BLOB NOT NULL, [cl] INTEGER NOT NULL, [seq] " 31 | "INTEGER NOT NULL)"); 32 | if (rc != SQLITE_OK) { 33 | *pzErr = sqlite3_mprintf("Could not define the table"); 34 | return rc; 35 | } 36 | pNew = sqlite3_malloc(sizeof(*pNew)); 37 | *ppVtab = (sqlite3_vtab *)pNew; 38 | if (pNew == 0) { 39 | *pzErr = sqlite3_mprintf("Out of memory"); 40 | return SQLITE_NOMEM; 41 | } 42 | memset(pNew, 0, sizeof(*pNew)); 43 | pNew->db = db; 44 | pNew->pExtData = (crsql_ExtData *)pAux; 45 | 46 | rc = crsql_ensure_table_infos_are_up_to_date(db, pNew->pExtData, 47 | &(*ppVtab)->zErrMsg); 48 | if (rc != SQLITE_OK) { 49 | *pzErr = sqlite3_mprintf("Could not update table infos"); 50 | sqlite3_free(pNew); 51 | return rc; 52 | } 53 | 54 | return rc; 55 | } 56 | 57 | /** 58 | * Called when the connection closes to free 59 | * all resources allocated by `changesConnect` 60 | * 61 | * I.e., free everything in `crsql_Changes_vtab` / `pVtab` 62 | */ 63 | static int changesDisconnect(sqlite3_vtab *pVtab) { 64 | crsql_Changes_vtab *p = (crsql_Changes_vtab *)pVtab; 65 | // ext data is free by other registered extensions 66 | sqlite3_free(p); 67 | return SQLITE_OK; 68 | } 69 | 70 | /** 71 | * Called to allocate a cursor for use in executing a query against 72 | * the virtual table. 73 | */ 74 | static int changesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor) { 75 | crsql_Changes_cursor *pCur; 76 | pCur = sqlite3_malloc(sizeof(*pCur)); 77 | if (pCur == 0) { 78 | return SQLITE_NOMEM; 79 | } 80 | memset(pCur, 0, sizeof(*pCur)); 81 | *ppCursor = &pCur->base; 82 | pCur->pTab = (crsql_Changes_vtab *)p; 83 | return SQLITE_OK; 84 | } 85 | 86 | static int changesCrsrFinalize(crsql_Changes_cursor *crsr) { 87 | // Assign pointers to null after freeing 88 | // since we can get into this twice for the same cursor object. 89 | int rc = SQLITE_OK; 90 | rc += sqlite3_finalize(crsr->pChangesStmt); 91 | crsr->pChangesStmt = 0; 92 | if (crsr->pRowStmt != 0) { 93 | rc += sqlite3_clear_bindings(crsr->pRowStmt); 94 | rc += sqlite3_reset(crsr->pRowStmt); 95 | } 96 | crsr->pRowStmt = 0; 97 | 98 | crsr->dbVersion = MIN_POSSIBLE_DB_VERSION; 99 | 100 | return rc; 101 | } 102 | 103 | /** 104 | * Called to reclaim all of the resources allocated in `changesOpen` 105 | * once a query against the virtual table has completed. 106 | * 107 | * We, of course, do not de-allocated the `pTab` reference 108 | * given `pTab` must persist for the life of the connection. 109 | * 110 | * `pChangesStmt` and `pRowStmt` must be finalized. 111 | * 112 | * `colVrsns` does not need to be freed as it comes from 113 | * `pChangesStmt` thus finalizing `pChangesStmt` will 114 | * release `colVrsnsr` 115 | */ 116 | static int changesClose(sqlite3_vtab_cursor *cur) { 117 | crsql_Changes_cursor *pCur = (crsql_Changes_cursor *)cur; 118 | changesCrsrFinalize(pCur); 119 | sqlite3_free(pCur); 120 | return SQLITE_OK; 121 | } 122 | 123 | /** 124 | * Invoked to kick off the pulling of rows from the virtual table. 125 | * Provides the constraints with which the vtab can work with 126 | * to compute what rows to pull. 127 | * 128 | * Provided constraints are filled in by the changesBestIndex method. 129 | */ 130 | int crsql_changes_filter(sqlite3_vtab_cursor *pVtabCursor, int idxNum, 131 | const char *idxStr, int argc, sqlite3_value **argv); 132 | 133 | /* 134 | ** SQLite will invoke this method one or more times while planning a query 135 | ** that uses the virtual table. This routine needs to create 136 | ** a query plan for each invocation and compute an estimated cost for that 137 | ** plan. 138 | ** TODO: should we support `where table` filters? 139 | */ 140 | int crsql_changes_best_index(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo); 141 | 142 | int crsql_changes_update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv, 143 | sqlite3_int64 *pRowid); 144 | // If xBegin is not defined xCommit is not called. 145 | int crsql_changes_begin(sqlite3_vtab *pVTab); 146 | int crsql_changes_commit(sqlite3_vtab *pVTab); 147 | int crsql_changes_rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid); 148 | int crsql_changes_column( 149 | sqlite3_vtab_cursor *cur, /* The cursor */ 150 | sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ 151 | int i /* Which column to return */ 152 | ); 153 | int crsql_changes_eof(sqlite3_vtab_cursor *cur); 154 | 155 | sqlite3_module crsql_changesModule = { 156 | /* iVersion */ 0, 157 | /* xCreate */ 0, 158 | /* xConnect */ changesConnect, 159 | /* xBestIndex */ crsql_changes_best_index, 160 | /* xDisconnect */ changesDisconnect, 161 | /* xDestroy */ 0, 162 | /* xOpen */ changesOpen, 163 | /* xClose */ changesClose, 164 | /* xFilter */ crsql_changes_filter, 165 | /* xNext */ crsql_changes_next, 166 | /* xEof */ crsql_changes_eof, 167 | /* xColumn */ crsql_changes_column, 168 | /* xRowid */ crsql_changes_rowid, 169 | /* xUpdate */ crsql_changes_update, 170 | /* xBegin */ crsql_changes_begin, 171 | /* xSync */ 0, 172 | /* xCommit */ crsql_changes_commit, 173 | /* xRollback */ 0, 174 | /* xFindMethod */ 0, 175 | /* xRename */ 0, 176 | /* xSavepoint */ 0, 177 | /* xRelease */ 0, 178 | /* xRollbackTo */ 0, 179 | /* xShadowName */ 0 180 | #ifdef LIBSQL 181 | , 182 | /* xPreparedSql */ 0 183 | #endif 184 | }; 185 | -------------------------------------------------------------------------------- /core/src/changes-vtab.h: -------------------------------------------------------------------------------- 1 | /** 2 | * The changes virtual table is an eponymous virtual table which can be used 3 | * to fetch and apply patches to a db. 4 | * 5 | * To fetch a changeset: 6 | * ```sql 7 | * SELECT * FROM crsql_chages WHERE site_id IS NOT SITE_ID AND version > V 8 | * ``` 9 | * 10 | * The site id parameter is used to prevent a site from fetching its own 11 | * changes that were patched into the remote. 12 | * 13 | * The version parameter is used to get changes after a specific version. 14 | * Sites should keep track of the latest version they've received from other 15 | * sites and use that number as a cursor to fetch future changes. 16 | * 17 | * The changes table has the following columns: 18 | * 1. table - the name of the table the patch is from 19 | * 2. pk - the primary key(s) that identify the row to be patched. If the 20 | * table has many columns that comprise the primary key then 21 | * the values are quote concatenated in pk order. 22 | * 3. col_vals - the values to patch. quote concatenated in cid order. 23 | * 4. col_versions - the cids of the changed columns and the versions of those 24 | * columns 25 | * 5. version - the min version of the patch. Used for filtering and for sites 26 | * to update their "last seen" version from other sites 27 | * 6. site_id - the site_id that is responsible for the update. If this is 0 28 | * then the update was made locally. 29 | * 30 | * To apply a changeset: 31 | * ```sql 32 | * INSERT INTO changes (table, pk, col_vals, col_versions, site_id) VALUES 33 | * (...) 34 | * ``` 35 | */ 36 | #ifndef CHANGES_VTAB_H 37 | #define CHANGES_VTAB_H 38 | 39 | #if !defined(SQLITEINT_H) 40 | #include "sqlite3ext.h" 41 | #endif 42 | SQLITE_EXTENSION_INIT3 43 | 44 | #include 45 | 46 | #include "crsqlite.h" 47 | #include "ext-data.h" 48 | 49 | extern sqlite3_module crsql_changesModule; 50 | 51 | /** 52 | * Data maintained by the virtual table across 53 | * queries. 54 | * 55 | * Per-query data is kept on crsql_Changes_cursor 56 | */ 57 | typedef struct crsql_Changes_vtab crsql_Changes_vtab; 58 | struct crsql_Changes_vtab { 59 | sqlite3_vtab base; 60 | sqlite3 *db; 61 | 62 | crsql_ExtData *pExtData; 63 | }; 64 | 65 | /** 66 | * Cursor used to return patches. 67 | * This is instantiated per-query and updated 68 | * on each row being returned. 69 | * 70 | * Contains a reference to the vtab structure in order 71 | * get a handle on the db which to fetch from 72 | * the underlying crr tables. 73 | * 74 | * Most columns are passed-through from 75 | * `pChangesStmt` and `pRowStmt` which are stepped 76 | * in each call to `changesNext`. 77 | * 78 | * `colVersion` is copied given it is unclear 79 | * what the behavior is of calling `sqlite3_column_x` on 80 | * the same column multiple times with, potentially, 81 | * different types. 82 | * 83 | * `colVersions` is used in the implementation as 84 | * a text column in order to fetch the correct columns 85 | * from the physical row. 86 | * 87 | * Everything allocated here must be constructed in 88 | * changesOpen and released in changesCrsrFinalize 89 | */ 90 | #define ROW_TYPE_UPDATE 0 91 | #define ROW_TYPE_DELETE 1 92 | #define ROW_TYPE_PKONLY 2 93 | 94 | typedef struct crsql_Changes_cursor crsql_Changes_cursor; 95 | struct crsql_Changes_cursor { 96 | sqlite3_vtab_cursor base; 97 | 98 | crsql_Changes_vtab *pTab; 99 | 100 | sqlite3_stmt *pChangesStmt; 101 | sqlite3_stmt *pRowStmt; 102 | 103 | sqlite3_int64 dbVersion; 104 | int rowType; 105 | 106 | sqlite3_int64 changesRowid; 107 | int tblInfoIdx; 108 | }; 109 | 110 | #endif -------------------------------------------------------------------------------- /core/src/changes-vtab.test.c: -------------------------------------------------------------------------------- 1 | #include "changes-vtab.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "consts.h" 9 | #include "crsqlite.h" 10 | 11 | int crsql_close(sqlite3 *db); 12 | 13 | static void testManyPkTable() { 14 | printf("ManyPkTable\n"); 15 | 16 | sqlite3 *db; 17 | sqlite3_stmt *pStmt; 18 | int rc; 19 | rc = sqlite3_open(":memory:", &db); 20 | 21 | rc = sqlite3_exec( 22 | db, "CREATE TABLE foo (a not null, b not null, c, primary key (a, b));", 23 | 0, 0, 0); 24 | rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo');", 0, 0, 0); 25 | assert(rc == SQLITE_OK); 26 | rc += sqlite3_exec(db, "INSERT INTO foo VALUES (4,5,6);", 0, 0, 0); 27 | assert(rc == SQLITE_OK); 28 | 29 | rc += sqlite3_prepare_v2(db, "SELECT [table], quote(pk) FROM crsql_changes", 30 | -1, &pStmt, 0); 31 | assert(rc == SQLITE_OK); 32 | 33 | while (sqlite3_step(pStmt) == SQLITE_ROW) { 34 | const unsigned char *pk = sqlite3_column_text(pStmt, 1); 35 | // pk: 4, 5 36 | // X'0209040905' 37 | // 02 -> columns 38 | // 09 -> 1 byte integer 39 | // 04 -> 4 40 | // 09 -> 1 byte integer 41 | // 05 -> 5 42 | assert(strcmp("X'0209040905'", (char *)pk) == 0); 43 | } 44 | 45 | sqlite3_finalize(pStmt); 46 | crsql_close(db); 47 | printf("\t\e[0;32mSuccess\e[0m\n"); 48 | } 49 | 50 | static void assertCount(sqlite3 *db, const char *sql, int expected) { 51 | sqlite3_stmt *pStmt; 52 | int rc = sqlite3_prepare_v2(db, sql, -1, &pStmt, 0); 53 | assert(rc == SQLITE_OK); 54 | assert(sqlite3_step(pStmt) == SQLITE_ROW); 55 | printf("expected: %d, actual: %d\n", expected, sqlite3_column_int(pStmt, 0)); 56 | assert(sqlite3_column_int(pStmt, 0) == expected); 57 | sqlite3_finalize(pStmt); 58 | } 59 | 60 | static void testFilters() { 61 | printf("Filters\n"); 62 | 63 | sqlite3 *db; 64 | int rc; 65 | rc = sqlite3_open(":memory:", &db); 66 | 67 | rc = sqlite3_exec(db, "CREATE TABLE foo (a primary key not null, b);", 0, 0, 68 | 0); 69 | rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo');", 0, 0, 0); 70 | assert(rc == SQLITE_OK); 71 | rc += sqlite3_exec(db, "INSERT INTO foo VALUES (1,2);", 0, 0, 0); 72 | rc += sqlite3_exec(db, "INSERT INTO foo VALUES (2,3);", 0, 0, 0); 73 | rc += sqlite3_exec(db, "INSERT INTO foo VALUES (3,4);", 0, 0, 0); 74 | assert(rc == SQLITE_OK); 75 | 76 | printf("no filters\n"); 77 | // 6 - 1 for each row creation, 1 for each b 78 | assertCount(db, "SELECT count(*) FROM crsql_changes", 3); 79 | 80 | // now test: 81 | // 1. site_id comparison 82 | // 2. db_version comparison 83 | 84 | printf("is null\n"); 85 | assertCount(db, "SELECT count(*) FROM crsql_changes WHERE site_id IS NULL", 86 | 0); 87 | 88 | printf("is not null\n"); 89 | assertCount( 90 | db, "SELECT count(*) FROM crsql_changes WHERE site_id IS NOT NULL", 3); 91 | 92 | printf("equals\n"); 93 | assertCount( 94 | db, "SELECT count(*) FROM crsql_changes WHERE site_id = crsql_site_id()", 95 | 3); 96 | 97 | // 0 rows is actually correct ANSI sql behavior. NULLs are never equal, or not 98 | // equal, to anything in ANSI SQL. So users must use `IS NOT` to check rather 99 | // than !=. 100 | // 101 | // https://stackoverflow.com/questions/60017275/why-null-is-not-equal-to-anything-is-a-false-statement 102 | printf("not equals\n"); 103 | assertCount( 104 | db, "SELECT count(*) FROM crsql_changes WHERE site_id != crsql_site_id()", 105 | 0); 106 | 107 | printf("is not\n"); 108 | // All rows are currently equal to site_id since all rows are currently local 109 | // writes 110 | assertCount( 111 | db, "SELECT count(*) FROM crsql_changes WHERE site_id IS crsql_site_id()", 112 | 3); 113 | 114 | // compare on db_version _and_ site_id 115 | 116 | // compare upper and lower bound on db_version 117 | printf("double bounded version\n"); 118 | assertCount(db, 119 | "SELECT count(*) FROM crsql_changes WHERE db_version >= 1 AND " 120 | "db_version < 2", 121 | 1); 122 | 123 | printf("OR condition\n"); 124 | assertCount(db, 125 | "SELECT count(*) FROM crsql_changes WHERE db_version > 2 OR " 126 | "site_id IS crsql_site_id()", 127 | 3); 128 | 129 | // compare on pks, table name, other not perfectly supported columns 130 | 131 | crsql_close(db); 132 | printf("\t\e[0;32mSuccess\e[0m\n"); 133 | } 134 | 135 | // test value extraction under all filter conditions 136 | 137 | // static void testSinglePksTable() 138 | // { 139 | // } 140 | 141 | // static void testOnlyPkTable() 142 | // { 143 | // } 144 | 145 | // static void testSciNotation() 146 | // { 147 | // } 148 | 149 | // static void testHex() 150 | // { 151 | // } 152 | 153 | void crsqlChangesVtabTestSuite() { 154 | printf("\e[47m\e[1;30mSuite: crsql_changesVtab\e[0m\n"); 155 | testManyPkTable(); 156 | testFilters(); 157 | } 158 | -------------------------------------------------------------------------------- /core/src/consts.h: -------------------------------------------------------------------------------- 1 | #ifndef CRSQLITE_CONSTS_H 2 | #define CRSQLITE_CONSTS_H 3 | 4 | // db version is a signed 64bit int since sqlite doesn't support saving and 5 | // retrieving unsigned 64bit ints. (2^64 / 2) is a big enough number to write 1 6 | // million entries per second for 3,000 centuries. 7 | #define MIN_POSSIBLE_DB_VERSION 0L 8 | 9 | #define __CRSQL_CLOCK_LEN 13 10 | 11 | #define CRR_SPACE 0 12 | #define USER_SPACE 1 13 | #define ROWID_SLAB_SIZE 10000000000000 14 | 15 | #define CLOCK_TABLES_SELECT \ 16 | "SELECT tbl_name FROM sqlite_master WHERE type='table' AND tbl_name LIKE " \ 17 | "'%__crsql_clock'" 18 | 19 | #define SET_SYNC_BIT "SELECT crsql_internal_sync_bit(1)" 20 | #define CLEAR_SYNC_BIT "SELECT crsql_internal_sync_bit(0)" 21 | 22 | #define TBL_SITE_ID "site_id" 23 | #define TBL_DB_VERSION "db_version" 24 | #define TBL_SCHEMA "crsql_master" 25 | #define UNION_ALL "UNION ALL" 26 | 27 | #define MAX_TBL_NAME_LEN 2048 28 | #define SITE_ID_LEN 16 29 | 30 | // Version int: 31 | // M - major 32 | // m - minor 33 | // p - patch 34 | // b - build 35 | // MM.mm.pp.bb 36 | // 00 00 00 00 37 | // Given we can't prefix an int with 0s, read from right to left. 38 | // Rightmost is always `bb` 39 | #define CRSQLITE_VERSION 130000 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /core/src/core_init.c: -------------------------------------------------------------------------------- 1 | /* 2 | This file is appended to the end of a sqlite3.c amalgammation 3 | file to include crsqlite functions statically in 4 | a build. This is used for the demo CLI and WASM implementations. 5 | */ 6 | #include "ext.h" 7 | 8 | int core_init(const char *dummy) { 9 | return sqlite3_auto_extension((void *)sqlite3_crsqlite_init); 10 | } 11 | -------------------------------------------------------------------------------- /core/src/crsqlite.c: -------------------------------------------------------------------------------- 1 | #include "crsqlite.h" 2 | SQLITE_EXTENSION_INIT1 3 | #ifdef LIBSQL 4 | LIBSQL_EXTENSION_INIT1 5 | #endif 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "changes-vtab.h" 14 | #include "consts.h" 15 | #include "ext-data.h" 16 | #include "rust.h" 17 | 18 | // see 19 | // https://github.com/chromium/chromium/commit/579b3dd0ea41a40da8a61ab87a8b0bc39e158998 20 | // & https://github.com/rust-lang/rust/issues/73632 & 21 | // https://sourcegraph.com/github.com/chromium/chromium/-/commit/579b3dd0ea41a40da8a61ab87a8b0bc39e158998?visible=1 22 | #ifdef CRSQLITE_WASM 23 | unsigned char __rust_no_alloc_shim_is_unstable; 24 | #endif 25 | 26 | int crsql_compact_post_alter(sqlite3 *db, const char *tblName, 27 | crsql_ExtData *pExtData, char **errmsg); 28 | 29 | static void freeConnectionExtData(void *pUserData) { 30 | crsql_ExtData *pExtData = (crsql_ExtData *)pUserData; 31 | 32 | crsql_freeExtData(pExtData); 33 | } 34 | 35 | static int commitHook(void *pUserData) { 36 | crsql_ExtData *pExtData = (crsql_ExtData *)pUserData; 37 | 38 | pExtData->dbVersion = pExtData->pendingDbVersion; 39 | pExtData->pendingDbVersion = -1; 40 | pExtData->seq = 0; 41 | pExtData->updatedTableInfosThisTx = 0; 42 | return SQLITE_OK; 43 | } 44 | 45 | static void rollbackHook(void *pUserData) { 46 | crsql_ExtData *pExtData = (crsql_ExtData *)pUserData; 47 | 48 | pExtData->pendingDbVersion = -1; 49 | pExtData->seq = 0; 50 | pExtData->updatedTableInfosThisTx = 0; 51 | } 52 | 53 | #ifdef LIBSQL 54 | static void closeHook(void *pUserData, sqlite3 *db) { 55 | crsql_ExtData *pExtData = (crsql_ExtData *)pUserData; 56 | crsql_finalize(pExtData); 57 | } 58 | #endif 59 | 60 | void *sqlite3_crsqlrustbundle_init(sqlite3 *db, char **pzErrMsg, 61 | const sqlite3_api_routines *pApi); 62 | 63 | #ifdef _WIN32 64 | __declspec(dllexport) 65 | #endif 66 | int sqlite3_crsqlite_init(sqlite3 *db, char **pzErrMsg, 67 | const sqlite3_api_routines *pApi 68 | #ifdef LIBSQL 69 | , 70 | const libsql_api_routines *pLibsqlApi 71 | #endif 72 | ) { 73 | int rc = SQLITE_OK; 74 | 75 | SQLITE_EXTENSION_INIT2(pApi); 76 | #ifdef LIBSQL 77 | LIBSQL_EXTENSION_INIT2(pLibsqlApi); 78 | #endif 79 | 80 | // TODO: should be moved lower once we finish migrating to rust. 81 | // RN it is safe here since the rust bundle init is largely just reigstering 82 | // function pointers. we need to init the rust bundle otherwise sqlite api 83 | // methods are not isntalled when we start calling rust 84 | crsql_ExtData *pExtData = sqlite3_crsqlrustbundle_init(db, pzErrMsg, pApi); 85 | if (pExtData == 0) { 86 | return SQLITE_ERROR; 87 | } 88 | 89 | if (rc == SQLITE_OK) { 90 | rc = sqlite3_create_module_v2(db, "crsql_changes", &crsql_changesModule, 91 | pExtData, 0); 92 | } 93 | 94 | if (rc == SQLITE_OK) { 95 | #ifdef LIBSQL 96 | libsql_close_hook(db, closeHook, pExtData); 97 | #endif 98 | // TODO: get the prior callback so we can call it rather than replace 99 | // it? 100 | sqlite3_commit_hook(db, commitHook, pExtData); 101 | sqlite3_rollback_hook(db, rollbackHook, pExtData); 102 | } 103 | 104 | return rc; 105 | } -------------------------------------------------------------------------------- /core/src/crsqlite.h: -------------------------------------------------------------------------------- 1 | #ifndef CRSQLITE_H 2 | #define CRSQLITE_H 3 | 4 | #include "sqlite3ext.h" 5 | SQLITE_EXTENSION_INIT3 6 | 7 | #include 8 | 9 | #ifndef UNIT_TEST 10 | #define STATIC static 11 | #else 12 | #define STATIC 13 | #endif 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /core/src/ext-data.c: -------------------------------------------------------------------------------- 1 | #include "ext-data.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "consts.h" 7 | 8 | void crsql_clear_stmt_cache(crsql_ExtData *pExtData); 9 | void crsql_init_table_info_vec(crsql_ExtData *pExtData); 10 | void crsql_drop_table_info_vec(crsql_ExtData *pExtData); 11 | 12 | crsql_ExtData *crsql_newExtData(sqlite3 *db, unsigned char *siteIdBuffer) { 13 | crsql_ExtData *pExtData = sqlite3_malloc(sizeof *pExtData); 14 | 15 | pExtData->siteId = siteIdBuffer; 16 | 17 | pExtData->pPragmaSchemaVersionStmt = 0; 18 | int rc = sqlite3_prepare_v3(db, "PRAGMA schema_version", -1, 19 | SQLITE_PREPARE_PERSISTENT, 20 | &(pExtData->pPragmaSchemaVersionStmt), 0); 21 | pExtData->pPragmaDataVersionStmt = 0; 22 | rc += sqlite3_prepare_v3(db, "PRAGMA data_version", -1, 23 | SQLITE_PREPARE_PERSISTENT, 24 | &(pExtData->pPragmaDataVersionStmt), 0); 25 | pExtData->pSetSyncBitStmt = 0; 26 | rc += sqlite3_prepare_v3(db, SET_SYNC_BIT, -1, SQLITE_PREPARE_PERSISTENT, 27 | &(pExtData->pSetSyncBitStmt), 0); 28 | pExtData->pClearSyncBitStmt = 0; 29 | rc += sqlite3_prepare_v3(db, CLEAR_SYNC_BIT, -1, SQLITE_PREPARE_PERSISTENT, 30 | &(pExtData->pClearSyncBitStmt), 0); 31 | 32 | pExtData->pSetSiteIdOrdinalStmt = 0; 33 | rc += sqlite3_prepare_v3( 34 | db, "INSERT INTO crsql_site_id (site_id) VALUES (?) RETURNING ordinal", 35 | -1, SQLITE_PREPARE_PERSISTENT, &(pExtData->pSetSiteIdOrdinalStmt), 0); 36 | 37 | pExtData->pSelectSiteIdOrdinalStmt = 0; 38 | rc += sqlite3_prepare_v3( 39 | db, "SELECT ordinal FROM crsql_site_id WHERE site_id = ?", -1, 40 | SQLITE_PREPARE_PERSISTENT, &(pExtData->pSelectSiteIdOrdinalStmt), 0); 41 | 42 | pExtData->pSelectClockTablesStmt = 0; 43 | rc += 44 | sqlite3_prepare_v3(db, CLOCK_TABLES_SELECT, -1, SQLITE_PREPARE_PERSISTENT, 45 | &(pExtData->pSelectClockTablesStmt), 0); 46 | 47 | pExtData->dbVersion = -1; 48 | pExtData->pendingDbVersion = -1; 49 | pExtData->seq = 0; 50 | pExtData->pragmaSchemaVersion = -1; 51 | pExtData->pragmaDataVersion = -1; 52 | pExtData->pragmaSchemaVersionForTableInfos = -1; 53 | pExtData->pDbVersionStmt = 0; 54 | pExtData->tableInfos = 0; 55 | pExtData->rowsImpacted = 0; 56 | pExtData->updatedTableInfosThisTx = 0; 57 | crsql_init_table_info_vec(pExtData); 58 | 59 | sqlite3_stmt *pStmt; 60 | 61 | rc += sqlite3_prepare_v2(db, 62 | "SELECT ltrim(key, 'config.'), value FROM " 63 | "crsql_master WHERE key LIKE 'config.%';", 64 | -1, &pStmt, 0); 65 | 66 | if (rc != SQLITE_OK) { 67 | crsql_freeExtData(pExtData); 68 | return 0; 69 | } 70 | 71 | // set defaults! 72 | pExtData->mergeEqualValues = 0; 73 | 74 | while (sqlite3_step(pStmt) == SQLITE_ROW) { 75 | const unsigned char *name = sqlite3_column_text(pStmt, 0); 76 | int colType = sqlite3_column_type(pStmt, 1); 77 | 78 | if (strcmp("merge-equal-values", (char *)name) == 0) { 79 | if (colType == SQLITE_INTEGER) { 80 | const int value = sqlite3_column_int(pStmt, 1); 81 | pExtData->mergeEqualValues = value; 82 | } else { 83 | // broken setting... 84 | crsql_freeExtData(pExtData); 85 | return 0; 86 | } 87 | } else { 88 | // unhandled config setting 89 | } 90 | } 91 | 92 | sqlite3_finalize(pStmt); 93 | 94 | int pv = crsql_fetchPragmaDataVersion(db, pExtData); 95 | if (pv == -1 || rc != SQLITE_OK) { 96 | crsql_freeExtData(pExtData); 97 | return 0; 98 | } 99 | 100 | return pExtData; 101 | } 102 | 103 | void crsql_freeExtData(crsql_ExtData *pExtData) { 104 | sqlite3_free(pExtData->siteId); 105 | sqlite3_finalize(pExtData->pDbVersionStmt); 106 | sqlite3_finalize(pExtData->pPragmaSchemaVersionStmt); 107 | sqlite3_finalize(pExtData->pPragmaDataVersionStmt); 108 | sqlite3_finalize(pExtData->pSetSyncBitStmt); 109 | sqlite3_finalize(pExtData->pClearSyncBitStmt); 110 | sqlite3_finalize(pExtData->pSetSiteIdOrdinalStmt); 111 | sqlite3_finalize(pExtData->pSelectSiteIdOrdinalStmt); 112 | sqlite3_finalize(pExtData->pSelectClockTablesStmt); 113 | crsql_clear_stmt_cache(pExtData); 114 | crsql_drop_table_info_vec(pExtData); 115 | sqlite3_free(pExtData); 116 | } 117 | 118 | // Should _only_ be called when disconnecting from the db 119 | // for some reason finalization in extension unload methods doesn't 120 | // work as expected 121 | // see https://sqlite.org/forum/forumpost/c94f943821 122 | // `freeExtData` is called after finalization when the extension unloads 123 | void crsql_finalize(crsql_ExtData *pExtData) { 124 | sqlite3_finalize(pExtData->pDbVersionStmt); 125 | sqlite3_finalize(pExtData->pPragmaSchemaVersionStmt); 126 | sqlite3_finalize(pExtData->pPragmaDataVersionStmt); 127 | sqlite3_finalize(pExtData->pSetSyncBitStmt); 128 | sqlite3_finalize(pExtData->pClearSyncBitStmt); 129 | sqlite3_finalize(pExtData->pSetSiteIdOrdinalStmt); 130 | sqlite3_finalize(pExtData->pSelectSiteIdOrdinalStmt); 131 | sqlite3_finalize(pExtData->pSelectClockTablesStmt); 132 | crsql_clear_stmt_cache(pExtData); 133 | pExtData->pDbVersionStmt = 0; 134 | pExtData->pPragmaSchemaVersionStmt = 0; 135 | pExtData->pPragmaDataVersionStmt = 0; 136 | pExtData->pSetSyncBitStmt = 0; 137 | pExtData->pClearSyncBitStmt = 0; 138 | pExtData->pSetSiteIdOrdinalStmt = 0; 139 | pExtData->pSelectSiteIdOrdinalStmt = 0; 140 | pExtData->pSelectClockTablesStmt = 0; 141 | } 142 | 143 | #define DB_VERSION_SCHEMA_VERSION 0 144 | #define TABLE_INFO_SCHEMA_VERSION 1 145 | 146 | int crsql_fetchPragmaSchemaVersion(sqlite3 *db, crsql_ExtData *pExtData, 147 | int which) { 148 | int rc = sqlite3_step(pExtData->pPragmaSchemaVersionStmt); 149 | if (rc == SQLITE_ROW) { 150 | int version = sqlite3_column_int(pExtData->pPragmaSchemaVersionStmt, 0); 151 | sqlite3_reset(pExtData->pPragmaSchemaVersionStmt); 152 | if (which == DB_VERSION_SCHEMA_VERSION) { 153 | if (version > pExtData->pragmaSchemaVersion) { 154 | pExtData->pragmaSchemaVersion = version; 155 | return 1; 156 | } 157 | } else { 158 | if (version > pExtData->pragmaSchemaVersionForTableInfos) { 159 | pExtData->pragmaSchemaVersionForTableInfos = version; 160 | return 1; 161 | } 162 | } 163 | 164 | return 0; 165 | } else { 166 | sqlite3_reset(pExtData->pPragmaSchemaVersionStmt); 167 | } 168 | 169 | return -1; 170 | } 171 | 172 | int crsql_fetchPragmaDataVersion(sqlite3 *db, crsql_ExtData *pExtData) { 173 | int rc = sqlite3_step(pExtData->pPragmaDataVersionStmt); 174 | if (rc != SQLITE_ROW) { 175 | sqlite3_reset(pExtData->pPragmaDataVersionStmt); 176 | return -1; 177 | } 178 | 179 | int version = sqlite3_column_int(pExtData->pPragmaDataVersionStmt, 0); 180 | sqlite3_reset(pExtData->pPragmaDataVersionStmt); 181 | 182 | if (version != pExtData->pragmaDataVersion) { 183 | pExtData->pragmaDataVersion = version; 184 | return 1; 185 | } 186 | 187 | return 0; 188 | } 189 | -------------------------------------------------------------------------------- /core/src/ext-data.h: -------------------------------------------------------------------------------- 1 | #ifndef CRSQLITE_EXTDATA_H 2 | #define CRSQLITE_EXTDATA_H 3 | 4 | #include "sqlite3ext.h" 5 | SQLITE_EXTENSION_INIT3 6 | 7 | // NOTE: any changes here must be updated in `c.rs` until we've finished porting 8 | // to rust. 9 | typedef struct crsql_ExtData crsql_ExtData; 10 | struct crsql_ExtData { 11 | // perma statement -- used to check db schema version 12 | sqlite3_stmt *pPragmaSchemaVersionStmt; 13 | sqlite3_stmt *pPragmaDataVersionStmt; 14 | int pragmaDataVersion; 15 | 16 | // this gets set at the start of each transaction on the first invocation 17 | // to crsql_next_db_version() 18 | // and re-set on transaction commit or rollback. 19 | sqlite3_int64 dbVersion; 20 | // the version that the db will be set to at the end of the transaction 21 | // if that transaction were to commit at the time this value is checked. 22 | sqlite3_int64 pendingDbVersion; 23 | int pragmaSchemaVersion; 24 | int updatedTableInfosThisTx; 25 | 26 | // we need another schema version number that tracks when we checked it 27 | // for zpTableInfos. 28 | int pragmaSchemaVersionForTableInfos; 29 | 30 | unsigned char *siteId; 31 | sqlite3_stmt *pDbVersionStmt; 32 | void *tableInfos; 33 | 34 | // tracks the number of rows impacted by all inserts into crsql_changes in the 35 | // current transaction. This number is reset on transaction commit. 36 | int rowsImpacted; 37 | 38 | int seq; 39 | 40 | sqlite3_stmt *pSetSyncBitStmt; 41 | sqlite3_stmt *pClearSyncBitStmt; 42 | sqlite3_stmt *pSetSiteIdOrdinalStmt; 43 | sqlite3_stmt *pSelectSiteIdOrdinalStmt; 44 | sqlite3_stmt *pSelectClockTablesStmt; 45 | 46 | int mergeEqualValues; 47 | }; 48 | 49 | crsql_ExtData *crsql_newExtData(sqlite3 *db, unsigned char *siteIdBuffer); 50 | void crsql_freeExtData(crsql_ExtData *pExtData); 51 | int crsql_fetchPragmaSchemaVersion(sqlite3 *db, crsql_ExtData *pExtData, 52 | int which); 53 | int crsql_fetchPragmaDataVersion(sqlite3 *db, crsql_ExtData *pExtData); 54 | int crsql_recreate_db_version_stmt(sqlite3 *db, crsql_ExtData *pExtData); 55 | void crsql_finalize(crsql_ExtData *pExtData); 56 | 57 | #endif -------------------------------------------------------------------------------- /core/src/ext.h: -------------------------------------------------------------------------------- 1 | #ifndef CRSQLITE_H 2 | #define CRSQLITE_H 3 | 4 | /** 5 | * Extension initialization routine that is run once per connection. 6 | */ 7 | int sqlite3_crsqlite_init(sqlite3 *db, char **pzErrMsg, 8 | const sqlite3_api_routines *pApi); 9 | 10 | #endif -------------------------------------------------------------------------------- /core/src/fuzzer.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "sqlite3ext.h" 7 | SQLITE_EXTENSION_INIT3 8 | 9 | extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) 10 | { 11 | // Note: all fuzzing is done from Python in `test_sync_prop` 12 | return 0; 13 | } 14 | -------------------------------------------------------------------------------- /core/src/is-crr.test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "crsqlite.h" 5 | #include "rust.h" 6 | 7 | int crsql_close(sqlite3 *db); 8 | 9 | static void testTableIsNotCrr() { 10 | printf("TableIsNotCrr\n"); 11 | sqlite3 *db; 12 | int rc; 13 | rc = sqlite3_open(":memory:", &db); 14 | 15 | rc = 16 | sqlite3_exec(db, "CREATE TABLE foo (a PRIMARY KEY NOT NULL, b)", 0, 0, 0); 17 | assert(rc == SQLITE_OK); 18 | assert(crsql_is_crr(db, "foo") == 0); 19 | 20 | crsql_close(db); 21 | printf("\t\e[0;32mSuccess\e[0m\n"); 22 | } 23 | 24 | static void testCrrIsCrr() { 25 | printf("CrrIsCrr\n"); 26 | sqlite3 *db; 27 | int rc; 28 | rc = sqlite3_open(":memory:", &db); 29 | 30 | rc = 31 | sqlite3_exec(db, "CREATE TABLE foo (a PRIMARY KEY NOT NULL, b)", 0, 0, 0); 32 | assert(rc == SQLITE_OK); 33 | rc = sqlite3_exec(db, "SELECT crsql_as_crr('foo')", 0, 0, 0); 34 | assert(rc == SQLITE_OK); 35 | 36 | assert(crsql_is_crr(db, "foo") == 1); 37 | 38 | crsql_close(db); 39 | printf("\t\e[0;32mSuccess\e[0m\n"); 40 | } 41 | 42 | static void testDestroyedCrrIsNotCrr() { 43 | printf("DestroyedCrrIsNotCrr\n"); 44 | sqlite3 *db; 45 | int rc; 46 | rc = sqlite3_open(":memory:", &db); 47 | 48 | rc = 49 | sqlite3_exec(db, "CREATE TABLE foo (a PRIMARY KEY NOT NULL, b)", 0, 0, 0); 50 | assert(rc == SQLITE_OK); 51 | rc = sqlite3_exec(db, "SELECT crsql_as_crr('foo')", 0, 0, 0); 52 | assert(rc == SQLITE_OK); 53 | rc = sqlite3_exec(db, "SELECT crsql_as_table('foo')", 0, 0, 0); 54 | assert(rc == SQLITE_OK); 55 | assert(crsql_is_crr(db, "foo") == 0); 56 | 57 | crsql_close(db); 58 | printf("\t\e[0;32mSuccess\e[0m\n"); 59 | } 60 | 61 | void crsqlIsCrrTestSuite() { 62 | printf("\e[47m\e[1;30mSuite: is_crr\e[0m\n"); 63 | 64 | testTableIsNotCrr(); 65 | testCrrIsCrr(); 66 | testDestroyedCrrIsNotCrr(); 67 | } 68 | -------------------------------------------------------------------------------- /core/src/rust.h: -------------------------------------------------------------------------------- 1 | #ifndef CRSQLITE_RUST_H 2 | #define CRSQLITE_RUST_H 3 | 4 | #include "crsqlite.h" 5 | #include "ext-data.h" 6 | 7 | // Parts of CR-SQLite are written in Rust and parts are in C. 8 | // As we gradually convert more code to Rust, we'll have to expose 9 | // structures to the old C-code that hasn't been converted yet. 10 | // These are those definitions. 11 | 12 | int crsql_is_crr(sqlite3 *db, const char *tblName); 13 | 14 | int crsql_init_site_id(sqlite3 *db, unsigned char *ret); 15 | int crsql_init_peer_tracking_table(sqlite3 *db); 16 | int crsql_create_schema_table_if_not_exists(sqlite3 *db); 17 | int crsql_maybe_update_db(sqlite3 *db, char **pzErrMsg); 18 | int crsql_is_table_compatible(sqlite3 *db, const char *tblName, char **err); 19 | int crsql_create_crr(sqlite3 *db, const char *schemaName, const char *tblName, 20 | int isCommitAlter, int noTx, char **err); 21 | int crsql_ensure_table_infos_are_up_to_date(sqlite3 *db, 22 | crsql_ExtData *pExtData, 23 | char **err); 24 | int crsql_fill_db_version_if_needed(sqlite3 *db, crsql_ExtData *pExtData, 25 | char **errmsg); 26 | sqlite_int64 crsql_next_db_version(sqlite3 *db, crsql_ExtData *pExtData, 27 | sqlite3_int64 mergingVersion, char **errmsg); 28 | 29 | void crsql_after_update(sqlite3_context *context, int argc, 30 | sqlite3_value **argv); 31 | void crsql_after_insert(sqlite3_context *context, int argc, 32 | sqlite3_value **argv); 33 | void crsql_after_delete(sqlite3_context *context, int argc, 34 | sqlite3_value **argv); 35 | #endif 36 | -------------------------------------------------------------------------------- /core/src/sandbox.test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "crsqlite.h" 5 | #include "rust.h" 6 | 7 | int crsql_close(sqlite3 *db); 8 | int syncLeftToRight(sqlite3 *db1, sqlite3 *db2, sqlite3_int64 since); 9 | 10 | static void testSandbox() { 11 | printf("Sandbox\n"); 12 | sqlite3 *db1; 13 | sqlite3 *db2; 14 | int rc; 15 | rc = sqlite3_open(":memory:", &db1); 16 | rc += sqlite3_open(":memory:", &db2); 17 | 18 | rc += 19 | sqlite3_exec(db1, "CREATE TABLE foo (a primary key not null);", 0, 0, 0); 20 | rc += 21 | sqlite3_exec(db2, "CREATE TABLE foo (a primary key not null);", 0, 0, 0); 22 | rc += sqlite3_exec(db1, "SELECT crsql_as_crr('foo')", 0, 0, 0); 23 | rc += sqlite3_exec(db2, "SELECT crsql_as_crr('foo')", 0, 0, 0); 24 | rc += sqlite3_exec(db1, "INSERT INTO foo VALUES (1)", 0, 0, 0); 25 | assert(rc == SQLITE_OK); 26 | 27 | assert(syncLeftToRight(db1, db2, 0) == SQLITE_OK); 28 | 29 | crsql_close(db1); 30 | crsql_close(db2); 31 | printf("\t\e[0;32mSuccess\e[0m\n"); 32 | } 33 | 34 | void crsqlSandboxSuite() { 35 | testSandbox(); 36 | printf("\e[47m\e[1;30mSuite: sandbox\e[0m\n"); 37 | } 38 | -------------------------------------------------------------------------------- /core/src/sqlite/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vlcn-io/cr-sqlite/891fe9e0190dd20917f807d739c809e1bc32f6a3/core/src/sqlite/Makefile -------------------------------------------------------------------------------- /core/src/tests.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "sqlite3ext.h" 5 | SQLITE_EXTENSION_INIT3 6 | 7 | #define SUITE(N) if (strcmp(suite, "all") == 0 || strcmp(suite, N) == 0) 8 | 9 | int crsql_close(sqlite3 *db) { 10 | int rc = SQLITE_OK; 11 | rc += sqlite3_exec(db, "SELECT crsql_finalize()", 0, 0, 0); 12 | rc += sqlite3_close(db); 13 | return rc; 14 | } 15 | 16 | // void crsqlTableInfoTestSuite(); 17 | void crsqlTestSuite(); 18 | // void crsqlTriggersTestSuite(); 19 | // void crsqlChangesVtabReadTestSuite(); 20 | void crsqlChangesVtabTestSuite(); 21 | void crsqlChangesVtabCommonTestSuite(); 22 | void crsqlExtDataTestSuite(); 23 | void crsqlFractSuite(); 24 | void crsqlIsCrrTestSuite(); 25 | void rowsImpactedTestSuite(); 26 | void crsqlChangesVtabRowidTestSuite(); 27 | void crsqlSandboxSuite(); 28 | void crsql_integration_check(); 29 | 30 | int main(int argc, char *argv[]) { 31 | char *suite = "all"; 32 | if (argc == 2) { 33 | suite = argv[1]; 34 | } 35 | 36 | SUITE("vtab") crsqlChangesVtabTestSuite(); 37 | SUITE("extdata") crsqlExtDataTestSuite(); 38 | // integration tests should come at the end given fixing unit tests will 39 | // likely fix integration tests 40 | SUITE("crsql") crsqlTestSuite(); 41 | SUITE("fract") crsqlFractSuite(); 42 | SUITE("is_crr") crsqlIsCrrTestSuite(); 43 | SUITE("rows_impacted") rowsImpactedTestSuite(); 44 | SUITE("rowid") crsqlChangesVtabRowidTestSuite(); 45 | SUITE("sandbox") crsqlSandboxSuite(); 46 | SUITE("rust_integration") crsql_integration_check(); 47 | 48 | sqlite3_shutdown(); 49 | } 50 | -------------------------------------------------------------------------------- /core/src/util.h: -------------------------------------------------------------------------------- 1 | #ifndef CRSQLITE_UTIL 2 | #define CRSQLITE_UTIL 3 | 4 | #include 5 | #include 6 | 7 | #include "crsqlite.h" 8 | 9 | #endif -------------------------------------------------------------------------------- /libsql-sync.sh: -------------------------------------------------------------------------------- 1 | rsync -vhra ./core/ ../libsql/libsql-sqlite3/ext/crr/ \ 2 | --include='**.gitignore' \ 3 | --exclude='**.git' \ 4 | --exclude='shell.c' \ 5 | --exclude='sqlite3.c' \ 6 | --exclude='sqlite' \ 7 | --exclude='sqlite3.h' \ 8 | --exclude='sqlite3ext.h' \ 9 | --filter=':- .gitignore' \ 10 | --delete-after 11 | -------------------------------------------------------------------------------- /notes.md: -------------------------------------------------------------------------------- 1 | # back-to-cid 2 | 3 | - cid from `pragma_table_info` 4 | - send `cid` over the wire too? 5 | - maybe in future update 6 | - sentinel becomes actual -1? or empty string or some such? Neg numbers are large varints 7 | - begin_alter needs to save off cid mappings in temp table so we can detect dropped columns 8 | - commit_alter would need to find where names no longer exist and remove them from clock tables based on their `old cid` mapping 9 | - commit_alter would need to re-number entries in clock tables whose `name via old cid` matches `name via new cid` but where `old cid != new cid` 10 | 11 | # db-version 12 | 13 | db version as col version for tx preservation on merge. 14 | 15 | Each `col_version` is currently incremented independently. We can instead set it to the current `db_version` to ensure that all values set in the same transaction can also all win together when merging. 16 | 17 | This isn't guaranteed since the peer being merged into could be way ahead in db_version overall but have some records behind in db_version. 18 | 19 | # pk-lookup 20 | 21 | - New table 22 | 23 | ```sql 24 | CREATE TABLE foo__crsql_pks (num INTEGER PRIMARY KEY, ...pks); 25 | CREATE UNIQUE INDEX ON foo__crsql_pks (...pks); 26 | ``` 27 | 28 | Merges... We still send actual PKs over the wire. Each host has its own nums. 29 | 30 | Merge: 31 | 32 | 1. Lookup num 33 | -- num missing means we have no record can do some short-circuits here 34 | 2. Do clock table stuff with num 35 | 36 | Pull changes: 37 | 38 | 1. Join pks via num 39 | 40 | # next db version optimization 41 | 42 | We currently nuke this on commit. 43 | 44 | We can keep a variable in ext data to represent it and only nuke / refresh it if the data change bit is set. 45 | 46 | The variable needs to be set on merge 47 | 48 | ```ts 49 | crsql_next_db_version(arg?) 50 | 51 | // arg is optional. If present, we set the `pending next db version` 52 | function crsql_next_db_version(arg?) { 53 | const ret = max(crsql_db_version() + 1, pExtData.pendingDbVersion, arg); 54 | pExtData.pendingDbVersion = ret; 55 | return ret; 56 | } 57 | ``` 58 | 59 | On commit, pending becomes actual. 60 | 61 | # Trigger opt 62 | 63 | - function to `insert and save` a lookaside in `insert_trigger` 64 | - function to `get and save` a lookaside in `update_trigger` 65 | - replace these lookups: `(SELECT __crsql_key FROM \"{table_name}__crsql_pks\" WHERE {pk_where_list})` 66 | - 67 | 68 | - Test changing pks in update creates key lookaside correctly. 69 | 70 | --- 71 | 72 | Assuming we re-write to use a function... 73 | 74 | ```ts 75 | function after_update(table, cols_new, cols_old) { 76 | // cols_new and cols_old are _in order_ as according to table_info 77 | 78 | // 1. Lookup the old record with `cols_old` 79 | // 2. Do the pk delete stuff if there exists a record with old 80 | // 3. If any pk is different, record a create record (sqlite value compare) 81 | // 4. For each non_pk, record the clock metadata 82 | } 83 | ``` 84 | -------------------------------------------------------------------------------- /py/README.md: -------------------------------------------------------------------------------- 1 | # crsql/py 2 | 3 | Integration and performance tests. 4 | -------------------------------------------------------------------------------- /py/correctness/.gitignore: -------------------------------------------------------------------------------- 1 | env/ 2 | .hypothesis/ 3 | -------------------------------------------------------------------------------- /py/correctness/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # @vlcn.io/py-correctness 2 | 3 | ## 0.13.0-next.0 4 | 5 | ### Minor Changes 6 | 7 | - re-insertion, api naming consistencies, metadata size reduction, websocket server, websocket client, websocket demo 8 | 9 | ## 0.12.0 10 | 11 | ### Minor Changes 12 | 13 | - 68deb1c: binary encoded primary keys, no string encoding on values, cache prepared statements on merge, fix webkit JIT crash 14 | 15 | ## 0.12.0-next.0 16 | 17 | ### Minor Changes 18 | 19 | - binary encoded primary keys, no string encoding on values, cache prepared statements on merge, fix webkit JIT crash 20 | 21 | ## 0.11.0 22 | 23 | ### Minor Changes 24 | 25 | - 62912ad: split up large transactions, compact out unneeded delete records, coordinate dedicated workers for android, null merge fix 26 | 27 | ## 0.11.0-next.0 28 | 29 | ### Minor Changes 30 | 31 | - split up large transactions, compact out unneeded delete records, coordinate dedicated workers for android, null merge fix 32 | 33 | ## 0.10.0 34 | 35 | ### Minor Changes 36 | 37 | - 7885afd: 50x perf boost when pulling changesets 38 | 39 | ## 0.10.0-next.0 40 | 41 | ### Minor Changes 42 | 43 | - 15c8e04: 50x perf boost when pulling changesets 44 | 45 | ## 0.9.0 46 | 47 | ### Minor Changes 48 | 49 | - automigrate fixes for WASM, react fixes for referential equality, direct-connect networking implementations, sync in shared worker, dbProvider hooks for React 50 | 51 | ## 0.8.1 52 | 53 | ### Patch Changes 54 | 55 | - fts5, sqlite 3.42.1, direct-connect packages 56 | 57 | ## 0.8.0 58 | 59 | ### Minor Changes 60 | 61 | - e0de95c: ANSI SQL compliance for crsql_changes, all filters available for crsql_changes, removal of tracked_peers, simplified crsql_master table 62 | 63 | ### Patch Changes 64 | 65 | - 9b483aa: npm is not updating on package publish -- bump versions to try to force it 66 | 67 | ## 0.8.0-next.1 68 | 69 | ### Patch Changes 70 | 71 | - npm is not updating on package publish -- bump versions to try to force it 72 | 73 | ## 0.8.0-next.0 74 | 75 | ### Minor Changes 76 | 77 | - ANSI SQL compliance for crsql_changes, all filters available for crsql_changes, removal of tracked_peers, simplified crsql_master table 78 | 79 | ## 0.7.2 80 | 81 | ### Patch Changes 82 | 83 | - e5919ae: fix xcommit deadlock, bump versions on dependencies 84 | 85 | ## 0.7.2-next.0 86 | 87 | ### Patch Changes 88 | 89 | - fix xcommit deadlock, bump versions on dependencies 90 | 91 | ## 0.7.1 92 | 93 | ### Patch Changes 94 | 95 | - aad733d: -- 96 | 97 | ## 0.7.1-next.0 98 | 99 | ### Patch Changes 100 | 101 | --- 102 | 103 | ## 0.7.0 104 | 105 | ### Minor Changes 106 | 107 | - 6316ec315: update to support prebuild binaries, include primary key only table fixes 108 | 109 | ## 0.7.0-next.0 110 | 111 | ### Minor Changes 112 | 113 | - update to support prebuild binaries, include primary key only table fixes 114 | 115 | ## 0.6.2 116 | 117 | ### Patch Changes 118 | 119 | - 3d09cd595: preview all the hook improvements and multi db open fixes 120 | - 567d8acba: auto-release prepared statements 121 | - 54666261b: fractional indexing inclusion 122 | - fractional indexing, better react hooks, many dbs opened concurrently 123 | 124 | ## 0.6.2-next.2 125 | 126 | ### Patch Changes 127 | 128 | - preview all the hook improvements and multi db open fixes 129 | 130 | ## 0.6.2-next.1 131 | 132 | ### Patch Changes 133 | 134 | - auto-release prepared statements 135 | 136 | ## 0.6.2-next.0 137 | 138 | ### Patch Changes 139 | 140 | - fractional indexing inclusion 141 | 142 | ## 0.6.1 143 | 144 | ### Patch Changes 145 | 146 | - 519bcfc2a: hooks, fixes to support examples, auto-determine tables queried 147 | - hooks package, used_tables query, web only target for wa-sqlite 148 | 149 | ## 0.6.1-next.0 150 | 151 | ### Patch Changes 152 | 153 | - hooks, fixes to support examples, auto-determine tables queried 154 | 155 | ## 0.6.0 156 | 157 | ### Minor Changes 158 | 159 | - seen peers, binary encoding for network layer, retry on disconnect for server, auto-track peers 160 | 161 | ## 0.5.3 162 | 163 | ### Patch Changes 164 | 165 | - deploy table validation fix 166 | 167 | ## 0.5.2 168 | 169 | ### Patch Changes 170 | 171 | - cid winner selection bugfix 172 | 173 | ## 0.5.1 174 | 175 | ### Patch Changes 176 | 177 | - rebuild all 178 | 179 | ## 0.5.0 180 | 181 | ### Minor Changes 182 | 183 | - breaking change -- fix version recording problem that prevented convergence in p2p cases 184 | 185 | ## 0.4.1 186 | 187 | ### Patch Changes 188 | 189 | - fix mem leak and cid win value selection bug 190 | 191 | ## 0.4.0 192 | 193 | ### Minor Changes 194 | 195 | - fix tie breaking for merge, add example client-server sync 196 | 197 | ## 0.3.1 198 | 199 | ### Patch Changes 200 | 201 | - fix bigint overflow in wasm, fix site_id not being returned with changesets 202 | 203 | ## 0.3.0 204 | 205 | ### Minor Changes 206 | 207 | - fix multi-way merge 208 | 209 | ## 0.2.0 210 | 211 | ### Minor Changes 212 | 213 | - incorporate schema fitness checks 214 | 215 | ## 0.1.0 216 | 217 | ### Minor Changes 218 | 219 | - update to use `wa-sqlite`, fix site id forwarding, fix scientific notation replication, etc. 220 | 221 | ## 0.0.1 222 | 223 | ### Patch Changes 224 | 225 | - fix linking issues on linux distros 226 | -------------------------------------------------------------------------------- /py/correctness/install-and-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # source env/bin/activate 4 | python3 -m pip install -r requirements.txt 5 | python3 -m pytest tests -------------------------------------------------------------------------------- /py/correctness/notes.md: -------------------------------------------------------------------------------- 1 | source env/bin/activate python3 -m pip install -r requirements.txt python -m pytest tests deactivate 2 | 3 | saving reqs: python -m pip freeze > requirements.txt 4 | 5 | -e git+ssh://git@github.com/vlcn-io/cr-sqlite.git@26ed5d8adcacbe1624650bc0c3872bdd944747e1#egg=crsql_correctness&subdirectory=py/correctness 6 | -------------------------------------------------------------------------------- /py/correctness/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@vlcn.io/py-correctness", 3 | "version": "0.13.0-next.0", 4 | "private": true, 5 | "description": "CR-SQLite loadable extension", 6 | "homepage": "https://vlcn.io", 7 | "repository": { 8 | "type": "git", 9 | "url": "git://github.com/vlcn-io/cr-sqlite" 10 | }, 11 | "scripts": { 12 | "test": "export PYTHONPATH=./src && pytest" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /py/correctness/prior-dbs/v0.12.0.prior-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vlcn-io/cr-sqlite/891fe9e0190dd20917f807d739c809e1bc32f6a3/py/correctness/prior-dbs/v0.12.0.prior-db -------------------------------------------------------------------------------- /py/correctness/prior-dbs/v0.13.0.prior-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vlcn-io/cr-sqlite/891fe9e0190dd20917f807d739c809e1bc32f6a3/py/correctness/prior-dbs/v0.13.0.prior-db -------------------------------------------------------------------------------- /py/correctness/prior-dbs/v0.15.0.prior-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vlcn-io/cr-sqlite/891fe9e0190dd20917f807d739c809e1bc32f6a3/py/correctness/prior-dbs/v0.15.0.prior-db -------------------------------------------------------------------------------- /py/correctness/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "crsql_correctness" 3 | version = "0.0.1" 4 | 5 | [tool.pytest.ini_options] 6 | addopts = [ 7 | "--import-mode=importlib", 8 | ] 9 | -------------------------------------------------------------------------------- /py/correctness/requirements.txt: -------------------------------------------------------------------------------- 1 | attrs==23.1.0 2 | -e . 3 | hypothesis==6.75.9 4 | iniconfig==2.0.0 5 | packaging==23.1 6 | pluggy==1.0.0 7 | pytest==7.3.1 8 | sortedcontainers==2.4.0 9 | -------------------------------------------------------------------------------- /py/correctness/spec.md: -------------------------------------------------------------------------------- 1 | # Correctness 2 | 3 | ## DB Version 4 | 5 | 1. [x] Min int on first db creation 6 | 2. [x] Increments with every modification to a crr datum 7 | 3. [x] Restored from disk on db load 8 | 4. [x] Unique for each transaction 9 | 10 | ## Site id 11 | 12 | 1. [x] Initialized to a uuid at startup 13 | 2. [x] Persisted 14 | 3. [x] Loaded from disk if exists on disk 15 | 4. [x] Does not ever change after being set, even between restarts 16 | 17 | ## Schema modification 18 | 19 | 1. [ ] create table 20 | 1. [ ] if not exists support 21 | 2. [ ] temp table support 22 | 3. [x] quoted identifiers 23 | 4. [x] no primary keys table 24 | 5. [x] compound primary key table 25 | 6. [x] single primary key 26 | 2. [ ] create index 27 | 1. [ ] create unique index not allowed 28 | 3. [ ] drop index 29 | 4. [ ] drop table 30 | 5. [ ] alter table 31 | 6. [ ] table constraints 32 | 1. [ ] fk constraints not allowed 33 | 7. [ ] `crr_from` is idempotent 34 | 35 | ## Inserts of new rows 36 | 37 | 1. [x] version cols start at 0 38 | 2. [x] cl starts at 1 39 | 3. [x] db version incremented 40 | 4. [x] clock record written with new db version and current site id for current row 41 | 5. [ ] ~~db version is not in use on any other row~~ 42 | 6. [x] cols have the inserted values 43 | 7. [x] update src is 0 44 | 45 | ## Updates of rows 46 | 47 | 1. [ ] version cols for changed rows increment by 1 48 | 2. [ ] version cols for unchanges rows do not change 49 | 3. [ ] db version is incremented 50 | 4. [ ] clock record for this row records new db version that is greater than last recorded db version 51 | 5. [ ] db version for the row is globally unique 52 | 6. [ ] local updates are always taken -- no conflict resolution required 53 | 7. [ ] update src is 0 54 | 55 | ## Deletes of rows 56 | 57 | 1. [ ] db version is incremented 58 | 2. [ ] clock record for this row records new db version that is greater than last recorded db version 59 | 3. [ ] db version for the row is globally unique 60 | 4. [ ] local deletes are always taken -- no conflict resolution required 61 | 5. [ ] if causal length was odd, it is incremented 62 | 6. [ ] if causal length was even, it is unchanged 63 | 7. [ ] version columns are unchanged 64 | 8. [ ] value columns are unchanged 65 | 9. [ ] update src is 0 66 | 67 | ## Inserts of existing rows 68 | 69 | 1. [ ] if causal length was odd, it is unchanged 70 | 2. [ ] if causal length was even, it is incremented 71 | 3. [ ] only cols referenced in insert are changed 72 | 4. [ ] version cols are incremented for changed cols 73 | 5. [ ] version cols are unchanged for unchanged cols 74 | 6. [ ] clock record for this row records new db version that is greater than last recorded db version 75 | 7. [ ] db version for the row is globally unique 76 | 8. [ ] update src is 0 77 | 78 | ## Reads 79 | 80 | 1. [ ] deleted rows (even cl) are not returned 81 | 2. [ ] undeleted (odd cl) rows are returned 82 | 3. [ ] version cols are not returned 83 | 4. [ ] cl is not returned 84 | 5. [ ] update src is note returned 85 | 86 | ## Merging remote changes 87 | 88 | 1. [ ] merges against a row are idempotent 89 | 1. [ ] merging an old row (by vclock) does not change the new row 90 | 2. [ ] merging a row with an identical copy of itself does not change the row 91 | 3. [ ] reapplications of a merge, after the first, does not impact the state of the row 92 | 2. [ ] update src is set to 1 93 | 3. [ ] only columns with higher versions are taken 94 | 4. [ ] if versions match for a column, the greater value is taken 95 | 5. [ ] physical deletion is final 96 | 97 | ## Sync Bit 98 | 1. [ ] no replication on changes from sync 99 | 100 | ## Computing deltas against remote clock 101 | 102 | ## Concurrency 103 | 104 | 105 | ## Primary key only tables 106 | -------------------------------------------------------------------------------- /py/correctness/src/crsql_correctness.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.1 2 | Name: crsql-correctness 3 | Version: 0.0.1 4 | -------------------------------------------------------------------------------- /py/correctness/src/crsql_correctness.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | pyproject.toml 2 | src/crsql_correctness/__init__.py 3 | src/crsql_correctness.egg-info/PKG-INFO 4 | src/crsql_correctness.egg-info/SOURCES.txt 5 | src/crsql_correctness.egg-info/dependency_links.txt 6 | src/crsql_correctness.egg-info/top_level.txt 7 | tests/test_as_ordered.py 8 | tests/test_cl_merging.py 9 | tests/test_cl_triggers.py 10 | tests/test_commit_alter_perf.py 11 | tests/test_crsql_changes_filters.py 12 | tests/test_dbversion.py 13 | tests/test_insert_new_rows.py 14 | tests/test_lookaside_key_creation.py 15 | tests/test_prior_versions.py 16 | tests/test_sandbox.py 17 | tests/test_schema_modification.py 18 | tests/test_sentinel_omission.py 19 | tests/test_seq.py 20 | tests/test_site_id_lookaside.py 21 | tests/test_siteid.py 22 | tests/test_sync.py 23 | tests/test_sync_bit.py 24 | tests/test_sync_prop.py 25 | tests/test_update_rows.py -------------------------------------------------------------------------------- /py/correctness/src/crsql_correctness.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /py/correctness/src/crsql_correctness.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | crsql_correctness 2 | -------------------------------------------------------------------------------- /py/correctness/src/crsql_correctness/__init__.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | 3 | extension = '../../core/dist/crsqlite' 4 | 5 | 6 | def connect(db_file, uri=False): 7 | c = sqlite3.connect(db_file, uri=uri) 8 | c.enable_load_extension(True) 9 | c.load_extension(extension) 10 | return c 11 | 12 | 13 | def close(c): 14 | c.execute("select crsql_finalize()") 15 | c.close() 16 | 17 | 18 | def get_site_id(c): 19 | return c.execute("SELECT crsql_site_id()").fetchone()[0] 20 | 21 | 22 | min_db_v = 0 23 | -------------------------------------------------------------------------------- /py/correctness/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # source env/bin/activate 4 | # python -m pytest tests -s -k test_cl_merging 5 | python3 -m pytest tests -s 6 | -------------------------------------------------------------------------------- /py/correctness/tests/test_as_ordered.py: -------------------------------------------------------------------------------- 1 | from crsql_correctness import connect, close, min_db_v 2 | from pprint import pprint 3 | 4 | 5 | def make_simple_schema(): 6 | c = connect(":memory:") 7 | # O... so the type must be `ANY` for the orderable column for our triggers and the like to work... 8 | # we could cast the whatever to an int... 9 | c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, spot TEXT, list)") 10 | c.execute("SELECT crsql_as_crr('foo')") 11 | c.execute("SELECT crsql_fract_as_ordered('foo', 'spot', 'list')") 12 | c.commit() 13 | return c 14 | 15 | 16 | def test_first_insertion_prepend(): 17 | c = make_simple_schema() 18 | c.execute("INSERT INTO foo VALUES (1, -1, 'a')") 19 | c.commit() 20 | 21 | rows = c.execute("SELECT * FROM foo").fetchall() 22 | assert (rows == [(1, 'a ', 'a')]) 23 | 24 | 25 | def test_first_insertion_append(): 26 | c = make_simple_schema() 27 | c.execute("INSERT INTO foo VALUES (1, 1, 'a')") 28 | c.commit() 29 | 30 | rows = c.execute("SELECT * FROM foo").fetchall() 31 | assert (rows == [(1, 'a ', 'a')]) 32 | 33 | 34 | def test_middle_insertion(): 35 | c = make_simple_schema() 36 | c.execute("INSERT INTO foo VALUES (1, -1, 'list')") 37 | c.execute("INSERT INTO foo VALUES (3, 1, 'list')") 38 | c.execute("INSERT INTO foo_fractindex (a, list, after_a) VALUES (2, 'list', 1)") 39 | c.commit() 40 | 41 | rows = c.execute("SELECT * FROM foo ORDER BY spot ASC").fetchall() 42 | assert (rows == [(1, 'a ', 'list'), (2, 'a P', 'list'), (3, 'a!', 'list')]) 43 | 44 | 45 | def test_front_insertion(): 46 | c = make_simple_schema() 47 | c.execute("INSERT INTO foo VALUES (2, -1, 'list')") 48 | c.execute("INSERT INTO foo VALUES (3, 1, 'list')") 49 | c.execute( 50 | "INSERT INTO foo_fractindex (a, list, after_a) VALUES (1, 'list', NULL)") 51 | c.commit() 52 | 53 | rows = c.execute("SELECT * FROM foo ORDER BY spot ASC").fetchall() 54 | assert ([(1, 'Z~', 'list'), (2, 'a ', 'list'), (3, 'a!', 'list')] == rows) 55 | 56 | 57 | def test_endinsertion(): 58 | c = make_simple_schema() 59 | c.execute("INSERT INTO foo VALUES (1, -1, 'list')") 60 | c.execute("INSERT INTO foo VALUES (2, 1, 'list')") 61 | c.execute( 62 | "INSERT INTO foo_fractindex (a, list, after_a) VALUES (3, 'list', 2)") 63 | c.commit() 64 | 65 | rows = c.execute("SELECT * FROM foo ORDER BY spot ASC").fetchall() 66 | assert (rows == [(1, 'a ', 'list'), (2, 'a!', 'list'), (3, 'a"', 'list')]) 67 | 68 | 69 | def test_view_first_insertion(): 70 | c = make_simple_schema() 71 | c.execute( 72 | "INSERT INTO foo_fractindex (a, list, after_a) VALUES (1, 'list', NULL)") 73 | c.commit() 74 | 75 | rows = c.execute("SELECT * FROM foo").fetchall() 76 | assert (rows == [(1, 'a ', 'list')]) 77 | 78 | 79 | def test_view_move(): 80 | c = make_simple_schema() 81 | c.execute("INSERT INTO foo VALUES (1, 1, 'list')") 82 | c.execute("INSERT INTO foo VALUES (2, 1, 'list')") 83 | c.execute("INSERT INTO foo VALUES (3, 1, 'list')") 84 | c.commit() 85 | 86 | # move 3 to be after 1 87 | c.execute("UPDATE foo_fractindex SET after_a = 1 WHERE a = 3") 88 | c.commit() 89 | rows = c.execute("SELECT * FROM foo ORDER BY spot ASC").fetchall() 90 | assert (rows == [(1, 'a ', 'list'), (3, 'a P', 'list'), (2, 'a!', 'list')]) 91 | -------------------------------------------------------------------------------- /py/correctness/tests/test_commit_alter_perf.py: -------------------------------------------------------------------------------- 1 | from crsql_correctness import connect, close, min_db_v 2 | from pprint import pprint 3 | import pytest 4 | import time 5 | 6 | def test_commit_alter_perf(): 7 | c = connect(":memory:") 8 | c.execute("CREATE TABLE issue (id INTEGER PRIMARY KEY NOT NULL, title TEXT, owner TEXT, status INTEGER, priority INTEGER)") 9 | c.execute("SELECT crsql_as_crr('issue')") 10 | c.commit() 11 | 12 | start_time = time.time() 13 | for i in range(10_000): 14 | c.execute("INSERT INTO issue (title, owner, status, priority) VALUES ('title', 'owner', 1, 1)") 15 | c.commit() 16 | end_time = time.time() 17 | print(f"insert time: {end_time - start_time}") 18 | 19 | start_time = time.time() 20 | c.execute("SELECT crsql_begin_alter('issue')") 21 | c.execute("SELECT crsql_commit_alter('issue')") 22 | end_time = time.time() 23 | print(f"no alter alter time: {end_time - start_time}") 24 | 25 | start_time = time.time() 26 | c.execute("SELECT crsql_begin_alter('issue')") 27 | c.execute("ALTER TABLE issue ADD COLUMN description TEXT") 28 | c.execute("SELECT crsql_commit_alter('issue')") 29 | end_time = time.time() 30 | print(f"alter add col time: {end_time - start_time}") 31 | 32 | None -------------------------------------------------------------------------------- /py/correctness/tests/test_config.py: -------------------------------------------------------------------------------- 1 | from crsql_correctness import connect, close 2 | import pathlib 3 | 4 | def test_config_merge_equal_values(): 5 | dbfile = "./config.db" 6 | pathlib.Path(dbfile).unlink(missing_ok=True) 7 | db = connect(dbfile) 8 | value = db.execute("SELECT crsql_config_set('merge-equal-values', 1);").fetchone() 9 | assert (value == (1,)) 10 | db.commit() 11 | 12 | value = db.execute("SELECT value FROM crsql_master WHERE key = 'config.merge-equal-values'").fetchone() 13 | assert (value == (1,)) 14 | 15 | value = db.execute("SELECT crsql_config_get('merge-equal-values');").fetchone() 16 | assert (value == (1,)) 17 | 18 | close(db) 19 | db = connect(dbfile) 20 | 21 | value = db.execute("SELECT value FROM crsql_master WHERE key = 'config.merge-equal-values'").fetchone() 22 | assert (value == (1,)) 23 | 24 | value = db.execute("SELECT crsql_config_get('merge-equal-values')").fetchone() 25 | assert (value == (1,)) 26 | -------------------------------------------------------------------------------- /py/correctness/tests/test_crsql_changes_filters.py: -------------------------------------------------------------------------------- 1 | from crsql_correctness import connect, close 2 | from pprint import pprint 3 | 4 | 5 | def setup_db(): 6 | c = connect(":memory:") 7 | c.execute("CREATE TABLE item (id PRIMARY KEY NOT NULL, x INTEGER, y INTEGER, desc TEXT)") 8 | c.execute("SELECT crsql_as_crr('item')") 9 | c.commit() 10 | 11 | c.execute("INSERT INTO item VALUES (123, 0, 0, 'The bestest thing')") 12 | c.execute("INSERT INTO item VALUES (321, 10, 10, 'The okest thing')") 13 | c.commit() 14 | 15 | c.execute("UPDATE item SET x = 1000 WHERE id = 123") 16 | c.execute("INSERT INTO item VALUES (411, -1, -1, 'The worst thing')") 17 | c.commit() 18 | 19 | c.execute("DELETE FROM item WHERE id = 411") 20 | c.commit() 21 | 22 | return (c, c.execute(changes_query + " ORDER BY db_version, seq ASC").fetchall()) 23 | 24 | 25 | changes_query = "SELECT [table], pk, cid, val, col_version, db_version, site_id, seq FROM crsql_changes" 26 | col_mapping = { 27 | 'table': 0, 28 | 'pk': 1, 29 | 'cid': 2, 30 | 'val': 3, 31 | 'col_version': 4, 32 | 'db_version': 5, 33 | 'site_id': 6, 34 | 'seq': 7 35 | } 36 | 37 | operations = [ 38 | ['<', lambda x, y: x < y], 39 | ['>', lambda x, y: x > y], 40 | ['=', lambda x, y: False if x is None or y is None else x == y], 41 | ['!=', lambda x, y: False if x is None or y is None else x != y], 42 | ['IS', lambda x, y: x == y], 43 | ['IS NOT', lambda x, y: x != y] 44 | ] 45 | 46 | 47 | def run_test(constraint, operation_subset=None, range=range(5)): 48 | (c, all_changes) = setup_db() 49 | 50 | for x in range: 51 | for (opcode, predicate) in operations: 52 | if operation_subset is None or opcode in operation_subset: 53 | tbl_changes = c.execute( 54 | changes_query + " WHERE [{}] {} ? ORDER BY db_version, seq ASC".format(constraint, opcode), (x, )).fetchall() 55 | mnl_changes = list( 56 | filter(lambda row: predicate(row[col_mapping[constraint]], x), all_changes)) 57 | assert (tbl_changes == mnl_changes) 58 | 59 | close(c) 60 | 61 | 62 | def test_dbversion_filter(): 63 | run_test("db_version") 64 | 65 | 66 | def test_seq_filter(): 67 | run_test("seq") 68 | 69 | 70 | def test_cid_filter(): 71 | run_test("cid", {'=', '!='}) 72 | 73 | 74 | def test_table_filter(): 75 | run_test("table", {'=', '!='}) 76 | 77 | 78 | def test_col_version_filter(): 79 | run_test("col_version") 80 | 81 | 82 | # TODO: pks should be returned as their actual type. . . 83 | # well we can't exactly do this since primary keys must be concatenated into a single 84 | # column. Maybe if there is only 1 primary key we can optimize for that case? 85 | def test_pk_filter(): 86 | run_test("pk", {'=', '!='}, ['123', '321', '411']) 87 | 88 | 89 | def test_site_id_filter(): 90 | run_test('site_id', {'=', '!=', 'IS', 'IS NOT'}, [None]) 91 | 92 | 93 | # TODO: val filter should return `any` type to match the original 94 | # value type of the underlying storage rather than a stringified version 95 | # def test_val_filter(): 96 | # run_test("val") 97 | -------------------------------------------------------------------------------- /py/correctness/tests/test_dbversion.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | from crsql_correctness import connect, close, min_db_v 3 | 4 | # c1 5 | 6 | 7 | def test_min_on_init(): 8 | c = connect(":memory:") 9 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v 10 | 11 | # c2 12 | 13 | 14 | def test_increments_on_modification(): 15 | c = connect(":memory:") 16 | c.execute("create table foo (id primary key not null, a)") 17 | c.execute("select crsql_as_crr('foo')") 18 | c.execute("insert into foo values (1, 2)") 19 | c.execute("commit") 20 | # +2 since create table statements bump version too 21 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v + 1 22 | c.execute("update foo set a = 3 where id = 1") 23 | c.execute("commit") 24 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v + 2 25 | c.execute("delete from foo where id = 1") 26 | c.execute("commit") 27 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v + 3 28 | close(c) 29 | 30 | # c3 31 | 32 | 33 | def test_db_version_restored_from_disk(): 34 | dbfile = "./dbversion_c3.db" 35 | pathlib.Path(dbfile).unlink(missing_ok=True) 36 | c = connect(dbfile) 37 | 38 | # C3 39 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v 40 | 41 | # close and re-open to check that we work with empty clock tables 42 | c.execute("create table foo (id primary key not null, a)") 43 | c.execute("select crsql_as_crr('foo')") 44 | c.close() 45 | c = connect(dbfile) 46 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v 47 | 48 | # insert so we get a clock entry 49 | c.execute("insert into foo values (1, 2)") 50 | c.commit() 51 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v + 1 52 | 53 | # Close and reopen to check that version was persisted and re-initialized correctly 54 | close(c) 55 | c = connect(dbfile) 56 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v + 1 57 | close(c) 58 | 59 | # c4 60 | 61 | 62 | def test_each_tx_gets_a_version(): 63 | c = connect(":memory:") 64 | 65 | c.execute("create table foo (id primary key not null, a)") 66 | c.execute("select crsql_as_crr('foo')") 67 | c.execute("insert into foo values (1, 2)") 68 | c.execute("insert into foo values (2, 2)") 69 | c.commit() 70 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v + 1 71 | 72 | c.execute("insert into foo values (3, 2)") 73 | c.execute("insert into foo values (4, 2)") 74 | c.commit() 75 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v + 2 76 | 77 | close(c) 78 | 79 | 80 | def test_rollback_does_not_move_db_version(): 81 | c = connect(":memory:") 82 | 83 | c.execute("create table foo (id primary key not null, a)") 84 | c.execute("select crsql_as_crr('foo')") 85 | 86 | c.execute("insert into foo values (1, 2)") 87 | c.execute("insert into foo values (2, 2)") 88 | c.rollback() 89 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v 90 | 91 | c.execute("insert into foo values (3, 2)") 92 | c.execute("insert into foo values (4, 2)") 93 | c.rollback() 94 | assert c.execute("SELECT crsql_db_version()").fetchone()[ 95 | 0] == min_db_v 96 | 97 | c.execute("insert into foo values (1, 2)") 98 | c.execute("insert into foo values (2, 2)") 99 | c.commit() 100 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v + 1 101 | 102 | c.execute("insert into foo values (3, 2)") 103 | c.execute("insert into foo values (4, 2)") 104 | c.commit() 105 | assert c.execute("SELECT crsql_db_version()").fetchone()[0] == min_db_v + 2 106 | 107 | close(c) 108 | -------------------------------------------------------------------------------- /py/correctness/tests/test_insert_new_rows.py: -------------------------------------------------------------------------------- 1 | from crsql_correctness import connect, min_db_v 2 | from pprint import pprint 3 | 4 | 5 | def test_c1_c2_c3_c4_c6_c7_crr_values(): 6 | c = connect(":memory:") 7 | init_version = c.execute("SELECT crsql_db_version()").fetchone()[0] 8 | c.execute("create table foo (id primary key not null, a)") 9 | c.execute("select crsql_as_crr('foo')") 10 | 11 | c.execute("insert into foo values(1, 2)") 12 | c.commit() 13 | 14 | rows = c.execute( 15 | "select key, col_name, col_version, db_version, site_id from foo__crsql_clock").fetchall() 16 | assert [(1, 'a', 1, init_version + 1, 0)] == rows 17 | new_version = c.execute("SELECT crsql_db_version()").fetchone()[0] 18 | 19 | assert new_version == init_version + 1 20 | 21 | clock_rows = c.execute("select * from foo__crsql_clock").fetchall() 22 | assert len(clock_rows) == 1 23 | 24 | row = c.execute("select id, a from foo").fetchone() 25 | assert row[0] == 1 26 | assert row[1] == 2 27 | 28 | new_version = c.execute("SELECT crsql_db_version()").fetchone()[0] 29 | 30 | assert new_version == init_version + 1 31 | -------------------------------------------------------------------------------- /py/correctness/tests/test_prior_versions.py: -------------------------------------------------------------------------------- 1 | from crsql_correctness import connect, close, min_db_v 2 | import shutil 3 | from pprint import pprint 4 | 5 | 6 | # We no longer support these versions. 7 | # def test_can_load_v0_12_0(): 8 | # prefix = "./prior-dbs/v0.12.0" 9 | # # copy the file given connecting might migrate it! 10 | # shutil.copyfile(prefix + ".prior-db", prefix + ".db") 11 | # c = connect(prefix + ".db") 12 | # rows = c.execute("SELECT *, seq FROM crsql_changes").fetchall() 13 | # assert (rows == [('foo', b'\x01\x0b\x03one', '-1', None, 1, 0, None, 1, 0), 14 | # ('bar', b'\x01\t\x01', '-1', None, 1, 0, None, 1, 0), 15 | # ('foo', b'\x01\x0b\x03one', 'b', 2, 1, 1, None, 1, 0), 16 | # ('bar', b'\x01\t\x01', 'b', 2, 1, 2, None, 1, 0)]) 17 | 18 | # version = c.execute( 19 | # "SELECT value FROM crsql_master WHERE key ='crsqlite_version'").fetchone() 20 | # assert (version[0] == 150000) 21 | # close(c) 22 | 23 | 24 | # def test_can_load_v0_13_0(): 25 | # prefix = "./prior-dbs/v0.13.0" 26 | # # copy the file given connecting might migrate it! 27 | # shutil.copyfile(prefix + ".prior-db", prefix + ".db") 28 | # c = connect(prefix + ".db") 29 | # rows = c.execute("SELECT *, seq FROM crsql_changes").fetchall() 30 | # assert (rows == [('foo', b'\x01\t\x01', '-1', None, 1, 0, None, 1, 0), 31 | # ('foo', b'\x01\t\x03', '-1', None, 1, 0, None, 1, 0), 32 | # ('foo', b'\x01\t\x05', '-1', None, 1, 0, None, 1, 0), 33 | # ('foo', b'\x01\t\x06', '-1', None, 1, 0, None, 1, 0), 34 | # ('foo', b'\x01\t\x08', '-1', None, 1, 0, None, 1, 0), 35 | # ('foo', b'\x01\t\x01', 'b', 2, 1, 1, None, 1, 0), 36 | # ('foo', b'\x01\t\x03', 'b', 4, 1, 2, None, 1, 0), 37 | # ('foo', b'\x01\t\x05', 'b', 6, 1, 2, None, 1, 1), 38 | # ('foo', b'\x01\t\x06', 'b', 7, 1, 2, None, 1, 2), 39 | # ('foo', b'\x01\t\x08', 'b', 9, 1, 3, None, 1, 0)]) 40 | 41 | # version = c.execute( 42 | # "SELECT value FROM crsql_master WHERE key ='crsqlite_version'").fetchone() 43 | # assert (version[0] == 150000) 44 | # close(c) 45 | 46 | 47 | def test_can_load_as_readonly(): 48 | prefix = "./prior-dbs/v0.15.0" 49 | # open it once r/w to create all the tables 50 | c = connect('file:' + prefix + ".db", uri=True) 51 | close(c) 52 | 53 | c = connect('file:' + prefix + ".db?mode=ro", uri=True) 54 | # just expecting not to throw. 55 | close(c) 56 | -------------------------------------------------------------------------------- /py/correctness/tests/test_sandbox.py: -------------------------------------------------------------------------------- 1 | from crsql_correctness import connect, close, min_db_v 2 | from pprint import pprint 3 | 4 | # exploratory tests to debug changes 5 | 6 | 7 | def sync_left_to_right(l, r, since): 8 | changes = l.execute( 9 | "SELECT * FROM crsql_changes WHERE db_version > ? ORDER BY db_version, seq ASC", (since,)) 10 | 11 | ret = 0 12 | for change in changes: 13 | ret = change[5] 14 | r.execute( 15 | "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", change) 16 | 17 | r.commit() 18 | return ret 19 | 20 | 21 | def test_sync(): 22 | def setup(): 23 | c = connect(":memory:") 24 | c.execute("CREATE TABLE item (id PRIMARY KEY NOT NULL, width INTEGER, height INTEGER, name TEXT, dscription TEXT, weight INTEGER)") 25 | c.execute("SELECT crsql_as_crr('item')") 26 | c.commit() 27 | return c 28 | 29 | def insert_item(c, args): 30 | c.execute("INSERT INTO item VALUES (?, ?, ?, ?, ?, ?)", args) 31 | c.commit() 32 | 33 | a = setup() 34 | b = setup() 35 | 36 | insert_item(a, ('9838abbe-6fa8-4755-af2b-9f0484888809', 37 | None, None, None, None, None)) 38 | insert_item(b, ('f94ef174-459f-4b07-bc7a-c1104a97ceb5', 39 | None, None, None, None, None)) 40 | 41 | since_a = sync_left_to_right(a, b, 0) 42 | 43 | a.execute("DELETE FROM item WHERE id = '9838abbe-6fa8-4755-af2b-9f0484888809'") 44 | a.commit() 45 | 46 | insert_item(a, ('d5653f10-b858-46c7-97e5-5660eca47d28', 47 | None, None, None, None, None)) 48 | 49 | sync_left_to_right(a, b, since_a) 50 | sync_left_to_right(b, a, 0) 51 | 52 | # pprint("A") 53 | # pprint(a.execute("SELECT * FROM item").fetchall()) 54 | # pprint("B") 55 | # pprint(b.execute("SELECT * FROM item").fetchall()) 56 | 57 | # pprint("A changes") 58 | # pprint(a.execute("SELECT * FROM crsql_changes").fetchall()) 59 | -------------------------------------------------------------------------------- /py/correctness/tests/test_sentinel_omission.py: -------------------------------------------------------------------------------- 1 | from crsql_correctness import connect, close, min_db_v 2 | from pprint import pprint 3 | 4 | 5 | def sync_left_to_right(l, r, since): 6 | changes = l.execute( 7 | "SELECT * FROM crsql_changes WHERE db_version > ?", (since,)) 8 | for change in changes: 9 | r.execute( 10 | "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", change) 11 | r.commit() 12 | 13 | 14 | def make_simple_schema(): 15 | c = connect(":memory:") 16 | c.execute("CREATE TABLE test (id INTEGER PRIMARY KEY NOT NULL, [text] TEXT);") 17 | c.execute("SELECT crsql_as_crr('test')") 18 | c.execute("CREATE TABLE test2 (id INTEGER PRIMARY KEY NOT NULL, [text] TEXT);") 19 | c.execute("SELECT crsql_as_crr('test2')") 20 | c.commit() 21 | return c 22 | 23 | 24 | def make_data(c): 25 | for n in range(0, 200): 26 | c.execute("INSERT INTO test (id, text) VALUES (?, ?)", 27 | (n, "hello {}".format(n))) 28 | c.execute("INSERT INTO test2 (id, text) VALUES (?, ?)", 29 | (n, "hello {}".format(n))) 30 | c.execute("INSERT INTO test (id, text) VALUES (?, ?)", 31 | (n + 10000, "hello {}".format(n))) 32 | c.execute("INSERT INTO test2 (id, text) VALUES (?, ?)", 33 | (n + 10000, "hello {}".format(n))) 34 | c.commit() 35 | 36 | # https://discord.com/channels/989870439897653248/989870440585494530/1137099971284435124 37 | 38 | 39 | def test_omitted_on_insert(): 40 | c = make_simple_schema() 41 | make_data(c) 42 | 43 | assert (c.execute( 44 | "SELECT count(*) FROM crsql_changes WHERE cid = '-1'").fetchone()[0] == 0) 45 | 46 | 47 | def test_created_on_delete(): 48 | c = make_simple_schema() 49 | 50 | make_data(c) 51 | 52 | c.execute("DELETE FROM test") 53 | c.execute("DELETE FROM test2") 54 | c.commit() 55 | 56 | assert (c.execute( 57 | "SELECT count(*) FROM crsql_changes WHERE cid = '-1'").fetchone()[0] == 800) 58 | 59 | 60 | def test_not_created_on_replace(): 61 | c = make_simple_schema() 62 | make_data(c) 63 | 64 | for n in range(0, 200): 65 | c.execute("INSERT OR REPLACE INTO test (id, text) VALUES (?, ?)", 66 | (n, "hello {}".format(n))) 67 | c.execute("INSERT OR REPLACE INTO test2 (id, text) VALUES (?, ?)", 68 | (n, "hello {}".format(n))) 69 | c.execute("INSERT OR REPLACE INTO test (id, text) VALUES (?, ?)", 70 | (n + 10000, "hello {}".format(n))) 71 | c.execute("INSERT OR REPLACE INTO test2 (id, text) VALUES (?, ?)", 72 | (n + 10000, "hello {}".format(n))) 73 | c.commit() 74 | 75 | assert (c.execute( 76 | "SELECT count(*) FROM crsql_changes WHERE cid = '-1'").fetchone()[0] == 0) 77 | 78 | 79 | def test_not_created_on_merge(): 80 | a = make_simple_schema() 81 | b = make_simple_schema() 82 | make_data(a) 83 | 84 | sync_left_to_right(a, b, 0) 85 | 86 | assert (a.execute( 87 | "SELECT count(*) FROM crsql_changes WHERE cid = '-1'").fetchone()[0] == 0) 88 | assert (b.execute( 89 | "SELECT count(*) FROM crsql_changes WHERE cid = '-1'").fetchone()[0] == 0) 90 | 91 | 92 | def test_not_created_on_noop_merge(): 93 | a = make_simple_schema() 94 | b = make_simple_schema() 95 | make_data(a) 96 | make_data(b) 97 | 98 | # dbs have the same state, nothing should happen 99 | sync_left_to_right(a, b, 0) 100 | 101 | assert (a.execute( 102 | "SELECT count(*) FROM crsql_changes WHERE cid = '-1'").fetchone()[0] == 0) 103 | assert (b.execute( 104 | "SELECT count(*) FROM crsql_changes WHERE cid = '-1'").fetchone()[0] == 0) 105 | assert (a.execute("SELECT crsql_db_version()").fetchall() 106 | == b.execute("SELECT crsql_db_version()").fetchall()) 107 | 108 | 109 | def test_not_created_on_update_merge(): 110 | a = make_simple_schema() 111 | b = make_simple_schema() 112 | make_data(a) 113 | make_data(b) 114 | 115 | for n in range(0, 200): 116 | a.execute("UPDATE test SET text = 'goodbye {}' WHERE id = {}".format(n, n)) 117 | a.execute( 118 | "UPDATE test SET text = 'goodbye {}' WHERE id = {}".format(n, n + 10000)) 119 | 120 | a.commit() 121 | sync_left_to_right(a, b, 0) 122 | assert (b.execute( 123 | "SELECT count(*) FROM crsql_changes WHERE cid = '-1'").fetchone()[0] == 0) 124 | 125 | 126 | def test_sentinel_propagated_when_present(): 127 | a = make_simple_schema() 128 | b = make_simple_schema() 129 | 130 | make_data(a) 131 | 132 | a.execute("DELETE FROM test") 133 | a.execute("DELETE FROM test2") 134 | a.commit() 135 | 136 | sync_left_to_right(a, b, 0) 137 | 138 | assert (b.execute( 139 | "SELECT count(*) FROM crsql_changes WHERE cid = '-1'").fetchone()[0] == 800) 140 | -------------------------------------------------------------------------------- /py/correctness/tests/test_site_id_lookaside.py: -------------------------------------------------------------------------------- 1 | from crsql_correctness import connect, close, min_db_v 2 | from pprint import pprint 3 | import random 4 | # Test that we can insert with site id and then get it back out properly on read 5 | # from crsql_changes 6 | 7 | 8 | def make_simple_schema(): 9 | c = connect(":memory:") 10 | c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") 11 | c.execute("SELECT crsql_as_crr('foo')") 12 | c.commit() 13 | return c 14 | 15 | 16 | def test_insert_site_id(): 17 | # is in lookaside 18 | # is an ordinal in actual table 19 | a = make_simple_schema() 20 | a.execute( 21 | "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 1, 1, x'1dc8d6bb7f8941088327d9439a7927a4', 1, 0)") 22 | a.commit() 23 | 24 | # Ordinal value, not site id, is in the clock table 25 | ord = a.execute( 26 | "SELECT site_id FROM foo__crsql_clock").fetchone()[0] 27 | assert (ord == 1) 28 | # site id is in the site id table for that given ordinal 29 | assert ( 30 | a.execute( 31 | "SELECT quote(site_id) FROM crsql_site_id WHERE ordinal = ?", (ord,) 32 | ).fetchone()[0] == "x'1dc8d6bb7f8941088327d9439a7927a4'".upper()) 33 | 34 | # site id comes out of crsql_changes as expected 35 | assert (a.execute("SELECT quote(site_id) FROM crsql_changes").fetchone()[ 36 | 0] == "x'1dc8d6bb7f8941088327d9439a7927a4'".upper()) 37 | 38 | 39 | def test_site_id_filter(): 40 | a = make_simple_schema() 41 | a.execute( 42 | "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 1, 1, x'1dc8d6bb7f8941088327d9439a7927a4', 1, 0)") 43 | a.commit() 44 | 45 | assert (a.execute( 46 | "SELECT quote(site_id) FROM crsql_changes WHERE site_id = x'1dc8d6bb7f8941088327d9439a7927a4'").fetchone()[0] == "x'1dc8d6bb7f8941088327d9439a7927a4'".upper()) 47 | 48 | 49 | def test_local_changes_have_local_site(): 50 | a = make_simple_schema() 51 | a.execute("INSERT INTO foo VALUES (2,2)") 52 | a.execute("INSERT INTO foo VALUES (3,2)") 53 | a.execute("INSERT INTO foo VALUES (4,2)") 54 | a.commit() 55 | a.execute( 56 | "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 1, 1, x'1dc8d6bb7f8941088327d9439a7927a4', 1, 0)") 57 | a.commit() 58 | 59 | assert (a.execute( 60 | "SELECT count(*) FROM crsql_changes WHERE site_id IS crsql_site_id()").fetchone()[0] == 3) 61 | assert (a.execute( 62 | "SELECT count(*) FROM crsql_changes").fetchone()[0] == 4) 63 | None 64 | 65 | 66 | def test_site_id_ordinals_do_not_move_on_merge(): 67 | a = make_simple_schema() 68 | 69 | a.execute( 70 | "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 1, 1, x'1dc8d6bb7f8941088327d9439a7927a4', 1, 0)") 71 | a.commit() 72 | 73 | x = a.execute("SELECT quote(site_id) FROM crsql_changes").fetchall() 74 | assert (x == [("X'1DC8D6BB7F8941088327D9439A7927A4'",)]) 75 | 76 | # insert again with the same site id 77 | a.execute( 78 | "INSERT INTO crsql_changes VALUES ('foo', x'010902', 'b', 1, 1, 1, x'1dc8d6bb7f8941088327d9439a7927a4', 1, 0)") 79 | a.commit() 80 | 81 | x = a.execute("SELECT quote(site_id) FROM crsql_changes").fetchall() 82 | # site ids should be returned with changes 83 | assert (x == [("X'1DC8D6BB7F8941088327D9439A7927A4'",), 84 | ("X'1DC8D6BB7F8941088327D9439A7927A4'",)]) 85 | 86 | # insert a new site id 87 | a.execute( 88 | "INSERT INTO crsql_changes VALUES ('foo', x'010903', 'b', 1, 1, 1, x'2dc8d6bb7f8941088327d9439a7927a4', 1, 0)") 89 | a.commit() 90 | # insert again with that new site id 91 | a.execute( 92 | "INSERT INTO crsql_changes VALUES ('foo', x'010904', 'b', 1, 1, 1, x'2dc8d6bb7f8941088327d9439a7927a4', 1, 0)") 93 | a.commit() 94 | # should only be 2 site ids w/ ordinals 1 and 2. 1DC... -> 1, 2DC... -> 2 95 | x = a.execute( 96 | "SELECT quote(site_id), ordinal FROM crsql_site_id WHERE ordinal != 0").fetchall() 97 | assert (x == [("X'1DC8D6BB7F8941088327D9439A7927A4'", 1), 98 | ("X'2DC8D6BB7F8941088327D9439A7927A4'", 2)]) 99 | 100 | None 101 | -------------------------------------------------------------------------------- /py/correctness/tests/test_siteid.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | from uuid import UUID 3 | from crsql_correctness import connect 4 | from pprint import pprint 5 | 6 | 7 | def sync_left_to_right(l, r, since): 8 | r_site_id = r.execute("SELECT crsql_site_id()").fetchone()[0] 9 | changes = l.execute( 10 | "SELECT * FROM crsql_changes WHERE db_version > ? AND site_id IS NOT ?", (since, r_site_id)) 11 | for change in changes: 12 | r.execute( 13 | "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", change) 14 | r.commit() 15 | 16 | 17 | def test_c1(): 18 | c = connect(":memory:") 19 | siteid_bytes = c.execute("select crsql_site_id()").fetchone()[0] 20 | siteid = UUID(bytes=siteid_bytes) 21 | assert siteid.bytes == siteid_bytes 22 | 23 | 24 | def test_c2(): 25 | c = connect(":memory:") 26 | siteid_fn = c.execute("select crsql_site_id()").fetchone()[0] 27 | siteid_tbl = c.execute("select site_id from crsql_site_id").fetchone()[0] 28 | 29 | assert siteid_fn == siteid_tbl 30 | 31 | 32 | def test_c3c4(): 33 | dbfile = "./siteid_c3c4.db" 34 | pathlib.Path(dbfile).unlink(missing_ok=True) 35 | c = connect(dbfile) 36 | 37 | siteid_initial = c.execute("select crsql_site_id()").fetchone()[0] 38 | c.close() 39 | 40 | c = connect(dbfile) 41 | siteid_restored = c.execute("select crsql_site_id()").fetchone()[0] 42 | 43 | assert siteid_initial == siteid_restored 44 | 45 | 46 | # Site id is set to crsql_site_id on local writes 47 | def test_site_id_for_local_writes(): 48 | c = connect(":memory:") 49 | c.execute("CREATE TABLE foo (id not null, x, y, primary key (id))") 50 | c.execute("SELECT crsql_as_crr('foo')") 51 | c.commit() 52 | 53 | c.execute("INSERT INTO foo VALUES (1, 2, 3)") 54 | c.commit() 55 | 56 | def check_counts(): 57 | total_changes_count = c.execute( 58 | "SELECT count(*) FROM crsql_changes").fetchone()[0] 59 | changes_with_local_site_count = c.execute( 60 | "SELECT count(*) FROM crsql_changes WHERE site_id = crsql_site_id()").fetchone()[0] 61 | assert total_changes_count == changes_with_local_site_count 62 | 63 | c.execute("UPDATE foo SET x = 3 WHERE id = 1") 64 | c.commit() 65 | check_counts() 66 | 67 | c.execute("INSERT OR REPLACE INTO foo VALUES (1, 5, 9)") 68 | c.commit() 69 | check_counts() 70 | 71 | c.execute("DELETE FROM foo") 72 | c.commit() 73 | check_counts() 74 | 75 | 76 | def test_site_id_from_merge(): 77 | def simple_schema(): 78 | a = connect(":memory:") 79 | a.execute("create table foo (a primary key not null, b);") 80 | a.commit() 81 | a.execute("SELECT crsql_as_crr('foo')") 82 | a.commit() 83 | return a 84 | 85 | a = simple_schema() 86 | a.execute("INSERT INTO foo VALUES (1, 2.0e2);") 87 | a.commit() 88 | a.execute("INSERT INTO foo VALUES (2, X'1232');") 89 | a.commit() 90 | 91 | b = simple_schema() 92 | c = simple_schema() 93 | 94 | sync_left_to_right(a, b, 0) 95 | sync_left_to_right(b, c, 0) 96 | 97 | site_ids_fromC = c.execute( 98 | "SELECT site_id FROM crsql_changes ORDER BY pk ASC").fetchall() 99 | site_ids_fromB = b.execute( 100 | "SELECT site_id FROM crsql_changes ORDER BY pk ASC").fetchall() 101 | site_ids_fromA = a.execute( 102 | "SELECT site_id FROM crsql_changes ORDER BY pk ASC").fetchall() 103 | assert site_ids_fromC == site_ids_fromA 104 | assert site_ids_fromB == site_ids_fromA 105 | -------------------------------------------------------------------------------- /py/correctness/tests/test_sync_bit.py: -------------------------------------------------------------------------------- 1 | from crsql_correctness import connect, close, min_db_v 2 | from pprint import pprint 3 | 4 | # Test that no trigger are run during merging / sync bit is respected. 5 | # How can we test this? 6 | # 1. We can install our own trigger which checks the sync bit and writes something 7 | # 2. We can check that only the expected clock rows are written? 8 | 9 | 10 | def create_db(): 11 | c = connect(":memory:") 12 | c.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b)") 13 | c.execute("SELECT crsql_as_crr('foo')") 14 | c.commit() 15 | return c 16 | 17 | 18 | def test_insert_row(): 19 | # db version, seq, col version, site id, cl should all be from the insertion 20 | c = create_db() 21 | c.execute( 22 | "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 4, 4, x'1dc8d6bb7f8941088327d9439a7927a4', 3, 6)") 23 | c.commit() 24 | 25 | changes = c.execute("SELECT * FROM crsql_changes").fetchall() 26 | # what we wrote should be what we get back 27 | assert (changes == [('foo', 28 | b'\x01\t\x01', 29 | '-1', 30 | None, 31 | 3, 32 | 4, 33 | b"\x1d\xc8\xd6\xbb\x7f\x89A\x08\x83'\xd9C\x9ay'\xa4", 34 | 3, 35 | 6), 36 | ('foo', 37 | b'\x01\t\x01', 38 | 'b', 39 | 1, 40 | 4, 41 | 4, 42 | b"\x1d\xc8\xd6\xbb\x7f\x89A\x08\x83'\xd9C\x9ay'\xa4", 43 | 3, 44 | 6)]) 45 | 46 | 47 | def test_update_row(): 48 | c = create_db() 49 | c.execute("INSERT INTO foo VALUES (1, 2)") 50 | c.commit() 51 | c.execute( 52 | "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 4, 4, x'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', 3, 6)") 53 | changes = c.execute("SELECT * FROM crsql_changes").fetchall() 54 | # what we wrote should be what we get back since we win the merge 55 | assert (changes == [('foo', 56 | b'\x01\t\x01', 57 | '-1', 58 | None, 59 | 3, 60 | 4, 61 | b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff', 62 | 3, 63 | 6), 64 | ('foo', 65 | b'\x01\t\x01', 66 | 'b', 67 | 1, 68 | 4, 69 | 4, 70 | b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff', 71 | 3, 72 | 6)]) 73 | 74 | 75 | def test_delete_row(): 76 | c = create_db() 77 | c.execute("INSERT INTO foo VALUES (1, 2)") 78 | c.commit() 79 | c.execute("INSERT INTO crsql_changes VALUES ('foo', x'010901', '-1', 1, 4, 4, x'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', 4, 6)") 80 | c.commit() 81 | changes = c.execute("SELECT * FROM crsql_changes").fetchall() 82 | assert (changes == [('foo', 83 | b'\x01\t\x01', 84 | '-1', 85 | None, 86 | 4, 87 | 4, 88 | b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff', 89 | 4, 90 | 6)]) 91 | 92 | 93 | def test_custom_trigger(): 94 | c = create_db() 95 | c.execute("CREATE TABLE log (a integer primary key, b)") 96 | c.execute("""CREATE TRIGGER log_up AFTER INSERT ON foo WHEN crsql_internal_sync_bit() = 0 BEGIN 97 | INSERT INTO log (b) VALUES (1); 98 | END;""") 99 | c.commit() 100 | c.execute( 101 | "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 4, 4, x'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', 3, 6)") 102 | c.commit() 103 | rows = c.execute("SELECT * FROM log").fetchall() 104 | assert (rows == []) 105 | rows = c.execute("SELECT * FROM foo").fetchall() 106 | assert (rows == [(1, 1)]) 107 | 108 | c.execute("INSERT INTO foo VALUES (5, 5)") 109 | c.commit() 110 | rows = c.execute("SELECT * FROM log").fetchall() 111 | assert (rows == [(1, 1)]) 112 | 113 | c.commit() 114 | c.execute("DROP TRIGGER log_up") 115 | c.execute("""CREATE TRIGGER log_up AFTER INSERT ON foo WHEN crsql_internal_sync_bit() = 1 BEGIN 116 | INSERT INTO log (b) VALUES (1); 117 | END;""") 118 | c.commit() 119 | c.execute( 120 | "INSERT INTO crsql_changes VALUES ('foo', x'010902', 'b', 1, 4, 4, x'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', 3, 6)") 121 | rows = c.execute("SELECT * FROM log").fetchall() 122 | assert (rows == [(1, 1), (2, 1)]) 123 | -------------------------------------------------------------------------------- /py/correctness/tests/test_update_rows.py: -------------------------------------------------------------------------------- 1 | from crsql_correctness import connect 2 | 3 | -------------------------------------------------------------------------------- /py/perf/.gitignore: -------------------------------------------------------------------------------- 1 | correctness.db 2 | perf.db-shm 3 | perf.db 4 | perf.db-wal 5 | -------------------------------------------------------------------------------- /py/perf/notes.md: -------------------------------------------------------------------------------- 1 | jupyter-lab from this dir 2 | --------------------------------------------------------------------------------