├── .cargo └── config ├── .editorconfig ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ └── actions.yaml ├── .gitignore ├── .prettierrc ├── .rustfmt.toml ├── .vscode ├── extensions.json ├── launch.json └── settings.json ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Cargo.lock ├── Cargo.toml ├── GUIDE.md ├── LICENSE ├── README.md ├── biome.json ├── demo ├── cloudflare-backend │ ├── .gitignore │ ├── Cargo.toml │ ├── package.json │ ├── src │ │ ├── coordinator.rs │ │ ├── lib.rs │ │ └── persistence.rs │ └── wrangler.toml ├── demo-reducer │ ├── Cargo.toml │ └── src │ │ └── lib.rs └── frontend │ ├── .gitignore │ ├── index.html │ ├── package.json │ ├── postcss.config.js │ ├── public │ ├── favicon-dark-32x32.png │ ├── favicon-light-32x32.png │ ├── favicon.svg │ └── robots.txt │ ├── src │ ├── App.tsx │ ├── components │ │ ├── ConnectionStatus.tsx │ │ ├── Header.tsx │ │ ├── QueryViewer.tsx │ │ ├── TaskForm.tsx │ │ ├── TaskItem.tsx │ │ └── TaskList.tsx │ ├── doctype.ts │ ├── main.tsx │ ├── theme.ts │ └── vite-env.d.ts │ ├── tsconfig.json │ ├── tsconfig.node.json │ └── vite.config.ts ├── examples ├── guestbook-react │ ├── README.md │ ├── index.html │ ├── package.json │ ├── src │ │ ├── doctype.ts │ │ ├── main.tsx │ │ └── vite-env.d.ts │ ├── tsconfig.json │ ├── tsconfig.node.json │ └── vite.config.ts ├── guestbook-solid-js │ ├── README.md │ ├── index.html │ ├── package.json │ ├── src │ │ ├── doctype.ts │ │ └── main.tsx │ ├── tsconfig.json │ ├── tsconfig.node.json │ └── vite.config.ts └── reducer-guestbook │ ├── Cargo.toml │ └── src │ └── lib.rs ├── justfile ├── lib ├── sqlite-vfs │ ├── Cargo.toml │ ├── README.md │ └── src │ │ └── lib.rs ├── sqlsync-react │ ├── package.json │ ├── rollup.config.mjs │ ├── src │ │ ├── context.tsx │ │ ├── hooks.ts │ │ └── index.ts │ └── tsconfig.json ├── sqlsync-reducer │ ├── Cargo.toml │ ├── examples │ │ ├── guest.rs │ │ └── host.rs │ └── src │ │ ├── guest_ffi.rs │ │ ├── guest_reactor.rs │ │ ├── host_ffi.rs │ │ ├── lib.rs │ │ └── types.rs ├── sqlsync-solid-js │ ├── package.json │ ├── rollup.config.js │ ├── src │ │ ├── context.tsx │ │ ├── hooks.ts │ │ └── index.ts │ └── tsconfig.json ├── sqlsync-worker │ ├── package.json │ ├── rollup.config.mjs │ ├── sqlsync-wasm │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ └── src │ │ │ ├── api.rs │ │ │ ├── doc_task.rs │ │ │ ├── lib.rs │ │ │ ├── net.rs │ │ │ ├── reactive.rs │ │ │ ├── signal.rs │ │ │ ├── sql.rs │ │ │ └── utils.rs │ ├── src │ │ ├── index.ts │ │ ├── journal-id.ts │ │ ├── port.ts │ │ ├── sql.ts │ │ ├── sqlsync.ts │ │ ├── types.ts │ │ ├── util.ts │ │ └── worker.ts │ └── tsconfig.json ├── sqlsync │ ├── Cargo.toml │ ├── examples │ │ ├── counter-reducer.rs │ │ ├── end-to-end-local-net.rs │ │ ├── end-to-end-local.rs │ │ ├── hello-reducer.rs │ │ └── task-reducer.rs │ └── src │ │ ├── coordinator.rs │ │ ├── db.rs │ │ ├── error.rs │ │ ├── iter.rs │ │ ├── journal │ │ ├── cursor.rs │ │ ├── journalid.rs │ │ ├── memory.rs │ │ └── mod.rs │ │ ├── lib.rs │ │ ├── local.rs │ │ ├── lsn.rs │ │ ├── page.rs │ │ ├── positioned_io.rs │ │ ├── reactive_query.rs │ │ ├── reducer.rs │ │ ├── replication.rs │ │ ├── serialization.rs │ │ ├── storage.rs │ │ ├── timeline.rs │ │ ├── unixtime.rs │ │ └── vfs.rs └── testutil │ ├── Cargo.toml │ └── src │ └── lib.rs ├── package.json ├── pnpm-lock.yaml └── pnpm-workspace.yaml /.cargo/config: -------------------------------------------------------------------------------- 1 | [target.x86_64-apple-darwin] 2 | rustflags = [ "-C", "link-args=-Wl,-undefined,dynamic_lookup" ] 3 | 4 | [target.aarch64-apple-darwin] 5 | rustflags = [ "-C", "link-args=-Wl,-undefined,dynamic_lookup" ] -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | max_line_length = 100 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | 10 | [*.{js,ts,jsx,tsx,json}] 11 | indent_style = space 12 | indent_size = 2 13 | 14 | [*.md] 15 | indent_style = space 16 | indent_size = 2 17 | max_line_length = 80 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '[BUG]' 5 | labels: 'bug' 6 | assignees: 'carlsverre' 7 | --- 8 | 9 | **Describe the bug** 10 | A clear and concise description of what the bug is. 11 | 12 | **To Reproduce** 13 | Steps to reproduce the behavior: 14 | 1. Run `...` 15 | 2. Do ... 16 | 3. See error 17 | 18 | **Expected behavior** 19 | A clear and concise description of what you expected to happen. 20 | 21 | **Additional context** 22 | Add any other context about the problem here. 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: 'feature' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/actions.yaml: -------------------------------------------------------------------------------- 1 | name: build & test 2 | on: 3 | push: 4 | paths-ignore: 5 | - "**.md" 6 | branches: 7 | - main 8 | pull_request: 9 | paths-ignore: 10 | - "**.md" 11 | jobs: 12 | build: 13 | permissions: 14 | pull-requests: write 15 | runs-on: ubuntu-latest 16 | steps: 17 | # checkout repo 18 | - uses: actions/checkout@v4 19 | # rust setup 20 | - uses: dtolnay/rust-toolchain@stable 21 | with: 22 | toolchain: stable 23 | target: wasm32-unknown-unknown 24 | components: clippy, rustfmt 25 | - uses: Swatinem/rust-cache@v2 26 | - uses: extractions/setup-just@v1 27 | - name: install wasm-pack 28 | run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh 29 | # clang is needed to compile sqlite 30 | - name: Install LLVM and Clang 31 | uses: KyleMayes/install-llvm-action@v1.9.0 32 | with: 33 | version: 11 34 | # node setup 35 | - uses: actions/setup-node@v3 36 | with: 37 | node-version: "18" 38 | - uses: pnpm/action-setup@v2 39 | with: 40 | version: "8" 41 | run_install: true 42 | # build, test, and package sqlsync 43 | - name: Build all 44 | run: just build 45 | - name: Lint & Format 46 | run: just lint 47 | - name: Unit tests 48 | run: just unit-test 49 | - name: end-to-end-local 50 | run: just test-end-to-end-local 51 | - name: end-to-end-local-net 52 | run: just test-end-to-end-local-net 53 | - name: test sqlsync-reducer 54 | run: just test-sqlsync-reducer 55 | - name: build sqlsync js packages 56 | run: just package-sqlsync-react package-sqlsync-worker package-sqlsync-solid-js 57 | - name: build frontend 58 | run: cd demo/frontend && pnpm build 59 | - name: build cloudflare backend 60 | run: cd demo/cloudflare-backend && cargo install -q worker-build && worker-build 61 | - name: build examples 62 | run: cd examples/guestbook-react && pnpm build 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | 26 | /lib/sqlsync-worker/sqlsync-wasm/pkg 27 | /demo/cloudflare-backend/build 28 | /demo/cloudflare-backend/dist 29 | /lib/sqlsync-worker/dist 30 | /lib/sqlsync-react/dist 31 | /llvm 32 | /target 33 | /.pnpm-store 34 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "proseWrap": "always" 3 | } -------------------------------------------------------------------------------- /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | # https://rust-lang.github.io/rustfmt 2 | max_width = 100 3 | struct_lit_width = 40 4 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations. 3 | // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp 4 | 5 | // List of extensions which should be recommended for users of this workspace. 6 | "recommendations": ["biomejs.biome"], 7 | // List of extensions recommended by VS Code that should not be recommended for users of this workspace. 8 | "unwantedRecommendations": [] 9 | } 10 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Debug end-to-end-local", 9 | "type": "lldb", 10 | "request": "launch", 11 | "cargo": { 12 | "args": ["build", "--example=end-to-end-local"], 13 | "filter": { 14 | "name": "end-to-end-local", 15 | "kind": "example" 16 | } 17 | }, 18 | "args": [], 19 | "cwd": "${workspaceFolder}" 20 | }, 21 | { 22 | "name": "Debug end-to-end-local-net", 23 | "type": "lldb", 24 | "request": "launch", 25 | "cargo": { 26 | "args": ["build", "--example=end-to-end-local-net"], 27 | "filter": { 28 | "name": "end-to-end-local-net", 29 | "kind": "example" 30 | } 31 | }, 32 | "args": [], 33 | "cwd": "${workspaceFolder}" 34 | }, 35 | { 36 | "name": "Debug sqlsync reducer host example", 37 | "type": "lldb", 38 | "request": "launch", 39 | "cargo": { 40 | "args": ["build", "--example=host", "--features=host"], 41 | "filter": { 42 | "name": "host", 43 | "kind": "example" 44 | } 45 | }, 46 | "args": [], 47 | "cwd": "${workspaceFolder}" 48 | } 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "lldb.showDisassembly": "auto", 3 | "lldb.dereferencePointers": true, 4 | "lldb.consoleMode": "commands", 5 | // this allows rust-analyzer to compile sqlsync_reducer::host_ffi 6 | "rust-analyzer.cargo.features": ["host"], 7 | "rust-analyzer.showUnlinkedFileNotification": false, 8 | "[typescriptreact][typescript][javascript][javascriptreact][json][jsonc]": { 9 | "editor.defaultFormatter": "biomejs.biome", 10 | "editor.tabSize": 2, 11 | "editor.insertSpaces": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | This changelog documents changes across multiple projects contained in this monorepo. Each project is released for every SQLSync version, even if the project has not changed. The reason for this decision is to simplify testing and debugging. Lockstep versioning will be relaxed as SQLSync matures. 2 | 3 | # 0.3.2 - Mar 11 2024 4 | 5 | - Fix [#54](https://github.com/orbitinghail/sqlsync/issues/54) 6 | - Better reducer errors 7 | 8 | # 0.3.1 - Mar 9 2024 9 | 10 | - Fixed bug that caused errors to be swallowed 11 | 12 | # 0.3.0 - Jan 7 2023 13 | 14 | - Moved the majority of functionality from `sqlsync-react` to `sqlsync-worker` to make it easier to add additional JS framework support. ([#38]) 15 | - Introduce Reducer trait, allowing non-Wasm reducers to be used with the Coordinator. ([#40]) Contribution by @matthewgapp. 16 | - Allow SQLite to be directly modified in the coordinator via a new mutate_direct method. ([#43]) 17 | - Solid.js library ([#37]) Contribution by @matthewgapp. 18 | 19 | # 0.2.0 - Dec 1 2023 20 | 21 | - Reducer can now handle query errors ([#29]) 22 | 23 | # 0.1.0 - Oct 23 2023 24 | 25 | - Initial release 26 | 27 | [#29]: https://github.com/orbitinghail/sqlsync/pull/29 28 | [#38]: https://github.com/orbitinghail/sqlsync/pull/38 29 | [#40]: https://github.com/orbitinghail/sqlsync/pull/40 30 | [#43]: https://github.com/orbitinghail/sqlsync/pull/43 31 | [#37]: https://github.com/orbitinghail/sqlsync/pull/37 32 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | - Demonstrating empathy and kindness toward other people 21 | - Being respectful of differing opinions, viewpoints, and experiences 22 | - Giving and gracefully accepting constructive feedback 23 | - Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | - Focusing on what is best not just for us as individuals, but for the overall 26 | community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | - The use of sexualized language or imagery, and sexual attention or advances of 31 | any kind 32 | - Trolling, insulting or derogatory comments, and personal or political attacks 33 | - Public or private harassment 34 | - Publishing others' private information, such as a physical or email address, 35 | without their explicit permission 36 | - Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | bhayes@singlestore.com. All complaints will be reviewed and investigated 64 | promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series of 86 | actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or permanent 93 | ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within the 113 | community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by 122 | [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to SQLSync 2 | 3 | This document attempts to explain how to work on SQLSync itself. Buckle up, it's pretty rough and is changing fast. 4 | 5 | ### Dependencies 6 | 7 | - [Just](https://github.com/casey/just) 8 | - [Rust](https://www.rust-lang.org/) 9 | - [wasm-pack](https://rustwasm.github.io/wasm-pack/) 10 | - [node.js](https://nodejs.org/en) 11 | - [pnpm](https://pnpm.io/) 12 | - [llvm](https://llvm.org/) 13 | - [clang](https://clang.llvm.org/) 14 | 15 | ### Build Wasm artifacts 16 | 17 | ```bash 18 | just run-with-prefix 'wasm-' 19 | just wasm-demo-reducer --release 20 | just package-sqlsync-worker dev 21 | ``` 22 | 23 | ### Local Coordinator 24 | 25 | > [!WARNING] 26 | > Currently this seems to require modifying the wrangler.toml config file to point at your own Cloudflare buckets (even though they aren't being used). Work is underway to replace the local coordinator with a wrangler agnostic alternative optimized for local development. 27 | 28 | ```bash 29 | cd demo/cloudflare-backend 30 | pnpm i 31 | pnpm dev 32 | 33 | # then in another shell 34 | just upload-demo-reducer release local 35 | ``` 36 | 37 | ### Local Todo Demo 38 | 39 | ```bash 40 | cd demo/frontend 41 | pnpm i 42 | pnpm dev 43 | ``` 44 | 45 | Then go to http://localhost:5173 46 | 47 | ### Run some tests 48 | 49 | These tests are useful for learning more about how SQLSync works. 50 | 51 | ```bash 52 | just unit-test 53 | just test-end-to-end-local 54 | just test-end-to-end-local-net 55 | just test-sqlsync-reducer 56 | ``` 57 | 58 | You can run all the tests in one command like so: 59 | 60 | ```bash 61 | just test 62 | ``` 63 | 64 | ### Submitting a pull request 65 | 66 | When submitting a pull request, it's appreciated if you run `just lint` as well as the above tests before each change. These commands also will run via GitHub actions which will be enabled on your PR once it's been reviewed. Thanks for your contributions! 67 | 68 | ## Community & Contributing 69 | 70 | If you are interested in contributing to SQLSync, please [join the Discord community][discord] and let us know what you want to build. All contributions will be held to a high standard, and are more likely to be accepted if they are tied to an existing task and agreed upon specification. 71 | 72 | [![Join the SQLSync Community](https://discordapp.com/api/guilds/1149205110262595634/widget.png?style=banner2)][discord] 73 | 74 | [discord]: https://discord.gg/etFk2N9nzC 75 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | 4 | members = [ 5 | "lib/sqlsync", 6 | "lib/sqlsync-worker/sqlsync-wasm", 7 | "lib/sqlsync-reducer", 8 | "lib/sqlite-vfs", 9 | "lib/testutil", 10 | 11 | "examples/reducer-guestbook", 12 | 13 | "demo/demo-reducer", 14 | "demo/cloudflare-backend", 15 | ] 16 | 17 | [workspace.package] 18 | authors = ["Carl Sverre", "orbitinghail"] 19 | edition = "2021" 20 | homepage = "https://sqlsync.dev" 21 | license = "Apache-2.0" 22 | repository = "https://github.com/orbitinghail/sqlsync" 23 | version = "0.3.2" 24 | 25 | [profile.release] 26 | lto = true 27 | strip = "debuginfo" 28 | codegen-units = 1 29 | 30 | [workspace.dependencies] 31 | anyhow = "1.0" 32 | bincode = "1.3" 33 | futures = "0.3" 34 | getrandom = "0.2" 35 | js-sys = "0.3" 36 | web-sys = "0.3" 37 | log = "0.4" 38 | rand = "0.8" 39 | serde = "1.0" 40 | simple_logger = "4.1" 41 | thiserror = "1.0" 42 | time = "0.3" 43 | wasmi = "0.31" 44 | wasm-bindgen = "0.2" 45 | console_error_panic_hook = "0.1" 46 | bs58 = "0.5" 47 | hex = "0.4" 48 | wasm-bindgen-futures = "0.4" 49 | serde_bytes = "0.11" 50 | worker = "0.0.18" 51 | event-listener = "3.0" 52 | sha2 = "0.10.8" 53 | serde-wasm-bindgen = "0.6" 54 | pin-project = "1.1" 55 | 56 | # specific revision of tsify needed for serde updates 57 | tsify = { git = "https://github.com/siefkenj/tsify", rev = "145ed4c8ef6417003e182fad41d1c0f26ed645e5", default-features = false } 58 | 59 | # specific revision of gloo needed for: 60 | # - parse_message receiving a uint8array directly 61 | # - cloudflare compatibility fix for writing to a websocket 62 | [workspace.dependencies.gloo] 63 | git = "https://github.com/carlsverre/gloo" 64 | rev = "8f48a39a0a1e126e3c455525d5b4c51487102333" 65 | features = ["futures"] 66 | 67 | [workspace.dependencies.libsqlite3-sys] 68 | git = "https://github.com/trevyn/rusqlite" 69 | features = ["bundled", "wasm32-unknown-unknown-openbsd-libc"] 70 | # on branch: https://github.com/trevyn/rusqlite/tree/wasm32-unknown-unknown 71 | rev = "504eff51ece3e4f07b2c01c57e2e06602f63bb01" 72 | 73 | [workspace.dependencies.rusqlite] 74 | git = "https://github.com/trevyn/rusqlite" 75 | features = ["bundled", "hooks", "modern_sqlite"] 76 | # on branch: https://github.com/trevyn/rusqlite/tree/wasm32-unknown-unknown 77 | rev = "504eff51ece3e4f07b2c01c57e2e06602f63bb01" 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SQLSync 2 | 3 | [![github actions](https://github.com/orbitinghail/sqlsync/actions/workflows/actions.yaml/badge.svg?branch=main)](https://github.com/orbitinghail/sqlsync/actions?query=branch%3Amain) 4 | [![Join the SQLSync Community](https://discordapp.com/api/guilds/1149205110262595634/widget.png?style=shield)][discord] 5 | 6 | **SQLSync is a collaborative offline-first wrapper around SQLite** designed to synchronize web application state between users, devices, and the edge. 7 | 8 | **Example use cases** 9 | 10 | - A web app with a structured file oriented data model like Figma. Each file could be a SQLSync database, enabling real-time local first collaboration and presence 11 | - Running SQLSync on the edge with high tolerance for unreliable network conditions 12 | - Enabling optimistic mutations on SQLite read replicas 13 | 14 | **SQLSync Demo** 15 | 16 | The best way to get a feel for how SQLSync behaves is to play with the [Todo list demo][todo-demo]. Clicking [this link][todo-demo] will create a unique to-do list and redirect you to its unique URL. You can then share that URL with friends or open it on multiple devices (or browsers) to see the power of offline-first collaborative SQLite. 17 | 18 | [todo-demo]: https://sqlsync-todo.pages.dev/ 19 | 20 | You can also learn more about SQLSync and it's goals by watching Carl's WasmCon 2023 talk. [The recording can be found here][wasmcon-talk]. 21 | 22 | [wasmcon-talk]: https://youtu.be/oLYda9jmNpk?si=7BBBdLxEj9ZQ4OvS 23 | 24 | **Features** 25 | 26 | - Eventually consistent SQLite 27 | - Optimistic reads and writes 28 | - Reactive query subscriptions 29 | - Real-time collaboration 30 | - Offline-first 31 | - Cross-tab sync 32 | - React library 33 | 34 | If you are interested in using or contributing to SQLSync, please [join the Discord community][discord] and let us know what you want to build. We are excited to collaborate with you! 35 | 36 | ## Installation & Getting started 37 | 38 | Please refer to [the guide](./GUIDE.md) to learn how to add SQLSync to your application. 39 | 40 | ## Tips & Tricks 41 | 42 | ### How to debug SQLSync in the browser 43 | By default SQLSync runs in a shared web worker. This allows the database to automatically be shared between different tabs, however results in making SQLSync a bit harder to debug. 44 | 45 | The easiest way is to use Google Chrome, and go to the special URL: [chrome://inspect/#workers](chrome://inspect/#workers). On that page you'll find a list of all the running shared workers in other tabs. Assuming another tab is running SQLSync, you'll see the shared worker listed. Click `inspect` to open up dev-tools for the worker. 46 | 47 | ### My table is missing, or multiple statements aren't executing 48 | SQLSync uses [rusqlite] under the hood to run and query SQLite. Unfortunately, the `execute` method only supports single statements and silently ignores trailing statements. Thus, if you are using `execute!(...)` in your reducer, make sure that each call only runs a single SQL statement. 49 | 50 | For example: 51 | ```rust 52 | // DON'T DO THIS: 53 | execute!("create table foo (id int); create table bar (id int);").await?; 54 | 55 | // DO THIS: 56 | execute!("create table foo (id int)").await?; 57 | execute!("create table bar (id int)").await?; 58 | ``` 59 | 60 | ## Community & Contributing 61 | 62 | If you are interested in contributing to SQLSync, please [join the Discord community][discord] and let us know what you want to build. All contributions will be held to a high standard, and are more likely to be accepted if they are tied to an existing task and agreed upon specification. 63 | 64 | [![Join the SQLSync Community](https://discordapp.com/api/guilds/1149205110262595634/widget.png?style=banner2)][discord] 65 | 66 | [discord]: https://discord.gg/etFk2N9nzC 67 | [rusqlite]: https://github.com/rusqlite/rusqlite 68 | -------------------------------------------------------------------------------- /biome.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://biomejs.dev/schemas/1.5.0/schema.json", 3 | "organizeImports": { 4 | "enabled": false 5 | }, 6 | "vcs": { 7 | "enabled": true, 8 | "clientKind": "git", 9 | "useIgnoreFile": true, 10 | "defaultBranch": "main" 11 | }, 12 | "linter": { 13 | "enabled": true, 14 | "rules": { 15 | "recommended": true, 16 | "style": { 17 | "useSingleVarDeclarator": "off" 18 | } 19 | } 20 | }, 21 | "formatter": { 22 | "enabled": true, 23 | "formatWithErrors": false, 24 | "indentStyle": "space", 25 | "indentWidth": 2, 26 | "lineWidth": 100 27 | }, 28 | "json": { 29 | "parser": { 30 | "allowComments": true 31 | }, 32 | "formatter": { 33 | "enabled": true, 34 | "indentStyle": "space", 35 | "indentWidth": 2, 36 | "lineWidth": 100 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /demo/cloudflare-backend/.gitignore: -------------------------------------------------------------------------------- 1 | /.mf 2 | /.wrangler 3 | -------------------------------------------------------------------------------- /demo/cloudflare-backend/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sqlsync-cloudflare-backend" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | crate-type = ["cdylib"] 8 | 9 | [dependencies] 10 | gloo = { workspace = true, features = ["cloudflare"] } 11 | futures.workspace = true 12 | worker.workspace = true 13 | console_error_panic_hook.workspace = true 14 | sqlsync = { path = "../../lib/sqlsync" } 15 | bincode.workspace = true 16 | serde-wasm-bindgen.workspace = true 17 | serde_bytes.workspace = true 18 | anyhow.workspace = true 19 | wasm-bindgen.workspace = true 20 | js-sys.workspace = true 21 | bs58.workspace = true 22 | 23 | web-sys = { workspace = true, features = ["Crypto", "SubtleCrypto"] } 24 | -------------------------------------------------------------------------------- /demo/cloudflare-backend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqlsync-cloudflare-backend", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "wrangler dev", 7 | "wrangler-deploy": "wrangler deploy", 8 | "wrangler-tail": "wrangler tail" 9 | }, 10 | "devDependencies": { 11 | "wrangler": "^3.32.0" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /demo/cloudflare-backend/src/persistence.rs: -------------------------------------------------------------------------------- 1 | use std::io::Cursor; 2 | 3 | use js_sys::Uint8Array; 4 | use sqlsync::{replication::ReplicationDestination, JournalId, Lsn, LsnRange}; 5 | use wasm_bindgen::JsValue; 6 | use worker::*; 7 | 8 | const RANGE_KEY: &str = "RANGE"; 9 | 10 | pub struct Persistence { 11 | /// The range of lsns that have been written to storage 12 | range: LsnRange, 13 | storage: Storage, 14 | } 15 | 16 | impl Persistence { 17 | pub async fn init(mut storage: Storage) -> Result { 18 | let range = match storage.get::(RANGE_KEY).await { 19 | Ok(range) => range, 20 | Err(_) => { 21 | let range = LsnRange::empty(); 22 | storage.put(RANGE_KEY, &range).await?; 23 | range 24 | } 25 | }; 26 | Ok(Self { range, storage }) 27 | } 28 | 29 | /// the next lsn that should be written to storage 30 | pub fn expected_lsn(&self) -> Lsn { 31 | self.range.next() 32 | } 33 | 34 | pub async fn write_lsn(&mut self, lsn: Lsn, frame: Vec) -> Result<()> { 35 | let obj = js_sys::Object::new(); 36 | 37 | // get the new range, assuming the write goes through 38 | let new_range = self.range.append(lsn); 39 | 40 | // convert our range into a jsvalue 41 | let range = serde_wasm_bindgen::to_value(&new_range) 42 | .map_err(|e| Error::RustError(e.to_string()))?; 43 | 44 | js_sys::Reflect::set(&obj, &JsValue::from_str(RANGE_KEY), &range)?; 45 | 46 | // convert frame into a uint8array 47 | let uint8_array = Uint8Array::from(frame.as_slice()); 48 | let key = format!("lsn-{}", lsn); 49 | js_sys::Reflect::set(&obj, &JsValue::from_str(&key), &uint8_array)?; 50 | 51 | // write to storage 52 | self.storage.put_multiple_raw(obj).await?; 53 | 54 | // update our in-memory range 55 | self.range = new_range; 56 | Ok(()) 57 | } 58 | 59 | pub async fn replay( 60 | &self, 61 | id: JournalId, 62 | dest: &mut T, 63 | ) -> Result<()> { 64 | for lsn in 0..self.range.next() { 65 | console_log!("replaying lsn {}", lsn); 66 | let key = format!("lsn-{}", lsn); 67 | let mut frame = Cursor::new(self.storage.get::(&key).await?); 68 | dest.write_lsn(id, lsn, &mut frame) 69 | .map_err(|e| Error::RustError(e.to_string()))?; 70 | } 71 | Ok(()) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /demo/cloudflare-backend/wrangler.toml: -------------------------------------------------------------------------------- 1 | name = "sqlsync" 2 | main = "build/worker/shim.mjs" 3 | compatibility_date = "2023-08-14" 4 | compatibility_flags = ["web_socket_compression"] 5 | 6 | r2_buckets = [ 7 | { binding = "SQLSYNC_REDUCERS", bucket_name = "sqlsync-reducers", preview_bucket_name = "sqlsync-reducers-dev" }, 8 | ] 9 | 10 | [build] 11 | # TODO: automatically switch between these two commands based on the environment 12 | # command = "cargo install -q worker-build && worker-build --dev" 13 | command = "cargo install -q worker-build && worker-build" 14 | 15 | [durable_objects] 16 | bindings = [{ name = "COORDINATOR", class_name = "DocumentCoordinator" }] 17 | 18 | [[migrations]] 19 | tag = "2023-08-25" 20 | new_classes = ["DocumentCoordinator"] 21 | -------------------------------------------------------------------------------- /demo/demo-reducer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "demo-reducer" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | crate-type = ["cdylib"] 8 | 9 | [dependencies] 10 | sqlsync-reducer = { path = "../../lib/sqlsync-reducer" } 11 | serde = { version = "1.0", features = ["derive"] } 12 | serde_json = "1.0" 13 | log = "0.4" 14 | -------------------------------------------------------------------------------- /demo/demo-reducer/src/lib.rs: -------------------------------------------------------------------------------- 1 | // build: "cargo build --target wasm32-unknown-unknown -p demo-reducer --release" 2 | 3 | use serde::Deserialize; 4 | use sqlsync_reducer::{execute, init_reducer, types::ReducerError}; 5 | 6 | #[derive(Deserialize, Debug)] 7 | #[serde(tag = "tag")] 8 | enum Mutation { 9 | InitSchema, 10 | 11 | CreateTask { id: String, description: String }, 12 | 13 | DeleteTask { id: String }, 14 | 15 | ToggleCompleted { id: String }, 16 | } 17 | 18 | init_reducer!(reducer); 19 | async fn reducer(mutation: Vec) -> Result<(), ReducerError> { 20 | let mutation: Mutation = serde_json::from_slice(&mutation[..])?; 21 | 22 | match mutation { 23 | Mutation::InitSchema => { 24 | execute!( 25 | "CREATE TABLE IF NOT EXISTS tasks ( 26 | id TEXT PRIMARY KEY, 27 | description TEXT NOT NULL, 28 | completed BOOLEAN NOT NULL, 29 | created_at TEXT NOT NULL 30 | )" 31 | ) 32 | .await?; 33 | } 34 | 35 | Mutation::CreateTask { id, description } => { 36 | log::debug!("appending task({}): {}", id, description); 37 | execute!( 38 | "insert into tasks (id, description, completed, created_at) 39 | values (?, ?, false, datetime('now'))", 40 | id, 41 | description 42 | ) 43 | .await?; 44 | } 45 | 46 | Mutation::DeleteTask { id } => { 47 | execute!("delete from tasks where id = ?", id).await?; 48 | } 49 | 50 | Mutation::ToggleCompleted { id } => { 51 | execute!( 52 | "update tasks set completed = not completed where id = ?", 53 | id 54 | ) 55 | .await?; 56 | } 57 | } 58 | 59 | Ok(()) 60 | } 61 | -------------------------------------------------------------------------------- /demo/frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /demo/frontend/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | SQLSync Demo 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /demo/frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frontend", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "tsc && vite build", 9 | "lint": "biome check .", 10 | "preview": "vite preview", 11 | "release": "pnpm build && wrangler pages deploy dist" 12 | }, 13 | "dependencies": { 14 | "@mantine/code-highlight": "^7.6.1", 15 | "@mantine/core": "^7.6.1", 16 | "@mantine/form": "^7.6.1", 17 | "@mantine/hooks": "^7.6.1", 18 | "@orbitinghail/sqlsync-react": "workspace:*", 19 | "@orbitinghail/sqlsync-worker": "workspace:*", 20 | "@tabler/icons-react": "^2.47.0", 21 | "react": "^18.2.0", 22 | "react-dom": "^18.2.0", 23 | "react-github-btn": "^1.4.0", 24 | "react-qr-code": "^2.0.12", 25 | "react-router-dom": "^6.22.3", 26 | "uuid": "^9.0.1" 27 | }, 28 | "devDependencies": { 29 | "@types/node": "^20.11.25", 30 | "@types/react": "^18.2.64", 31 | "@types/react-dom": "^18.2.21", 32 | "@types/uuid": "^9.0.8", 33 | "@vitejs/plugin-react": "^4.2.1", 34 | "autoprefixer": "^10.4.18", 35 | "postcss": "^8.4.35", 36 | "postcss-preset-mantine": "^1.13.0", 37 | "typescript": "^5.4.2", 38 | "vite": "^5.1.5", 39 | "wrangler": "^3.32.0" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /demo/frontend/postcss.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | plugins: { 3 | "postcss-preset-mantine": {}, 4 | }, 5 | }; 6 | -------------------------------------------------------------------------------- /demo/frontend/public/favicon-dark-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orbitinghail/sqlsync/e8ad1168a2931087b308a8db1a0edba2d73bc213/demo/frontend/public/favicon-dark-32x32.png -------------------------------------------------------------------------------- /demo/frontend/public/favicon-light-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orbitinghail/sqlsync/e8ad1168a2931087b308a8db1a0edba2d73bc213/demo/frontend/public/favicon-light-32x32.png -------------------------------------------------------------------------------- /demo/frontend/public/favicon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 10 | 11 | -------------------------------------------------------------------------------- /demo/frontend/public/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: / 3 | -------------------------------------------------------------------------------- /demo/frontend/src/App.tsx: -------------------------------------------------------------------------------- 1 | import { Container, Stack } from "@mantine/core"; 2 | import { JournalId } from "@orbitinghail/sqlsync-worker"; 3 | import { useEffect } from "react"; 4 | import { Header } from "./components/Header"; 5 | import { QueryViewer } from "./components/QueryViewer"; 6 | import { TaskList } from "./components/TaskList"; 7 | import { useMutate } from "./doctype"; 8 | 9 | export const App = ({ docId }: { docId: JournalId }) => { 10 | const mutate = useMutate(docId); 11 | 12 | useEffect(() => { 13 | mutate({ tag: "InitSchema" }).catch((err) => { 14 | console.error("Failed to init schema", err); 15 | }); 16 | }, [mutate]); 17 | 18 | return ( 19 | 20 | 21 |
22 | 23 | 24 | 25 | 26 | ); 27 | }; 28 | -------------------------------------------------------------------------------- /demo/frontend/src/components/ConnectionStatus.tsx: -------------------------------------------------------------------------------- 1 | import { Button, rem } from "@mantine/core"; 2 | import { useConnectionStatus } from "@orbitinghail/sqlsync-react"; 3 | import { JournalId } from "@orbitinghail/sqlsync-worker"; 4 | import { IconWifi, IconWifiOff } from "@tabler/icons-react"; 5 | import { ReactElement, useCallback } from "react"; 6 | import { useSetConnectionEnabled } from "../doctype"; 7 | 8 | export const ConnectionStatus = ({ docId }: { docId: JournalId }) => { 9 | const status = useConnectionStatus(); 10 | const setConnectionEnabled = useSetConnectionEnabled(docId); 11 | 12 | const handleClick = useCallback(() => { 13 | if (status === "disabled") { 14 | setConnectionEnabled(true).catch((err) => { 15 | console.error("Failed to enable connection", err); 16 | }); 17 | } else { 18 | setConnectionEnabled(false).catch((err) => { 19 | console.error("Failed to disable connection", err); 20 | }); 21 | } 22 | }, [status, setConnectionEnabled]); 23 | 24 | let color: string, 25 | icon: ReactElement | undefined, 26 | loading = false; 27 | switch (status) { 28 | case "disabled": 29 | color = "gray"; 30 | icon = ; 31 | break; 32 | case "disconnected": 33 | color = "gray"; 34 | icon = ; 35 | break; 36 | case "connecting": 37 | color = "yellow"; 38 | loading = true; 39 | break; 40 | case "connected": 41 | color = "green"; 42 | icon = ; 43 | break; 44 | } 45 | 46 | return ( 47 | 57 | ); 58 | }; 59 | -------------------------------------------------------------------------------- /demo/frontend/src/components/Header.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | ActionIcon, 3 | Anchor, 4 | Center, 5 | Flex, 6 | Paper, 7 | Popover, 8 | Stack, 9 | Text, 10 | Title, 11 | } from "@mantine/core"; 12 | import { IconQrcode } from "@tabler/icons-react"; 13 | import GitHubButton from "react-github-btn"; 14 | import QRCode from "react-qr-code"; 15 | 16 | const SQLSYNC_URL = "https://sqlsync.dev"; 17 | 18 | export const Header = () => { 19 | return ( 20 | <> 21 | 22 | 23 |
24 | SQLSync Demo 25 |
26 | 32 | Star 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | SQLSync is a collaborative offline-first wrapper 47 | around SQLite. It is designed to synchronize web application state between users, devices, 48 | and the edge. 49 | 50 | 51 | 52 | ); 53 | }; 54 | -------------------------------------------------------------------------------- /demo/frontend/src/components/QueryViewer.tsx: -------------------------------------------------------------------------------- 1 | import { CodeHighlight } from "@mantine/code-highlight"; 2 | import { Alert, Button, Code, Collapse, Paper, Textarea } from "@mantine/core"; 3 | import { useDisclosure } from "@mantine/hooks"; 4 | import { JournalId } from "@orbitinghail/sqlsync-worker"; 5 | import { IconAlertCircle, IconCaretDownFilled, IconCaretRightFilled } from "@tabler/icons-react"; 6 | import { ReactElement, useMemo, useState } from "react"; 7 | import { useQuery } from "../doctype"; 8 | 9 | interface Props { 10 | docId: JournalId; 11 | } 12 | 13 | export const QueryViewerInner = ({ docId }: Props) => { 14 | const [inputValue, setInputValue] = useState("select * from tasks"); 15 | const result = useQuery(docId, inputValue); 16 | 17 | const rowsJson = useMemo(() => { 18 | return JSON.stringify( 19 | result.rows ?? [], 20 | (_, value) => { 21 | // handle bigint values 22 | if (typeof value === "bigint") { 23 | return value.toString(); 24 | } 25 | return value; 26 | }, 27 | 2, 28 | ); 29 | }, [result.rows]); 30 | 31 | let output: ReactElement; 32 | if (result.state === "error") { 33 | output = ( 34 | } p="sm"> 35 | {result.error.message} 36 | 37 | ); 38 | } else { 39 | output = ; 40 | } 41 | 42 | return ( 43 | <> 44 |