├── examples ├── react │ ├── index.js │ ├── src │ │ ├── binding_test.js │ │ ├── App.jsx │ │ ├── main.jsx │ │ └── worker.js │ ├── index.html │ ├── vite.config.js │ ├── package.json │ └── eslint.config.js ├── deno │ ├── src │ │ ├── binding_test.js │ │ ├── App.jsx │ │ ├── main.jsx │ │ └── worker.js │ ├── index.html │ ├── vite.config.js │ └── deno.json └── wasm │ ├── simple.html │ └── worker.js ├── tests └── sqllogictest │ └── Cargo.toml ├── .gitignore ├── .cargo └── config.toml ├── src ├── executor │ ├── mod.rs │ ├── wasm.rs │ └── tokio.rs ├── lib.rs ├── connection.rs ├── utils.rs └── db.rs ├── Cargo.toml ├── README.md └── LICENSE /examples/react/index.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sqllogictest/Cargo.toml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | node_modules 3 | -------------------------------------------------------------------------------- /examples/react/src/binding_test.js: -------------------------------------------------------------------------------- 1 | 2 | new Worker(new URL('./worker.js', import.meta.url)); 3 | 4 | -------------------------------------------------------------------------------- /examples/deno/src/binding_test.js: -------------------------------------------------------------------------------- 1 | 2 | const worker = new Worker(new URL('./worker.js', import.meta.url)); 3 | 4 | -------------------------------------------------------------------------------- /examples/deno/src/App.jsx: -------------------------------------------------------------------------------- 1 | 2 | function App() { 3 | 4 | return ( 5 | <> 6 |

Please see output in console

7 | 8 | ) 9 | } 10 | 11 | export default App 12 | -------------------------------------------------------------------------------- /examples/react/src/App.jsx: -------------------------------------------------------------------------------- 1 | 2 | function App() { 3 | 4 | return ( 5 | <> 6 |

Please see output in console

7 | 8 | ) 9 | } 10 | 11 | export default App 12 | -------------------------------------------------------------------------------- /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [target.wasm32-unknown-unknown] 2 | rustflags = ["-C", "target-feature=+atomics,+bulk-memory,+mutable-globals", "--cfg", 'getrandom_backend="wasm_js"'] 3 | 4 | [unstable] 5 | build-std = ["panic_abort", "std"] 6 | -------------------------------------------------------------------------------- /examples/wasm/simple.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Title 6 | 7 | 8 | 9 | 12 | 13 | -------------------------------------------------------------------------------- /examples/react/src/main.jsx: -------------------------------------------------------------------------------- 1 | import { StrictMode } from 'react' 2 | import { createRoot } from 'react-dom/client' 3 | import App from './App.jsx' 4 | import './binding_test.js' 5 | 6 | createRoot(document.getElementById('root')).render( 7 | 8 | 9 | , 10 | ) 11 | -------------------------------------------------------------------------------- /examples/deno/src/main.jsx: -------------------------------------------------------------------------------- 1 | // @deno-types="@types/react" 2 | import { StrictMode } from 'react' 3 | // @deno-types="@types/react-dom/client" 4 | import { createRoot } from 'react-dom/client' 5 | import App from './App.jsx' 6 | import './binding_test.js' 7 | 8 | createRoot(document.getElementById('root')).render( 9 | 10 | 11 | , 12 | ) 13 | -------------------------------------------------------------------------------- /examples/deno/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Tonbolite react example 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /examples/react/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | sqlite-tonbo react example 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /examples/react/vite.config.js: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import react from '@vitejs/plugin-react' 3 | 4 | // https://vite.dev/config/ 5 | export default defineConfig({ 6 | plugins: [react()], 7 | server: { 8 | headers: { 9 | "Cross-Origin-Opener-Policy": "same-origin", 10 | "Cross-Origin-Opener-Policy-Report-Only": "same-origin", 11 | "Cross-Origin-Embedder-Policy": "require-corp" 12 | } 13 | } 14 | }) 15 | -------------------------------------------------------------------------------- /src/executor/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "tokio")] 2 | pub(crate) mod tokio; 3 | #[cfg(feature = "wasm")] 4 | pub(crate) mod wasm; 5 | 6 | use std::future::Future; 7 | 8 | #[cfg(not(feature = "wasm"))] 9 | pub type SQLiteExecutor = tokio::TokioExecutor; 10 | #[cfg(feature = "wasm")] 11 | pub type SQLiteExecutor = tonbo::executor::opfs::OpfsExecutor; 12 | 13 | pub trait BlockOnExecutor: tonbo::executor::Executor { 14 | fn block_on(&self, future: F) -> F::Output; 15 | } 16 | -------------------------------------------------------------------------------- /examples/deno/vite.config.js: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import deno from '@deno/vite-plugin' 3 | import react from '@vitejs/plugin-react' 4 | 5 | // https://vite.dev/config/ 6 | export default defineConfig({ 7 | plugins: [deno(), react()], 8 | server: { 9 | headers: { 10 | "Cross-Origin-Opener-Policy": "same-origin", 11 | "Cross-Origin-Opener-Policy-Report-Only": "same-origin", 12 | "Cross-Origin-Embedder-Policy": "require-corp" 13 | } 14 | } 15 | }) 16 | -------------------------------------------------------------------------------- /src/executor/wasm.rs: -------------------------------------------------------------------------------- 1 | use crate::executor::BlockOnExecutor; 2 | use fusio::MaybeSend; 3 | use tonbo::executor::Executor; 4 | 5 | use std::future::Future; 6 | 7 | #[derive(Clone, Default)] 8 | pub struct WasmExecutor; 9 | 10 | impl Executor for WasmExecutor { 11 | fn spawn(&self, future: F) 12 | where 13 | F: Future + MaybeSend + 'static, 14 | { 15 | wasm_bindgen_futures::spawn_local(future) 16 | } 17 | } 18 | 19 | impl BlockOnExecutor for WasmExecutor { 20 | fn block_on(&self, _future: F) -> F::Output { 21 | todo!() 22 | } 23 | } 24 | 25 | impl WasmExecutor { 26 | pub fn new() -> Self { 27 | Default::default() 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /examples/react/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "react", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "cp -r ../../pkg ./src/ && vite", 8 | "build": "vite build", 9 | "lint": "eslint .", 10 | "preview": "vite preview" 11 | }, 12 | "dependencies": { 13 | "react": "^18.3.1", 14 | "react-dom": "^18.3.1" 15 | }, 16 | "devDependencies": { 17 | "@eslint/js": "^9.15.0", 18 | "@types/react": "^18.3.12", 19 | "@types/react-dom": "^18.3.1", 20 | "@vitejs/plugin-react": "^4.3.4", 21 | "eslint": "^9.15.0", 22 | "eslint-plugin-react": "^7.37.2", 23 | "eslint-plugin-react-hooks": "^5.0.0", 24 | "eslint-plugin-react-refresh": "^0.4.14", 25 | "globals": "^15.12.0", 26 | "vite": "^6.0.1" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /examples/deno/src/worker.js: -------------------------------------------------------------------------------- 1 | (async () => { 2 | 3 | const tonbo = await import("./pkg/sqlite_tonbo.js"); 4 | await tonbo.default(); 5 | 6 | conn = new tonbo.Connection(); 7 | 8 | await conn.create(`CREATE VIRTUAL TABLE temp.tonbo USING tonbo( 9 | create_sql ='create table tonbo(id bigint primary key, name varchar, like int)', 10 | path = 'db_path/tonbo' 11 | );`); 12 | 13 | for (let i = 0; i < 10; i++) { 14 | await conn.insert( 15 | `INSERT INTO tonbo (id, name, like) VALUES (${i}, 'lol', ${i})` 16 | ); 17 | } 18 | 19 | await conn.delete("DELETE FROM tonbo WHERE id = 4"); 20 | await conn.update("UPDATE tonbo SET name = 'tonbo' WHERE id = 6"); 21 | 22 | const rows = await conn.select("SELECT * FROM tonbo limit 10"); 23 | console.log(rows); 24 | 25 | })(); 26 | 27 | 28 | -------------------------------------------------------------------------------- /examples/wasm/worker.js: -------------------------------------------------------------------------------- 1 | async function initWorker() { 2 | const tonbo = await import("./pkg/sqlite_tonbo.js"); 3 | await tonbo.default(); 4 | 5 | let conn = new tonbo.Connection(); 6 | 7 | await conn.create(`CREATE VIRTUAL TABLE temp.tonbo USING tonbo( 8 | create_sql ='create table tonbo(id bigint primary key, name varchar, like int)', 9 | path = 'db_path/tonbo' 10 | );`); 11 | 12 | for (let i = 0; i < 10; i ++) { 13 | if (i % 100 === 0) { 14 | await conn.flush("tonbo"); 15 | } 16 | await conn.insert( 17 | `INSERT INTO tonbo (id, name, like) VALUES (${i}, 'lol', ${i})` 18 | ); 19 | } 20 | const rows = await conn.select("SELECT * FROM tonbo limit 10"); 21 | console.log(rows) 22 | 23 | } 24 | 25 | 26 | 27 | initWorker() 28 | -------------------------------------------------------------------------------- /examples/react/src/worker.js: -------------------------------------------------------------------------------- 1 | let conn; 2 | 3 | (async () => { 4 | const tonbo = await import("./pkg/sqlite_tonbo.js"); 5 | await tonbo.default(); 6 | 7 | conn = new tonbo.Connection(); 8 | 9 | await conn.create(`CREATE VIRTUAL TABLE temp.tonbo USING tonbo( 10 | create_sql ='create table tonbo(id bigint primary key, name varchar, like int)', 11 | path = 'db_path/tonbo' 12 | );`); 13 | 14 | for (let i = 0; i < 10; i++) { 15 | await conn.insert( 16 | `INSERT INTO tonbo (id, name, like) VALUES (${i}, 'lol', ${i})` 17 | ); 18 | } 19 | 20 | await conn.delete("DELETE FROM tonbo WHERE id = 4"); 21 | await conn.update("UPDATE tonbo SET name = 'tonbo' WHERE id = 6"); 22 | 23 | const rows = await conn.select("SELECT * FROM tonbo limit 10"); 24 | console.log(rows); 25 | 26 | })(); 27 | 28 | 29 | -------------------------------------------------------------------------------- /examples/deno/deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "tasks": { 3 | "dev": "cp -r ../../pkg ./src/ && deno run -A --node-modules-dir npm:vite", 4 | "build": "deno run -A --node-modules-dir npm:vite build", 5 | "preview": "deno run -A --node-modules-dir npm:vite preview", 6 | "serve": "deno run --allow-net --allow-read jsr:@std/http@1/file-server dist/" 7 | }, 8 | "compilerOptions": { 9 | "lib": ["ES2020", "DOM", "DOM.Iterable"], 10 | "jsx": "react-jsx", 11 | "jsxImportSource": "react", 12 | "jsxImportSourceTypes": "@types/react" 13 | }, 14 | "imports": { 15 | "@deno/vite-plugin": "npm:@deno/vite-plugin@^1.0.0", 16 | "@types/react": "npm:@types/react@^18.3.12", 17 | "@types/react-dom": "npm:@types/react-dom@^18.3.1", 18 | "@vitejs/plugin-react": "npm:@vitejs/plugin-react@^4.3.4", 19 | "react": "npm:react@^18.3.1", 20 | "react-dom": "npm:react-dom@^18.3.1", 21 | "vite": "npm:vite@^6.0.1" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/executor/tokio.rs: -------------------------------------------------------------------------------- 1 | use crate::executor::BlockOnExecutor; 2 | use fusio::MaybeSend; 3 | use std::future::Future; 4 | use std::sync::Arc; 5 | use tokio::runtime::{Builder, Runtime}; 6 | use tonbo::executor::Executor; 7 | 8 | #[derive(Clone)] 9 | pub struct TokioExecutor { 10 | pub(crate) runtime: Arc, 11 | } 12 | 13 | impl Executor for TokioExecutor { 14 | fn spawn(&self, future: F) 15 | where 16 | F: Future + MaybeSend + 'static, 17 | { 18 | self.runtime.spawn(future); 19 | } 20 | } 21 | 22 | impl BlockOnExecutor for TokioExecutor { 23 | fn block_on(&self, future: F) -> F::Output { 24 | self.runtime.block_on(future) 25 | } 26 | } 27 | 28 | impl TokioExecutor { 29 | pub fn new() -> Self { 30 | let runtime = Arc::new( 31 | Builder::new_multi_thread() 32 | .worker_threads(4) 33 | .enable_all() 34 | .build() 35 | .unwrap(), 36 | ); 37 | Self { runtime } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /examples/react/eslint.config.js: -------------------------------------------------------------------------------- 1 | import js from '@eslint/js' 2 | import globals from 'globals' 3 | import react from 'eslint-plugin-react' 4 | import reactHooks from 'eslint-plugin-react-hooks' 5 | import reactRefresh from 'eslint-plugin-react-refresh' 6 | 7 | export default [ 8 | { ignores: ['dist'] }, 9 | { 10 | files: ['**/*.{js,jsx}'], 11 | languageOptions: { 12 | ecmaVersion: 2020, 13 | globals: globals.browser, 14 | parserOptions: { 15 | ecmaVersion: 'latest', 16 | ecmaFeatures: { jsx: true }, 17 | sourceType: 'module', 18 | }, 19 | }, 20 | settings: { react: { version: '18.3' } }, 21 | plugins: { 22 | react, 23 | 'react-hooks': reactHooks, 24 | 'react-refresh': reactRefresh, 25 | }, 26 | rules: { 27 | ...js.configs.recommended.rules, 28 | ...react.configs.recommended.rules, 29 | ...react.configs['jsx-runtime'].rules, 30 | ...reactHooks.configs.recommended.rules, 31 | 'react/jsx-no-target-blank': 'off', 32 | 'react-refresh/only-export-components': [ 33 | 'warn', 34 | { allowConstantExport: true }, 35 | ], 36 | }, 37 | }, 38 | ] 39 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | mod executor; 2 | 3 | #[cfg(feature = "wasm")] 4 | mod connection; 5 | #[cfg(feature = "wasm")] 6 | pub use connection::*; 7 | 8 | mod db; 9 | 10 | mod utils; 11 | use crate::db::{DbState, TonboTable}; 12 | use rusqlite::{ffi, vtab::update_module, Connection, Result}; 13 | use std::sync::Arc; 14 | #[cfg(feature = "loadable_extension")] 15 | use std::os::raw::{c_char, c_int}; 16 | 17 | #[no_mangle] 18 | #[cfg(feature = "loadable_extension")] 19 | pub unsafe extern "C" fn sqlite3_sqlitetonbo_init( 20 | db: *mut ffi::sqlite3, 21 | pz_err_msg: *mut *mut c_char, 22 | p_api: *mut ffi::sqlite3_api_routines, 23 | ) -> c_int { 24 | Connection::extension_init2(db, pz_err_msg, p_api, extension_init) 25 | } 26 | 27 | fn extension_init(db: Connection) -> Result { 28 | let aux = Some(Arc::new(DbState::new())); 29 | db.create_module("tonbo", update_module::(), aux)?; 30 | 31 | Ok(true) 32 | } 33 | 34 | pub fn load_module(conn: &Connection) -> Result<()> { 35 | let aux = Some(Arc::new(DbState::new())); 36 | conn.create_module("tonbo", update_module::(), aux) 37 | } 38 | 39 | #[cfg(feature = "wasm")] 40 | #[wasm_bindgen::prelude::wasm_bindgen(start)] 41 | pub fn startup() { 42 | console_log::init().expect("could not initialize logger"); 43 | console_error_panic_hook::set_once(); 44 | } 45 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sqlite-tonbo" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | path = "src/lib.rs" 8 | crate-type = ["cdylib", "rlib"] 9 | 10 | [features] 11 | default = ["tokio"] 12 | tokio = ["dep:tokio", "tonbo/tokio", "tonbo/tokio-http"] 13 | wasm = [ 14 | "dep:wasm_thread", 15 | "dep:wasm-bindgen", 16 | "dep:wasm-bindgen-futures", 17 | "fusio/no-send", 18 | "tonbo/wasm", 19 | "tonbo/sync" 20 | ] 21 | loadable_extension = ["rusqlite/functions", "rusqlite/loadable_extension"] 22 | 23 | [dependencies] 24 | flume = "0.11" 25 | fusio = { version = "=0.3.4", package = "fusio", features = [ 26 | "dyn", 27 | "fs", 28 | "aws", 29 | ] } 30 | fusio-dispatch = { version = "=0.3.4", package = "fusio-dispatch", features = ["aws"] } 31 | fusio-parquet = { version = "=0.3.4", package = "fusio-parquet" } 32 | futures-util = "0.3" 33 | tokio = { version = "1.41", optional = true, features = ["rt-multi-thread"] } 34 | tonbo = { git = "https://github.com/tonbo-io/tonbo", rev = "8c3ad6ffe0d40c8be1332ccbcf9d230f838f73ba", package = "tonbo", default-features = false, features = ["aws"] } 35 | rusqlite = { version = "0.32.1", features = [ 36 | "vtab", 37 | "bundled", 38 | ] } 39 | sqlparser = "0.52" 40 | 41 | wasm-bindgen = { version = "0.2.93", optional = true } 42 | wasm-bindgen-futures = { version = "0.4", optional = true } 43 | wasm_thread = { version = "0.3", optional = true } 44 | 45 | [target.'cfg(target_arch = "wasm32")'.dependencies] 46 | getrandom = { version = "0.3.1", features = ["wasm_js"] } 47 | log = "0.4" 48 | console_log = { version = "1.0", features = ["color"] } 49 | console_error_panic_hook = "0.1.7" 50 | 51 | [dev-dependencies] 52 | wasm-bindgen-test = "0.3" 53 | 54 | [patch.crates-io.rusqlite] 55 | git = "https://github.com/tonbo-io/rusqlite" 56 | branch = "feat/integrity" 57 | 58 | [patch.crates-io.wasm_thread] 59 | git = "https://github.com/Twey/wasm_thread" 60 | branch = "post-message" 61 | 62 | [profile.release] 63 | codegen-units = 1 64 | lto = true 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 | 4 | 5 |

6 | 7 |

8 | License - Apache 2.0 9 | Status - Alpha 10 | Chat - Discord 11 | 12 |

13 | 14 |
15 | 16 | # TonboLite 17 | 18 | TonboLite is a WASM compatible SQLite extension that allows users to create tables which supports analytical processing directly in SQLite. Its storage engine is powered by our open-source embedded key-value database, [Tonbo](https://github.com/tonbo-io/tonbo). 19 | 20 | ## Features 21 | - Organizing [Parquet](https://parquet.apache.org/) files using [Log-Structured Merge Tree](https://en.wikipedia.org/wiki/Log-structured_merge-tree) for fast writing 22 | - Supports OPFS, S3 as remote storage or mixed them together as storage back-end. 23 | - Compatibility with WebAssembly 24 | 25 | ## Usage 26 | 27 | ### Use in SQLite CLI 28 | Use [`.load`](https://www.sqlite.org/cli.html#loading_extensions) command to load a SQLite extension 29 | ```bash 30 | sqlite> .load target/release/libsqlite_tonbo 31 | 32 | sqlite> CREATE VIRTUAL TABLE temp.tonbo USING tonbo( 33 | create_sql = 'create table tonbo(id bigint primary key, name varchar, like int)', 34 | path = 'db_path/tonbo' 35 | ); 36 | sqlite> insert into tonbo (id, name, like) values (0, 'tonbo', 100); 37 | sqlite> insert into tonbo (id, name, like) values (1, 'sqlite', 200); 38 | 39 | sqlite> select * from tonbo; 40 | 0|tonbo|100 41 | 1|sqlite|200 42 | 43 | sqlite> update tonbo set like = 123 where id = 0; 44 | 45 | sqlite> select * from tonbo; 46 | 0|tonbo|123 47 | 1|sqlite|200 48 | 49 | sqlite> delete from tonbo where id = 0; 50 | 51 | sqlite> select * from tonbo; 52 | 1|sqlite|200 53 | ``` 54 | 55 | Or use SQLite extension in Python: 56 | ```python 57 | import sqlite3 58 | 59 | conn = sqlite3.connect(":memory") 60 | conn.enable_load_extension(True) 61 | # Load the tonbolite extension 62 | conn.load_extension("target/release/libsqlite_tonbo.dylib") 63 | con.enable_load_extension(False) 64 | 65 | conn.execute("CREATE VIRTUAL TABLE temp.tonbo USING tonbo(" 66 | "create_sql = 'create table tonbo(id bigint primary key, name varchar, like int)', " 67 | "path = 'db_path/tonbo'" 68 | ")") 69 | conn.execute("INSERT INTO tonbo (id, name, like) VALUES (0, 'lol', 1)") 70 | conn.execute("INSERT INTO tonbo (id, name, like) VALUES (1, 'lol', 100)") 71 | rows = conn.execute("SELECT * FROM tonbo;") 72 | for row in rows: 73 | print(row) 74 | ``` 75 | 76 | ### Use in Rust 77 | TonboLite is able to be used just like a regular SQLite program. 78 | > Please use our Rusqlite patch 79 | > ```toml 80 | > [patch.crates-io.rusqlite] 81 | > git = "https://github.com/tonbo-io/rusqlite" 82 | > branch = "feat/integrity" 83 | > ``` 84 | ```rust 85 | #[tokio::main] 86 | async fn main() -> rusqlite::Result<()> { 87 | let db = Connection::open_in_memory()?; 88 | // load TonboLite 89 | load_module(&db)?; 90 | 91 | // use TonboLite like normal SQLite 92 | db.execute_batch( 93 | "CREATE VIRTUAL TABLE temp.tonbo USING tonbo( 94 | create_sql = 'create table tonbo(id bigint primary key, name varchar, like int)' 95 | path = 'db_path/tonbo' 96 | );", 97 | )?; 98 | db.execute("INSERT INTO tonbo (id, name, like) VALUES (0, 'lol', 1)", [])?; 99 | 100 | // query from table 101 | let mut stmt = db.prepare("SELECT * FROM tonbo;")?; 102 | let mut rows = stmt.query([])?; 103 | while let Some(row) = rows.next()? { 104 | println!("{:#?}", row); 105 | } 106 | } 107 | ``` 108 | 109 | 110 | ### Use in Wasm 111 | 112 | TonboLite exposed an easy-to-use API 113 | 114 | ```js 115 | const conn = new tonbo.Connection(); 116 | 117 | // create table with `CREATE VIRTUAL TABLE` statement 118 | await conn.create( 119 | `CREATE VIRTUAL TABLE temp.tonbo USING tonbo( 120 | create_sql = 'create table tonbo(id bigint primary key, name varchar, like int)', 121 | path = 'db_path/tonbo' 122 | );` 123 | ); 124 | 125 | // insert/update/delete table 126 | await conn.insert( 127 | `INSERT INTO tonbo (id, name, like) VALUES (0, 'lol', 0)` 128 | ); 129 | await conn.update("UPDATE tonbo SET name = 'bar' WHERE id = 0"); 130 | await conn.delete("DELETE from tonbo WHERE id = 0"); 131 | 132 | // query from table 133 | const rows = await conn.select("SELECT * from tonbo"); 134 | 135 | // fulsh in-memory data to S3 136 | await conn.flush("tonbo"); 137 | ``` 138 | 139 | ## Configuration 140 | Configure tonbolite in `CREATE` statement: 141 | - `create_sql`(required): The `CREATE` SQL statement 142 | - `path`(required): Path to local storage 143 | - `fs`: `local`/`s3` 144 | - `level`: All data below the level will be stored in local storage, otherwise, it will be stored in S3. 145 | - S3 option: 146 | - `key_id`: The S3 access key 147 | - `secret_key`: The S3 secret access key 148 | - `bucket`: The S3 bucket 149 | - `endpoint`: The S3 endpoint 150 | - `region`: The S3 region 151 | - `sign_payload`: `true`/`false`. Whether to use payload 152 | - `checksum`: `true`/`false`. Whether to use checksum 153 | - `token`: security token 154 | 155 | Here is an example to configure S3 storage: 156 | ```sql 157 | CREATE VIRTUAL TABLE temp.tonbo USING tonbo( 158 | create_sql='create table tonbo(id bigint primary key, name varchar, like int)', 159 | path = 'db_path/test_s3', 160 | level = '0', 161 | fs = 's3', 162 | bucket = 'bucket', 163 | key_id = 'access_key', 164 | secret_key = 'access_secret_key', 165 | endpoint = 'https://xxx.s3.us-east.amazonaws.com' 166 | ); 167 | ``` 168 | 169 | ## Build 170 | 171 | ### Build as Extension 172 | ```sh 173 | cargo build --release --features loadable_extension 174 | ``` 175 | Once building successfully, you will get a file named libsqlite_tonbo.dylib(`.dll` on windows, `.so` on most other unixes) in *target/release/* 176 | ### Build on Rust 177 | 178 | ```sh 179 | cargo build 180 | ``` 181 | 182 | ### Build on Wasm 183 | 184 | To use TonboLite in wasm, it takes a few steps to build. 185 | 1. Add wasm32-unknown-unknown target 186 | ```sh 187 | rustup target add wasm32-unknown-unknown 188 | ``` 189 | 2. Override toolchain with nightly 190 | ```sh 191 | rustup override set nightly 192 | ``` 193 | 3. Build with [wasm-pack](https://github.com/rustwasm/wasm-pack) 194 | ```sh 195 | wasm-pack build --target web --no-default-features --features wasm 196 | ``` 197 | 198 | Once you build successfully, you will get a *pkg* folder containing compiled js and wasm files. Copy it to your project and then you can start to use it. 199 | ```js 200 | const tonbo = await import("./pkg/sqlite_tonbo.js"); 201 | await tonbo.default(); 202 | 203 | // start to use TonboLite ... 204 | ``` 205 | 206 | #### Limitation 207 | TonboLite should be used in a [secure context](https://developer.mozilla.org/en-US/docs/Web/Security/Secure_Contexts) and [cross-origin isolated](https://developer.mozilla.org/en-US/docs/Web/API/Window/crossOriginIsolated), since it uses [`SharedArrayBuffer`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer) to share memory. Please refer to [this article](https://web.dev/articles/coop-coep) for a detailed explanation. 208 | -------------------------------------------------------------------------------- /src/connection.rs: -------------------------------------------------------------------------------- 1 | use crate::load_module; 2 | use flume::{Receiver, Sender}; 3 | use js_sys::Object; 4 | use rusqlite::types::{FromSql, FromSqlResult, ValueRef}; 5 | use std::fmt::Debug; 6 | use std::sync::{Arc, Mutex}; 7 | use wasm_bindgen::prelude::*; 8 | use wasm_bindgen_futures::js_sys; 9 | 10 | #[wasm_bindgen] 11 | pub struct Connection { 12 | conn: Arc>, 13 | req_tx: Sender, 14 | resp_rx: Receiver, 15 | } 16 | 17 | enum Request { 18 | Insert(String), 19 | Delete(String), 20 | Update(String), 21 | Select(String), 22 | Flush(String), 23 | } 24 | 25 | enum Response { 26 | RowCount(usize), 27 | Rows(Vec), 28 | Empty, 29 | } 30 | 31 | impl Connection { 32 | fn handle_request( 33 | conn: Arc>, 34 | req_rx: Receiver, 35 | resp_tx: Sender, 36 | ) { 37 | wasm_thread::Builder::new() 38 | .spawn(move || async move { 39 | while let Ok(req) = req_rx.recv_async().await { 40 | match req { 41 | Request::Insert(sql) | Request::Delete(sql) | Request::Update(sql) => { 42 | let res = conn.lock().unwrap().execute(&sql, []).unwrap(); 43 | resp_tx.send(Response::RowCount(res)).unwrap(); 44 | } 45 | Request::Select(sql) => { 46 | let db = conn.lock().unwrap(); 47 | let mut result = vec![]; 48 | let mut stmt = db.prepare(&sql).unwrap(); 49 | let mut rows = stmt.query([]).unwrap(); 50 | while let Some(row) = rows.next().unwrap() { 51 | let stmt = row.as_ref(); 52 | let mut cols = vec![]; 53 | 54 | for i in 0..stmt.column_count() { 55 | let name = stmt.column_name(i).expect("valid column index"); 56 | let mut value: JsColumn = row.get(i).unwrap(); 57 | value.set_name(name.to_string()); 58 | 59 | cols.push(value); 60 | } 61 | result.push(JsRow::new(cols)) 62 | } 63 | resp_tx.send_async(Response::Rows(result)).await.unwrap(); 64 | } 65 | Request::Flush(table) => { 66 | // We use PRAGMA quick_check to flush 67 | conn.lock() 68 | .unwrap() 69 | .pragma( 70 | None, 71 | "quick_check", 72 | table.as_str(), 73 | |_r| -> rusqlite::Result<()> { Ok(()) }, 74 | ) 75 | .unwrap(); 76 | resp_tx.send(Response::Empty).unwrap(); 77 | } 78 | } 79 | } 80 | }) 81 | .expect("initialize web worker failed."); 82 | } 83 | } 84 | 85 | #[wasm_bindgen] 86 | impl Connection { 87 | #[wasm_bindgen(constructor)] 88 | pub fn open() -> Self { 89 | let conn = rusqlite::Connection::open_in_memory().unwrap(); 90 | load_module(&conn).unwrap(); 91 | 92 | let conn = Arc::new(Mutex::new(conn)); 93 | let (req_tx, req_rx) = flume::bounded(1); 94 | let (resp_tx, resp_rx) = flume::bounded(1); 95 | 96 | Self::handle_request(conn.clone(), req_rx, resp_tx); 97 | 98 | Self { 99 | conn, 100 | req_tx, 101 | resp_rx, 102 | } 103 | } 104 | 105 | /// Convenience method to execute a single `CREATE` SQL statement. 106 | pub async fn create(&mut self, sql: String) -> Result { 107 | let res = { self.conn.lock().unwrap().execute(&sql, []).unwrap() }; 108 | 109 | Ok(res) 110 | } 111 | 112 | /// Convenience method to execute a single `INSERT` SQL statement. 113 | pub async fn insert(&self, sql: String) -> Result { 114 | self.req_tx.send_async(Request::Insert(sql)).await.unwrap(); 115 | let Response::RowCount(count) = self.resp_rx.recv_async().await.unwrap() else { 116 | unreachable!() 117 | }; 118 | 119 | Ok(count) 120 | } 121 | 122 | /// Convenience method to prepare and execute a single select SQL statement. 123 | pub async fn select(&self, sql: String) -> Vec { 124 | self.req_tx.send_async(Request::Select(sql)).await.unwrap(); 125 | let Response::Rows(rows) = self.resp_rx.recv_async().await.unwrap() else { 126 | unreachable!() 127 | }; 128 | let mut result = vec![]; 129 | for row in rows { 130 | result.push(row.into()); 131 | } 132 | result 133 | } 134 | 135 | /// Convenience method to execute a single `DELETE` SQL statement. 136 | pub async fn delete(&self, sql: String) -> Result { 137 | self.req_tx.send_async(Request::Delete(sql)).await.unwrap(); 138 | let Response::RowCount(count) = self.resp_rx.recv_async().await.unwrap() else { 139 | unreachable!() 140 | }; 141 | 142 | Ok(count) 143 | } 144 | 145 | /// Convenience method to execute a single `UPDATE` SQL statement. 146 | pub async fn update(&self, sql: String) -> Result { 147 | self.req_tx.send_async(Request::Update(sql)).await.unwrap(); 148 | let Response::RowCount(count) = self.resp_rx.recv_async().await.unwrap() else { 149 | unreachable!() 150 | }; 151 | 152 | Ok(count) 153 | } 154 | 155 | /// Flush data to stable storage. 156 | pub async fn flush(&self, table: String) { 157 | self.req_tx.send_async(Request::Flush(table)).await.unwrap(); 158 | let _ = self.resp_rx.recv_async().await.unwrap(); 159 | } 160 | } 161 | 162 | #[derive(Debug, Clone)] 163 | struct JsColumn { 164 | name: Option, 165 | value: Value, 166 | } 167 | 168 | impl JsColumn { 169 | fn new(value: Value) -> JsColumn { 170 | JsColumn { name: None, value } 171 | } 172 | 173 | fn set_name(&mut self, name: String) { 174 | self.name = Some(name); 175 | } 176 | } 177 | 178 | #[derive(Debug)] 179 | struct JsRow { 180 | cols: Vec, 181 | } 182 | 183 | impl JsRow { 184 | fn new(cols: Vec) -> JsRow { 185 | JsRow { cols } 186 | } 187 | } 188 | 189 | #[derive(Clone, Debug)] 190 | enum Value { 191 | Null, 192 | Integer(i64), 193 | Real(f64), 194 | Text(String), 195 | Blob(Vec), 196 | } 197 | 198 | impl FromSql for JsColumn { 199 | fn column_result(value: ValueRef<'_>) -> FromSqlResult { 200 | let value = match value { 201 | ValueRef::Null => Value::Null, 202 | ValueRef::Integer(i) => Value::Integer(i), 203 | ValueRef::Real(f) => Value::Real(f), 204 | ValueRef::Text(s) => Value::Text(String::from_utf8_lossy(s).to_string()), 205 | ValueRef::Blob(b) => Value::Blob(b.to_vec()), 206 | }; 207 | 208 | Ok(JsColumn::new(value)) 209 | } 210 | } 211 | 212 | impl From for JsValue { 213 | fn from(row: JsRow) -> Self { 214 | let object = Object::new(); 215 | for col in row.cols { 216 | match col.value { 217 | Value::Null => { 218 | js_sys::Reflect::set(&object, &col.name.unwrap().into(), &JsValue::null()) 219 | .unwrap(); 220 | } 221 | Value::Integer(n) => { 222 | js_sys::Reflect::set(&object, &col.name.unwrap().into(), &JsValue::from(n)) 223 | .unwrap(); 224 | } 225 | Value::Real(f) => { 226 | js_sys::Reflect::set(&object, &col.name.unwrap().into(), &JsValue::from_f64(f)) 227 | .unwrap(); 228 | } 229 | Value::Text(s) => { 230 | js_sys::Reflect::set(&object, &col.name.unwrap().into(), &JsValue::from(s)) 231 | .unwrap(); 232 | } 233 | Value::Blob(vec) => { 234 | let js_arr = js_sys::Uint8Array::from(vec.as_slice()); 235 | js_sys::Reflect::set(&object, &col.name.unwrap().into(), &js_arr.into()) 236 | .unwrap(); 237 | } 238 | } 239 | } 240 | object.into() 241 | } 242 | } 243 | -------------------------------------------------------------------------------- /src/utils.rs: -------------------------------------------------------------------------------- 1 | use rusqlite::types::ValueRef; 2 | use rusqlite::vtab::Context; 3 | use rusqlite::Error; 4 | use sqlparser::ast::DataType; 5 | use std::any::Any; 6 | use std::sync::Arc; 7 | use tonbo::record::{Datatype, Value}; 8 | 9 | macro_rules! nullable_value { 10 | ($value:expr, $is_nullable:expr) => { 11 | if $is_nullable { 12 | Arc::new(Some($value)) as Arc 13 | } else { 14 | Arc::new($value) as Arc 15 | } 16 | }; 17 | } 18 | 19 | // TODO: Value Cast 20 | pub(crate) fn value_trans( 21 | value: ValueRef<'_>, 22 | ty: &Datatype, 23 | is_nullable: bool, 24 | ) -> rusqlite::Result> { 25 | match value { 26 | ValueRef::Null => { 27 | if !is_nullable { 28 | return Err(Error::ModuleError("value is not nullable".to_string())); 29 | } 30 | Ok(match ty { 31 | Datatype::UInt8 => Arc::new(Option::::None), 32 | Datatype::UInt16 => Arc::new(Option::::None), 33 | Datatype::UInt32 => Arc::new(Option::::None), 34 | Datatype::UInt64 => Arc::new(Option::::None), 35 | Datatype::Int8 => Arc::new(Option::::None), 36 | Datatype::Int16 => Arc::new(Option::::None), 37 | Datatype::Int32 => Arc::new(Option::::None), 38 | Datatype::Int64 => Arc::new(Option::::None), 39 | Datatype::String => Arc::new(Option::::None), 40 | Datatype::Boolean => Arc::new(Option::::None), 41 | Datatype::Bytes => Arc::new(Option::>::None), 42 | }) 43 | } 44 | ValueRef::Integer(v) => { 45 | let value = match ty { 46 | Datatype::UInt8 => nullable_value!(v as u8, is_nullable), 47 | Datatype::UInt16 => nullable_value!(v as u16, is_nullable), 48 | Datatype::UInt32 => nullable_value!(v as u32, is_nullable), 49 | Datatype::UInt64 => nullable_value!(v as u64, is_nullable), 50 | Datatype::Int8 => nullable_value!(v as i8, is_nullable), 51 | Datatype::Int16 => nullable_value!(v as i16, is_nullable), 52 | Datatype::Int32 => nullable_value!(v as i32, is_nullable), 53 | Datatype::Int64 => nullable_value!(v, is_nullable), 54 | _ => { 55 | return Err(Error::ModuleError(format!( 56 | "unsupported value: {:#?} cast to: {:#?}", 57 | v, ty 58 | ))) 59 | } 60 | }; 61 | 62 | Ok(value) 63 | } 64 | ValueRef::Real(_) => { 65 | todo!("tonbo f32/f64 unsupported yet") 66 | } 67 | ValueRef::Text(v) => { 68 | if let Datatype::Bytes = ty { 69 | return Ok(nullable_value!(v.to_vec(), is_nullable)); 70 | } 71 | let v = String::from_utf8(v.to_vec()).unwrap(); 72 | let value = match ty { 73 | Datatype::UInt8 => nullable_value!(v.parse::().unwrap(), is_nullable), 74 | Datatype::UInt16 => nullable_value!(v.parse::().unwrap(), is_nullable), 75 | Datatype::UInt32 => nullable_value!(v.parse::().unwrap(), is_nullable), 76 | Datatype::UInt64 => nullable_value!(v.parse::().unwrap(), is_nullable), 77 | Datatype::Int8 => nullable_value!(v.parse::().unwrap(), is_nullable), 78 | Datatype::Int16 => nullable_value!(v.parse::().unwrap(), is_nullable), 79 | Datatype::Int32 => nullable_value!(v.parse::().unwrap(), is_nullable), 80 | Datatype::Int64 => nullable_value!(v.parse::().unwrap(), is_nullable), 81 | Datatype::String => nullable_value!(v, is_nullable), 82 | Datatype::Boolean => nullable_value!(v.parse::().unwrap(), is_nullable), 83 | Datatype::Bytes => unreachable!(), 84 | }; 85 | Ok(value) 86 | } 87 | ValueRef::Blob(v) => { 88 | let v = v.to_vec(); 89 | Ok(match ty { 90 | Datatype::String => nullable_value!(String::from_utf8(v).unwrap(), is_nullable), 91 | Datatype::Bytes => nullable_value!(v, is_nullable), 92 | _ => { 93 | return Err(Error::ModuleError(format!( 94 | "unsupported value: {:#?} cast to: {:#?}", 95 | v, ty 96 | ))) 97 | } 98 | }) 99 | } 100 | } 101 | } 102 | 103 | pub(crate) fn set_result(ctx: &mut Context, col: &Value) -> rusqlite::Result<()> { 104 | match &col.datatype { 105 | Datatype::UInt8 => { 106 | if col.is_nullable { 107 | ctx.set_result(col.value.as_ref().downcast_ref::>().unwrap())?; 108 | } else { 109 | ctx.set_result(col.value.as_ref().downcast_ref::().unwrap())?; 110 | } 111 | } 112 | Datatype::UInt16 => { 113 | if col.is_nullable { 114 | ctx.set_result(col.value.as_ref().downcast_ref::>().unwrap())?; 115 | } else { 116 | ctx.set_result(col.value.as_ref().downcast_ref::().unwrap())?; 117 | } 118 | } 119 | Datatype::UInt32 => { 120 | if col.is_nullable { 121 | ctx.set_result(col.value.as_ref().downcast_ref::>().unwrap())?; 122 | } else { 123 | ctx.set_result(col.value.as_ref().downcast_ref::().unwrap())?; 124 | } 125 | } 126 | Datatype::UInt64 => { 127 | if col.is_nullable { 128 | ctx.set_result(col.value.as_ref().downcast_ref::>().unwrap())?; 129 | } else { 130 | ctx.set_result(col.value.as_ref().downcast_ref::().unwrap())?; 131 | } 132 | } 133 | Datatype::Int8 => { 134 | if col.is_nullable { 135 | ctx.set_result(col.value.as_ref().downcast_ref::>().unwrap())?; 136 | } else { 137 | ctx.set_result(col.value.as_ref().downcast_ref::().unwrap())?; 138 | } 139 | } 140 | Datatype::Int16 => { 141 | if col.is_nullable { 142 | ctx.set_result(col.value.as_ref().downcast_ref::>().unwrap())?; 143 | } else { 144 | ctx.set_result(col.value.as_ref().downcast_ref::().unwrap())?; 145 | } 146 | } 147 | Datatype::Int32 => { 148 | if col.is_nullable { 149 | ctx.set_result(col.value.as_ref().downcast_ref::>().unwrap())?; 150 | } else { 151 | ctx.set_result(col.value.as_ref().downcast_ref::().unwrap())?; 152 | } 153 | } 154 | Datatype::Int64 => { 155 | if col.is_nullable { 156 | ctx.set_result(col.value.as_ref().downcast_ref::>().unwrap())?; 157 | } else { 158 | ctx.set_result(col.value.as_ref().downcast_ref::().unwrap())?; 159 | } 160 | } 161 | Datatype::String => { 162 | if col.is_nullable { 163 | ctx.set_result(col.value.as_ref().downcast_ref::>().unwrap())?; 164 | } else { 165 | ctx.set_result(col.value.as_ref().downcast_ref::().unwrap())?; 166 | } 167 | } 168 | Datatype::Boolean => { 169 | if col.is_nullable { 170 | ctx.set_result(col.value.as_ref().downcast_ref::>().unwrap())?; 171 | } else { 172 | ctx.set_result(col.value.as_ref().downcast_ref::().unwrap())?; 173 | } 174 | } 175 | Datatype::Bytes => { 176 | if col.is_nullable { 177 | ctx.set_result( 178 | col.value 179 | .as_ref() 180 | .downcast_ref::>>() 181 | .unwrap(), 182 | )?; 183 | } else { 184 | ctx.set_result(col.value.as_ref().downcast_ref::>().unwrap())?; 185 | } 186 | } 187 | } 188 | Ok(()) 189 | } 190 | 191 | pub(crate) fn type_trans(ty: &DataType) -> Datatype { 192 | match ty { 193 | DataType::Int8(_) => Datatype::Int8, 194 | DataType::Int16 | DataType::SmallInt(_) => Datatype::Int16, 195 | DataType::Int(_) | DataType::Int32 | DataType::Integer(_) => Datatype::Int32, 196 | DataType::Int64 | DataType::BigInt(_) => Datatype::Int64, 197 | DataType::UnsignedInt(_) | DataType::UInt32 | DataType::UnsignedInteger(_) => { 198 | Datatype::UInt32 199 | } 200 | DataType::UInt8 | DataType::UnsignedInt8(_) => Datatype::UInt8, 201 | DataType::UInt16 => Datatype::UInt16, 202 | DataType::UInt64 | DataType::UnsignedBigInt(_) => Datatype::UInt64, 203 | DataType::Bool | DataType::Boolean => Datatype::Boolean, 204 | DataType::Bytes(_) => Datatype::Bytes, 205 | DataType::Varchar(_) => Datatype::String, 206 | _ => todo!(), 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /src/db.rs: -------------------------------------------------------------------------------- 1 | use crate::executor::{BlockOnExecutor, SQLiteExecutor}; 2 | use crate::utils::type_trans; 3 | use crate::utils::{set_result, value_trans}; 4 | use flume::{Receiver, Sender}; 5 | use fusio::path::Path; 6 | use fusio_dispatch::FsOptions; 7 | use futures_util::StreamExt; 8 | use rusqlite::types::ValueRef; 9 | use rusqlite::vtab::{ 10 | parse_boolean, Context, CreateVTab, IndexInfo, UpdateVTab, VTab, VTabConnection, VTabCursor, 11 | VTabKind, ValueIter, Values, 12 | }; 13 | use rusqlite::{ffi, vtab, Error}; 14 | use sqlparser::ast::ColumnOption; 15 | use sqlparser::ast::Statement; 16 | use sqlparser::dialect::SQLiteDialect; 17 | use sqlparser::parser::Parser; 18 | use std::collections::Bound; 19 | use std::ffi::c_int; 20 | use std::marker::PhantomData; 21 | use std::sync::Arc; 22 | use tonbo::executor::Executor; 23 | use tonbo::record::{DynRecord, DynSchema}; 24 | use tonbo::{DbOption, DB}; 25 | 26 | const MAX_LEVEL: usize = 7; 27 | 28 | pub struct DbState { 29 | executor: SQLiteExecutor, 30 | } 31 | 32 | impl DbState { 33 | pub(crate) fn new() -> Self { 34 | Self { 35 | executor: SQLiteExecutor::new(), 36 | } 37 | } 38 | } 39 | 40 | #[repr(C)] 41 | pub struct TonboTable { 42 | base: ffi::sqlite3_vtab, 43 | descs: Vec, 44 | pk_index: usize, 45 | req_tx: Sender, 46 | } 47 | 48 | enum Request { 49 | Scan { 50 | lower: Bound, 51 | upper: Bound, 52 | tuple_tx: Sender, usize)>>, 53 | }, 54 | Insert((Sender>, DynRecord)), 55 | Remove((Sender>, tonbo::record::Value)), 56 | Flush(Sender>), 57 | } 58 | 59 | impl TonboTable { 60 | async fn handle_request(db: DB, req_rx: Receiver) { 61 | while let Ok(req) = req_rx.recv() { 62 | match req { 63 | Request::Scan { 64 | lower, 65 | upper, 66 | tuple_tx, 67 | } => { 68 | let transaction = db.transaction().await; 69 | 70 | let mut stream = transaction 71 | .scan((lower.as_ref(), upper.as_ref())) 72 | .take() 73 | .await 74 | .unwrap(); 75 | 76 | while let Some(result) = stream.next().await { 77 | let entry = result.unwrap(); 78 | if let Some(value) = entry.value() { 79 | let _ = tuple_tx.send(Some((value.columns, value.primary_index))); 80 | } 81 | } 82 | let _ = tuple_tx.send(None); 83 | } 84 | Request::Insert((tx, record)) => { 85 | db.insert(record).await.unwrap(); 86 | tx.send(Ok(())).unwrap() 87 | } 88 | Request::Remove((tx, key)) => { 89 | db.remove(key).await.unwrap(); 90 | tx.send(Ok(())).unwrap() 91 | } 92 | Request::Flush(sender) => { 93 | let result = db 94 | .flush() 95 | .await 96 | .map_err(|err| Error::ModuleError(err.to_string())); 97 | 98 | sender.send(result).unwrap(); 99 | } 100 | } 101 | } 102 | } 103 | 104 | fn connect_create( 105 | _: &mut VTabConnection, 106 | aux: Option<&Arc>, 107 | args: &[&[u8]], 108 | _: bool, 109 | ) -> rusqlite::Result<(String, Self)> { 110 | let mut create_sql = None; 111 | let mut primary_key_index = None; 112 | let mut descs = Vec::new(); 113 | let mut is_local = true; 114 | // Local 115 | let mut path = None; 116 | // S3 117 | let mut bucket = None; 118 | let mut key_id = None; 119 | let mut secret_key = None; 120 | let mut token = None; 121 | let mut endpoint = None; 122 | let mut region = None; 123 | let mut sign_payload = None; 124 | let mut checksum = None; 125 | let mut table_name = None; 126 | let mut level = 2; 127 | 128 | let args = &args[3..]; 129 | for c_slice in args.iter() { 130 | let (param, value) = vtab::parameter(c_slice)?; 131 | match param { 132 | "create_sql" => { 133 | if create_sql.is_some() { 134 | return Err(Error::ModuleError("`create_sql` duplicate".to_string())); 135 | } 136 | create_sql = Some(value.to_string()); 137 | (table_name, descs, primary_key_index) = parse_create_sql(value)?; 138 | } 139 | "fs" => match value { 140 | "local" => is_local = true, 141 | "s3" => is_local = false, 142 | _ => { 143 | return Err(Error::ModuleError(format!( 144 | "unrecognized fs type '{param}'" 145 | ))) 146 | } 147 | }, 148 | "level" => level = value.parse::().unwrap(), 149 | "key_id" => key_id = Some(value.to_string()), 150 | "secret_key" => secret_key = Some(value.to_string()), 151 | "token" => token = Some(value.to_string()), 152 | "bucket" => bucket = Some(value.to_string()), 153 | "path" => path = Some(value.to_string()), 154 | "endpoint" => endpoint = Some(value.to_string()), 155 | "region" => region = Some(value.to_string()), 156 | "sign_payload" => sign_payload = parse_boolean(value), 157 | "checksum" => checksum = parse_boolean(value), 158 | _ => { 159 | return Err(Error::ModuleError(format!( 160 | "unrecognized parameter '{param}'" 161 | ))); 162 | } 163 | } 164 | } 165 | let pk_index = primary_key_index.ok_or_else(|| { 166 | Error::ModuleError("primary key not found on `create_sql`".to_string()) 167 | })?; 168 | let table_name = 169 | table_name.ok_or_else(|| Error::ModuleError("`create_sql` not found".to_string()))?; 170 | 171 | let (req_tx, req_rx): (_, Receiver) = flume::bounded(1); 172 | let fs_option = if is_local { 173 | FsOptions::Local 174 | } else { 175 | let mut credential = None; 176 | if key_id.is_some() || secret_key.is_some() || token.is_some() { 177 | credential = Some(fusio::remotes::aws::AwsCredential { 178 | key_id: key_id 179 | .ok_or_else(|| Error::ModuleError("`key_id` not found".to_string()))?, 180 | secret_key: secret_key 181 | .ok_or_else(|| Error::ModuleError("`secret_key` not found".to_string()))?, 182 | token, 183 | }); 184 | } 185 | FsOptions::S3 { 186 | bucket: bucket 187 | .ok_or_else(|| Error::ModuleError("`bucket` not found".to_string()))?, 188 | credential, 189 | endpoint, 190 | region, 191 | sign_payload, 192 | checksum, 193 | } 194 | }; 195 | let schema = DynSchema::new(descs.clone(), pk_index); 196 | 197 | #[cfg(feature = "wasm")] 198 | wasm_thread::Builder::new() 199 | .spawn(move || async move { 200 | let path = Path::from_opfs_path(path.expect("`path` not found")).unwrap(); 201 | let mut option = DbOption::new(path, &schema); 202 | if !is_local { 203 | option = level_path(option, fs_option, &table_name, level); 204 | } 205 | let db = DB::new(option, SQLiteExecutor::new(), schema) 206 | .await 207 | .unwrap(); 208 | Self::handle_request(db, req_rx).await; 209 | }) 210 | .expect("initialize web worker failed."); 211 | #[cfg(not(feature = "wasm"))] 212 | { 213 | let path = Path::from_filesystem_path( 214 | path.ok_or_else(|| Error::ModuleError("`path` not found".to_string()))?, 215 | ) 216 | .map_err(|err| Error::ModuleError(format!("path parser error: {err}")))?; 217 | let executor = aux.unwrap().executor.clone(); 218 | let db = aux.unwrap().executor.block_on(async { 219 | let mut option = DbOption::new(path, &schema); 220 | if !is_local { 221 | option = level_path(option, fs_option, &table_name, level); 222 | } 223 | 224 | DB::::new(option, executor, schema) 225 | .await 226 | .unwrap() 227 | }); 228 | aux.unwrap().executor.spawn(async move { 229 | Self::handle_request(db, req_rx).await; 230 | }); 231 | } 232 | 233 | Ok(( 234 | create_sql.ok_or_else(|| Error::ModuleError("`create_sql` not found".to_string()))?, 235 | Self { 236 | base: ffi::sqlite3_vtab::default(), 237 | req_tx, 238 | descs, 239 | pk_index, 240 | }, 241 | )) 242 | } 243 | 244 | fn _remove(&mut self, pk: i64) -> Result<(), Error> { 245 | let desc: &tonbo::record::ValueDesc = &self.descs[self.pk_index]; 246 | let value = tonbo::record::Value::new( 247 | desc.datatype, 248 | desc.name.clone(), 249 | Arc::new(pk), 250 | desc.is_nullable, 251 | ); 252 | let (remove_tx, remove_rx) = flume::bounded(1); 253 | 254 | self.req_tx 255 | .send(Request::Remove((remove_tx, value))) 256 | .unwrap(); 257 | let _ = remove_rx.recv().unwrap(); 258 | Ok(()) 259 | } 260 | 261 | fn _insert(&mut self, args: ValueIter) -> Result { 262 | let mut id = None; 263 | let mut values = Vec::with_capacity(self.descs.len()); 264 | 265 | for (i, (desc, value)) in self.descs.iter().zip(args).enumerate() { 266 | if i == self.pk_index { 267 | id = Some(value); 268 | } 269 | values.push(tonbo::record::Value::new( 270 | desc.datatype, 271 | desc.name.clone(), 272 | value_trans(value, &desc.datatype, desc.is_nullable)?, 273 | desc.is_nullable, 274 | )); 275 | } 276 | let (insert_tx, insert_rx) = flume::bounded(1); 277 | 278 | self.req_tx 279 | .send(Request::Insert(( 280 | insert_tx, 281 | DynRecord::new(values, self.pk_index), 282 | ))) 283 | .unwrap(); 284 | let _ = insert_rx.recv().unwrap(); 285 | 286 | Ok(id.unwrap().as_i64()?) 287 | } 288 | } 289 | 290 | unsafe impl<'vtab> VTab<'vtab> for TonboTable { 291 | type Aux = Arc; 292 | type Cursor = RecordCursor<'vtab>; 293 | 294 | fn connect( 295 | db: &mut VTabConnection, 296 | aux: Option<&Self::Aux>, 297 | args: &[&[u8]], 298 | ) -> rusqlite::Result<(String, Self)> { 299 | Self::connect_create(db, aux, args, false) 300 | } 301 | 302 | fn best_index(&self, info: &mut IndexInfo) -> rusqlite::Result<()> { 303 | info.set_estimated_cost(500.); 304 | info.set_estimated_rows(500); 305 | Ok(()) 306 | } 307 | 308 | fn open(&'vtab mut self) -> rusqlite::Result { 309 | Ok(RecordCursor { 310 | base: ffi::sqlite3_vtab_cursor::default(), 311 | req_tx: self.req_tx.clone(), 312 | tuple_rx: None, 313 | buf: None, 314 | _p: Default::default(), 315 | }) 316 | } 317 | 318 | fn integrity(&'vtab mut self, _flag: usize) -> rusqlite::Result<()> { 319 | let (flush_tx, flush_rx) = flume::bounded(1); 320 | 321 | self.req_tx.send(Request::Flush(flush_tx)).unwrap(); 322 | flush_rx.recv().unwrap() 323 | } 324 | } 325 | 326 | impl CreateVTab<'_> for TonboTable { 327 | const KIND: VTabKind = VTabKind::Default; 328 | 329 | fn create( 330 | db: &mut VTabConnection, 331 | aux: Option<&Self::Aux>, 332 | args: &[&[u8]], 333 | ) -> rusqlite::Result<(String, Self)> { 334 | Self::connect_create(db, aux, args, true) 335 | } 336 | 337 | fn destroy(&self) -> rusqlite::Result<()> { 338 | Ok(()) 339 | } 340 | } 341 | 342 | #[repr(C)] 343 | pub struct RecordCursor<'vtab> { 344 | /// Base class. Must be first 345 | base: ffi::sqlite3_vtab_cursor, 346 | req_tx: Sender, 347 | tuple_rx: Option, usize)>>>, 348 | buf: Option<(Vec, usize)>, 349 | _p: PhantomData<&'vtab TonboTable>, 350 | } 351 | 352 | impl RecordCursor<'_> { 353 | #[allow(unused)] 354 | fn vtab(&self) -> &TonboTable { 355 | unsafe { &*(self.base.pVtab as *const TonboTable) } 356 | } 357 | } 358 | 359 | impl UpdateVTab<'_> for TonboTable { 360 | fn delete(&mut self, arg: ValueRef<'_>) -> rusqlite::Result<()> { 361 | self._remove(arg.as_i64().unwrap())?; 362 | 363 | Ok(()) 364 | } 365 | 366 | fn insert(&mut self, args: &Values<'_>) -> rusqlite::Result { 367 | let mut args = args.iter(); 368 | 369 | let _ = args.next(); 370 | let _ = args.next(); 371 | 372 | self._insert(args) 373 | } 374 | 375 | fn update(&mut self, args: &Values<'_>) -> rusqlite::Result<()> { 376 | let mut args = args.iter(); 377 | let _ = args.next(); 378 | let Some(old_pk) = args.next().map(|v| v.as_i64().unwrap()) else { 379 | return Ok(()); 380 | }; 381 | let new_pk = self._insert(args)?; 382 | if new_pk != old_pk { 383 | self._remove(old_pk)?; 384 | } 385 | 386 | Ok(()) 387 | } 388 | } 389 | 390 | unsafe impl VTabCursor for RecordCursor<'_> { 391 | fn filter(&mut self, _: c_int, _: Option<&str>, v: &Values<'_>) -> rusqlite::Result<()> { 392 | let (tuple_tx, tuple_rx) = flume::bounded(5); 393 | 394 | self.req_tx 395 | .send(Request::Scan { 396 | lower: Bound::Unbounded, 397 | upper: Bound::Unbounded, 398 | tuple_tx, 399 | }) 400 | .unwrap(); 401 | self.tuple_rx = Some(tuple_rx); 402 | self.next()?; 403 | 404 | Ok(()) 405 | } 406 | 407 | fn next(&mut self) -> rusqlite::Result<()> { 408 | self.buf = self 409 | .tuple_rx 410 | .as_mut() 411 | .expect("`filter` was not called") 412 | .recv() 413 | .unwrap(); 414 | 415 | Ok(()) 416 | } 417 | 418 | fn eof(&self) -> bool { 419 | self.buf.is_none() 420 | } 421 | 422 | fn column(&self, ctx: &mut Context, i: c_int) -> rusqlite::Result<()> { 423 | if let Some((columns, _)) = &self.buf { 424 | set_result(ctx, &columns[i as usize])?; 425 | } 426 | Ok(()) 427 | } 428 | 429 | fn rowid(&self) -> rusqlite::Result { 430 | let (columns, primary_index) = self.buf.as_ref().unwrap(); 431 | 432 | Ok(*columns[*primary_index].value.downcast_ref().unwrap()) 433 | } 434 | } 435 | 436 | fn parse_create_sql( 437 | sql: &str, 438 | ) -> rusqlite::Result<(Option, Vec, Option)> { 439 | let dialect = SQLiteDialect {}; 440 | let mut primary_key_index = None; 441 | let mut descs = Vec::new(); 442 | let mut table_name = None; 443 | if let Statement::CreateTable(create_table) = 444 | &Parser::parse_sql(&dialect, sql).map_err(|err| Error::ModuleError(err.to_string()))?[0] 445 | { 446 | table_name = Some(create_table.name.to_string()); 447 | for (i, column_def) in create_table.columns.iter().enumerate() { 448 | let name = column_def.name.value.to_ascii_lowercase(); 449 | let datatype = type_trans(&column_def.data_type); 450 | let mut is_not_nullable = column_def 451 | .options 452 | .iter() 453 | .any(|option| matches!(option.option, ColumnOption::NotNull)); 454 | let is_primary_key = column_def.options.iter().any(|option| { 455 | matches!( 456 | option.option, 457 | ColumnOption::Unique { 458 | is_primary: true, 459 | .. 460 | } 461 | ) 462 | }); 463 | if is_primary_key { 464 | if primary_key_index.is_some() { 465 | return Err(Error::ModuleError( 466 | "the primary key must exist and only one is allowed".to_string(), 467 | )); 468 | } 469 | is_not_nullable = true; 470 | primary_key_index = Some(i) 471 | } 472 | descs.push(tonbo::record::ValueDesc { 473 | datatype, 474 | is_nullable: !is_not_nullable, 475 | name, 476 | }) 477 | } 478 | } else { 479 | return Err(Error::ModuleError(format!( 480 | "`CreateTable` SQL syntax error: '{sql}'" 481 | ))); 482 | } 483 | Ok((table_name, descs, primary_key_index)) 484 | } 485 | 486 | fn level_path( 487 | mut option: DbOption, 488 | fs_options: FsOptions, 489 | table_name: &str, 490 | level: usize, 491 | ) -> DbOption { 492 | for i in level..MAX_LEVEL { 493 | option = option 494 | .level_path( 495 | i, 496 | Path::from_url_path(format!("/{}/{}", table_name, i)).unwrap(), 497 | fs_options.clone(), 498 | ) 499 | .unwrap(); 500 | } 501 | 502 | option 503 | } 504 | 505 | #[cfg(test)] 506 | pub(crate) mod tests { 507 | use rusqlite::Connection; 508 | use std::fs; 509 | 510 | #[test] 511 | fn test_load_module() -> rusqlite::Result<()> { 512 | let _ = fs::create_dir_all("./db_path/test"); 513 | 514 | let db = Connection::open_in_memory()?; 515 | crate::load_module(&db)?; 516 | 517 | db.execute_batch( 518 | "CREATE VIRTUAL TABLE temp.tonbo USING tonbo( 519 | create_sql = 'create table tonbo(id bigint primary key, name varchar, like int)', 520 | path = 'db_path/test' 521 | );" 522 | )?; 523 | for i in 0..3 { 524 | db.execute( 525 | &format!("INSERT INTO tonbo (id, name, like) VALUES ({i}, 'lol', {i})"), 526 | [], 527 | )?; 528 | } 529 | let mut stmt = db.prepare("SELECT * FROM tonbo;")?; 530 | let mut rows = stmt.query([])?; 531 | let row = rows.next()?.unwrap(); 532 | assert_eq!(row.get_ref_unwrap(0).as_i64().unwrap(), 0); 533 | assert_eq!(row.get_ref_unwrap(1).as_str().unwrap(), "lol"); 534 | assert_eq!(row.get_ref_unwrap(2).as_i64().unwrap(), 0); 535 | let row = rows.next()?.unwrap(); 536 | assert_eq!(row.get_ref_unwrap(0).as_i64().unwrap(), 1); 537 | assert_eq!(row.get_ref_unwrap(1).as_str().unwrap(), "lol"); 538 | assert_eq!(row.get_ref_unwrap(2).as_i64().unwrap(), 1); 539 | let row = rows.next()?.unwrap(); 540 | assert_eq!(row.get_ref_unwrap(0).as_i64().unwrap(), 2); 541 | assert_eq!(row.get_ref_unwrap(1).as_str().unwrap(), "lol"); 542 | assert_eq!(row.get_ref_unwrap(2).as_i64().unwrap(), 2); 543 | assert!(rows.next()?.is_none()); 544 | 545 | db.execute( 546 | "UPDATE tonbo SET name = ?1, like = ?2, id = 4 WHERE id = ?3", 547 | ["ioi", "9", "0"], 548 | )?; 549 | let mut stmt = db.prepare("SELECT * FROM tonbo where id = ?1 or id = ?2;")?; 550 | let mut rows = stmt.query(["0", "4"])?; 551 | 552 | let row = rows.next()?.unwrap(); 553 | assert_eq!(row.get_ref_unwrap(0).as_i64().unwrap(), 4); 554 | assert_eq!(row.get_ref_unwrap(1).as_str().unwrap(), "ioi"); 555 | assert_eq!(row.get_ref_unwrap(2).as_i64().unwrap(), 9); 556 | assert!(rows.next()?.is_none()); 557 | 558 | db.execute("DELETE from tonbo WHERE id = ?1", ["2"])?; 559 | db.pragma(None, "quick_check", "tonbo", |_r| -> rusqlite::Result<()> { 560 | Ok(()) 561 | })?; 562 | Ok(()) 563 | } 564 | 565 | #[test] 566 | fn test_load_module_on_s3() -> rusqlite::Result<()> { 567 | let _ = fs::create_dir_all("./db_path/test_s3"); 568 | 569 | let db = Connection::open_in_memory()?; 570 | crate::load_module(&db)?; 571 | 572 | db.execute_batch( 573 | "CREATE VIRTUAL TABLE temp.tonbo USING tonbo( 574 | create_sql='create table tonbo(id bigint primary key, name varchar, like int)', 575 | path = './db_path/test_s3', 576 | level = '0', 577 | fs = 's3', 578 | bucket = 'data', 579 | key_id = 'user', 580 | secret_key = 'password', 581 | endpoint = 'http://localhost:9000', 582 | );", 583 | )?; 584 | let num = 100; 585 | for i in 0..num { 586 | db.execute( 587 | &format!("INSERT INTO tonbo (id, name, like) VALUES ({i}, 'lol', {i})"), 588 | [], 589 | )?; 590 | } 591 | db.pragma(None, "quick_check", "tonbo", |_r| -> rusqlite::Result<()> { 592 | Ok(()) 593 | })?; 594 | let mut stmt = db.prepare("SELECT * FROM tonbo limit 10;")?; 595 | let mut rows = stmt.query([])?; 596 | while let Some(row) = rows.next()? { 597 | println!("{:#?}", row); 598 | } 599 | 600 | Ok(()) 601 | } 602 | } 603 | --------------------------------------------------------------------------------