├── .eslintignore ├── .eslintrc.cjs ├── .github ├── dependabot.yml └── workflows │ ├── node.js.yml │ └── npm.yml ├── .gitignore ├── .mocharc.json ├── .npmignore ├── .vscode └── launch.json ├── CHANGELOG.md ├── LICENSE ├── README.md ├── build_sqlite.sh ├── codecov.yml ├── deps ├── XMLHttpRequest-stub.js ├── XMLHttpRequest.cjs ├── XMLHttpRequest.d.ts ├── dist │ ├── sqlite3-api-bundler-friendly.mjs │ ├── sqlite3-api-node.mjs │ ├── sqlite3-api.js │ ├── sqlite3-api.mjs │ ├── sqlite3-bundler-friendly.mjs │ ├── sqlite3-node.mjs │ ├── sqlite3-opfs-async-proxy.js │ ├── sqlite3-worker1-bundler-friendly.mjs │ ├── sqlite3-worker1-promiser-bundler-friendly.js │ ├── sqlite3-worker1-promiser.js │ ├── sqlite3-worker1.js │ ├── sqlite3.js │ ├── sqlite3.mjs │ └── sqlite3.wasm └── types │ └── sqlite3.d.ts ├── docs ├── API.md ├── overview.png └── overview.svg ├── examples ├── index.html ├── index.ts ├── nodejs-example.ts ├── pacman.svg └── tsconfig.json ├── package-lock.json ├── package.json ├── scripts ├── publish-ghpages.ts └── wait-test.js ├── src ├── endianness.ts ├── index.ts ├── sqlite-worker.ts ├── vfs-http-types.ts ├── vfs-http-worker.ts ├── vfs-http.ts └── vfs-sync-http.ts ├── test ├── http-pool.test.ts ├── integration.test.ts ├── integration │ ├── .gitignore │ ├── README.md │ ├── browser-react-js.karma.cjs │ ├── browser-react-js │ │ ├── .gitignore │ │ ├── package.json │ │ ├── public │ │ │ └── index.html │ │ └── src │ │ │ ├── App.js │ │ │ └── index.js │ ├── browser-rollup-js.karma.cjs │ ├── browser-rollup-js │ │ ├── index.html │ │ ├── index.js │ │ ├── package.json │ │ └── rollup.config.mjs │ ├── browser-webpack-js.karma.cjs │ ├── browser-webpack-js │ │ ├── index.js │ │ ├── package.json │ │ └── webpack.config.js │ ├── browser-webpack-ts-cjs.karma.cjs │ ├── browser-webpack-ts-cjs │ │ ├── index.ts │ │ ├── package.json │ │ ├── tsconfig.json │ │ └── webpack.config.cjs │ ├── browser-webpack-ts-esm.karma.cjs │ ├── browser-webpack-ts-esm │ │ ├── index.ts │ │ ├── modules.d.ts │ │ ├── package.json │ │ ├── tsconfig.json │ │ └── webpack.config.cjs │ ├── node-es6 │ │ ├── index.js │ │ ├── package.json │ │ └── setup.js │ └── node-ts-esm │ │ ├── index.ts │ │ ├── modules.d.ts │ │ ├── package.json │ │ ├── setup.ts │ │ └── tsconfig.json ├── modules.d.ts ├── setup.ts ├── sync-sqlite-vfs-http.test.ts ├── tsconfig.json └── vfs-http.test.ts ├── tsconfig.cjs.json ├── tsconfig.json └── webpack.config.cjs /.eslintignore: -------------------------------------------------------------------------------- 1 | /deps/** 2 | /docs/** 3 | /test/integration/** 4 | *.json 5 | *.html 6 | *.cjs 7 | *.js 8 | -------------------------------------------------------------------------------- /.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | browser: true, 4 | commonjs: false, 5 | es6: true, 6 | node: true, 7 | mocha: true 8 | }, 9 | extends: [ 10 | 'eslint:recommended', 11 | 'plugin:@typescript-eslint/eslint-recommended', 12 | 'plugin:@typescript-eslint/recommended' 13 | ], 14 | plugins: [ 15 | 'mocha' 16 | ], 17 | parserOptions: { 18 | ecmaVersion: 2017, 19 | sourceType: 'module' 20 | }, 21 | rules: { 22 | quotes: ['error', 'single'], 23 | semi: ['error', 'always'], 24 | 'mocha/no-exclusive-tests': 'error', 25 | 'max-len': ['error', { 'code': 120, 'tabWidth': 2 }], 26 | '@typescript-eslint/triple-slash-reference': 'off' 27 | } 28 | }; 29 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "npm" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | open-pull-requests-limit: 1 8 | groups: 9 | eslint: 10 | patterns: 11 | - "*eslint*" 12 | - package-ecosystem: "github-actions" 13 | directory: "/" 14 | schedule: 15 | interval: "weekly" 16 | open-pull-requests-limit: 1 17 | -------------------------------------------------------------------------------- /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | name: Node.js CI 2 | 3 | on: 4 | push: 5 | branches: [ '*' ] 6 | pull_request: 7 | branches: [ 'main' ] 8 | 9 | jobs: 10 | test: 11 | 12 | runs-on: ${{ matrix.platform }} 13 | 14 | strategy: 15 | matrix: 16 | platform: [ubuntu-latest, macos-latest, windows-latest] 17 | node: [22.x] 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Use Node.js ${{ matrix.node }} 22 | uses: actions/setup-node@v4 23 | with: 24 | node-version: ${{ matrix.node }} 25 | - run: npm ci 26 | - run: npm run build -if-present 27 | - name: Run headless test 28 | run: npm test 29 | 30 | codecov: 31 | 32 | runs-on: ubuntu-latest 33 | 34 | steps: 35 | - uses: actions/checkout@v4 36 | - name: Use Node.js 22.x 37 | uses: actions/setup-node@v4 38 | with: 39 | node-version: 22.x 40 | - run: npm ci 41 | - run: npm run build -if-present 42 | - run: npm run c8 43 | - run: npm run lcov 44 | - name: Upload coverage reports to Codecov 45 | uses: codecov/codecov-action@v5 46 | env: 47 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 48 | -------------------------------------------------------------------------------- /.github/workflows/npm.yml: -------------------------------------------------------------------------------- 1 | name: Test npm package 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | version: 7 | description: 'Package version to test' 8 | required: true 9 | 10 | jobs: 11 | integration: 12 | 13 | runs-on: ${{ matrix.platform }} 14 | 15 | strategy: 16 | matrix: 17 | platform: [ubuntu-latest, macos-latest, windows-latest] 18 | node: [21.x] 19 | 20 | steps: 21 | - uses: actions/checkout@v4 22 | - name: Use Node.js ${{ matrix.node }} 23 | uses: actions/setup-node@v4 24 | with: 25 | node-version: ${{ matrix.node }} 26 | - run: npm ci 27 | - name: Run headless test 28 | uses: GabrielBB/xvfb-action@v1 29 | env: 30 | SQLITE_INSTALL_CMD: npm install sqlite-wasm-http@${{ github.event.inputs.version }} 31 | with: 32 | run: npm test 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | /deps/sqlite 3 | !/deps/dist 4 | /dist 5 | /docs/examples 6 | /coverage 7 | temp 8 | stats.json 9 | -------------------------------------------------------------------------------- /.mocharc.json: -------------------------------------------------------------------------------- 1 | { 2 | "spec": "test/*.test.@(cjs|mjs|ts)", 3 | "require": [ 4 | "tsx", 5 | "test/setup.ts" 6 | ], 7 | "reporter": "tap", 8 | "timeout": 600000, 9 | "node-option": [ 10 | "no-warnings" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .github 3 | .git 4 | node_modules 5 | /deps/sqlite 6 | !/deps/dist 7 | /coverage 8 | temp 9 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch Chrome", 9 | "request": "launch", 10 | "type": "chrome", 11 | "url": "http://localhost:9000", 12 | "webRoot": "${workspaceFolder}" 13 | }, 14 | { 15 | "type": "node", 16 | "request": "launch", 17 | "name": "Launch mocha", 18 | "skipFiles": [ 19 | "/**" 20 | ], 21 | "env": { 22 | "NODE_OPTIONS": "--loader tsx" 23 | }, 24 | "program": "${workspaceFolder}/node_modules/mocha/lib/cli/cli.js" 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 1.2.0 2023-12-12 2 | 3 | - Update the SQLite WASM distribution to an official release, 3.44.2 built to WASM with emscripten 3.1.46 4 | - Update the integration tests after the breaking changes in Node.js 18.19 [TypeStrong/ts-node#2094](https://github.com/TypeStrong/ts-node/issues/2094) 5 | - Test the npm package on all platforms 6 | - Update the dependencies 7 | 8 | ## 1.1.2 2023-10-07 9 | 10 | - Test and officially support a pool of sync workers 11 | - Add `"module": "node16"` to all integration tests 12 | 13 | ## 1.1.1 2023-04-03 14 | 15 | - Rebuild the SQLite WASM distribution from the SQLite trunk 2023-03-30 16 | - Eliminate `sqlite3-worker1-promiser-node.mjs` 17 | - Greatly reduce the output bundle size by working around [webpack#16895](https://github.com/webpack/webpack/issues/16895) 18 | - Fix [#18](https://github.com/mmomtchev/sqlite-wasm-http/issues/18), a race condition occurring only when running on a single CPU core and using the shared backend 19 | 20 | # 1.1.0 2023-03-29 21 | 22 | - Support the fully synchronous SQLite OO1 API with both HTTP VFS backends 23 | - Transform the `sqlite3.js` import into a true TypeScript import that can be imported from user code without `sqlite-wasm-http` 24 | - Support SQLite `rowMode: 'object'` 25 | - Support passing bindable parameters in an array 26 | - Improve the TypeScript types to allow auto inferring of the overloaded argument of `Promiser` 27 | - Add `SQLite.SQLValue` and `SQLite.SQLBindable` types for the data coming from or going to the DB 28 | - Expose `sqlite3Worker1Promiser` to user code 29 | - Fix [#13](https://github.com/mmomtchev/sqlite-wasm-http/issues/13), `PRAGMA TABLE_INFO()` returns an empty result set 30 | 31 | # 1.0.0 2023-03-14 32 | 33 | - First release 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ISC License 2 | 3 | Copyright (c) 2023-2024, Momtchil Momtchev 4 | 5 | Permission to use, copy, modify, and/or distribute this software for any 6 | purpose with or without fee is hereby granted, provided that the above 7 | copyright notice and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sqlite-wasm-http 2 | 3 | SQLite WASM with HTTP VFS 4 | 5 | [![License: ISC](https://img.shields.io/github/license/mmomtchev/sqlite-wasm-http)](https://github.com/mmomtchev/sqlite-wasm-http/blob/main/LICENSE) 6 | [![Node.js CI](https://github.com/mmomtchev/sqlite-wasm-http/actions/workflows/node.js.yml/badge.svg)](https://github.com/mmomtchev/sqlite-wasm-http/actions/workflows/node.js.yml) 7 | [![codecov](https://codecov.io/gh/mmomtchev/sqlite-wasm-http/branch/main/graph/badge.svg?token=SLQOP9XTEV)](https://codecov.io/gh/mmomtchev/sqlite-wasm-http) 8 | 9 | 10 | This project is inspired by [@phiresky](https://github.com/phiresky/)/[sql.js-httpvfs](https://github.com/phiresky/sql.js-httpvfs) but uses the new official SQLite WASM distribution. 11 | 12 | It includes a number of improvements over the first version: 13 | * Based upon what will probably be the industry reference (backed by SQLite and Google) 14 | * Supports multiple concurrent connections to the same database with shared cache 15 | 16 | The shared cache version uses `SharedArrayBuffer` which requires that the server hosting the JS code sends [`Cross-Origin-Opener-Policy: same-origin` and `Cross-Origin-Embedder-Policy: require-corp` headers](https://web.dev/coop-coep/) (aka CORS 2). 17 | 18 | * Simplified fall-back version without support for sharing cache between worker threads that does not require `SharedArrayBuffer` 19 | * Aims to support all bundlers out-of-the-box without special configuration 20 | 21 | You can see a [live demo of the shared cache version here](https://sqlite-wasm-http.momtchev.com/). 22 | 23 | The [Github Pages live demo](https://mmomtchev.github.io/sqlite-wasm-http/) uses the sync backend since as of February 2024 Github Pages still does not support cross-origin isolation (please, [upvote](https://github.com/orgs/community/discussions/13309)). 24 | 25 | Ony ES6 module mode is supported at the moment, CommonJS is not supported and this includes TypeScript transpiled to CommonJS - you have to transpile to ES6 in order to use this module. 26 | 27 | You can check [test/integration](https://github.com/mmomtchev/sqlite-wasm-http/blob/main/test/integration) for examples for the various environments that are currently tested and supported. 28 | 29 | Node.js is fully supported but requires `web-worker` and `fetch` available in Node.js 18.x+. 30 | 31 | If you intend to use Node.js only for bundling without using SQLite in a standalone application, then the minimum required version is Node.js 16.x. 32 | 33 | # Status 34 | 35 | Experimental 36 | 37 | # Usage 38 | 39 | If you are not already familiar with [@phiresky](https://github.com/phiresky/)/[sql.js-httpvfs](https://github.com/phiresky/sql.js-httpvfs), there is a brief presentation in the [Overview](#Overview) section. 40 | 41 | You can also check [`ol-mbtiles`](https://github.com/mmomtchev/ol-mbtiles) for an example project that uses this library - it implements remote rendering of HTTP-hosted `.mbtiles` in OpenLayers. 42 | 43 | ## Page size 44 | 45 | It is highly recommended to decrease your SQLite page size to 1024 bytes for maximum performance: 46 | ``` 47 | PRAGMA JOURNAL_MODE = DELETE; 48 | PRAGMA page_size = 1024; 49 | -- Do it for every FTS table you have 50 | -- (geospatial datasets do not use full text search) 51 | INSERT INTO ftstable(ftstable) VALUES ('optimize'); 52 | -- Reorganize database and apply changed page size 53 | -- Sometimes you will be surprised by the new size of your DB 54 | VACUUM; 55 | ``` 56 | 57 | ## Using the SQLite API 58 | 59 | This method allows using the raw SQLite interface with the added support of an HTTP VFS. 60 | 61 | ```typescript 62 | import { createSQLiteThread, createHttpBackend } from 'sqlite-wasm-http'; 63 | 64 | // MBTiles is a common format for storing both vector and 65 | // raster maps in an SQLite database 66 | const remoteURL = 67 | 'https://velivole.b-cdn.net/maptiler-osm-2017-07-03-v3.6.1-europe.mbtiles'; 68 | // createHttpBackend will autodetect if you can use SharedArrayBuffer or not 69 | const httpBackend = createHttpBackend({ 70 | maxPageSize: 4096, // this is the current default SQLite page size 71 | timeout: 10000, // 10s 72 | cacheSize: 4096 // 4 MB 73 | }); 74 | // Multiple DB workers can be created, all sharing the same backend cache 75 | // db is a raw SQLite Promiser object as described here: 76 | // https://sqlite.org/wasm/doc/trunk/api-worker1.md 77 | const db = await createSQLiteThread({ http: httpBackend }); 78 | // This API is compatible with all SQLite VFS 79 | await db('open', { filename: 'file:' + encodeURI(remoteURL), vfs: 'http' }); 80 | await db('exec', { 81 | sql: 'SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles ' + 82 | 'WHERE zoom_level = 10 AND tile_column = $col AND tile_row = $row', 83 | bind: { $col: 600, $row: 600 }, 84 | callback: (msg) => { 85 | if (msg.row) { 86 | console.log(msg.columnNames); 87 | console.log(msg.row); 88 | } else { 89 | console.log('end'); 90 | } 91 | } 92 | }); 93 | // This closes the DB connection 94 | await db('close', {}); 95 | // This terminates the SQLite worker 96 | db.close(); 97 | await httpBackend.close(); 98 | ``` 99 | 100 | ## Using the automated pool 101 | 102 | A higher-level API allows to automatically use concurrent HTTP connections to the same SQLite database. 103 | 104 | Unlike the previous API which is compatible with all SQLite VFS, this one works only for HTTP remote access. 105 | 106 | ```typescript 107 | const remoteURL = 108 | 'https://velivole.b-cdn.net/maptiler-osm-2017-07-03-v3.6.1-europe.mbtiles'; 109 | const pool = await createSQLiteHTTPPool({ workers: 8 }); 110 | await pool.open(remoteURL); 111 | // This will automatically use a free thread from the pool 112 | const tile = await pool.exec( 113 | 'SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles ' + 114 | 'WHERE zoom_level = 10 AND tile_column = $col AND tile_row = $row', 115 | { $col: 600, $row: 600 }); 116 | console.log(tile[0].columnNames); 117 | console.log(tile[0].row); 118 | // This shutdowns the pool 119 | await pool.close(); 120 | ``` 121 | 122 | ## Further information 123 | 124 | * [The full `sqlite-wasm-http` API](https://github.com/mmomtchev/sqlite-wasm-http/blob/integration-tests/docs/API.md) 125 | * [The SQLite WASM main documentation](https://sqlite.org/wasm/doc/trunk/index.md) 126 | 127 | # Overview 128 | 129 | This package includes a browser-compatible version of SQLite3 compiled to WASM. SQLite3 already supports user-defined VFS - both in its original C API and in its very recent new JS API. The JS API includes an OPFS VFS driver which is the reason this project is being pushed by Google and has very good chances of becoming an industry reference. This package adds an additional HTTP VFS driver that uses HTTP `Range` requests - the same that are used by clients supporting resuming of failed downloads - to implement a filesystem-like random access for SQLite3. 130 | 131 | The main drawback of SQLite3 - as it is the case of almost all C/C++ software built to WASM for the web - is that it is fully synchronous. Accordingly, the SQLite3 WASM comes with two APIs - one, fully synchronous, which works a lot like the C/C++ version, and another one - which runs SQLite3 in a Web Worker and communicates with it by message passing implemented in a `Promise`-based wrapper. 132 | 133 | Currently, the builtin multithreading of the C/C++ version of SQLite3 is not enabled in the WASM version. This means that the only way to have multiple concurrent connections to one (or more) databases is to run several independent SQLite3 workers. 134 | 135 | OPFS and HTTP further complicate that situation - as both are intrinsically asynchronous APIs. This is why the HTTP driver comes in two flavors: 136 | * a more modern one, that uses a dedicated HTTP worker thread to run asynchronously all HTTP operations for all SQLite3 workers and implements sharing of the cache between those 137 | * and a more compatible one, that runs synchronously all HTTP operations in the SQLite3 thread that invoked them and does not support cache sharing between workers 138 | 139 | If you do not intend to run concurrent queries using multiple workers, both backends will be equivalent to you. 140 | 141 | The driver is smart enough to select the appropriate backend according to whether `SharedArrayBuffer` is available or not. 142 | 143 | # Will write access ever be possible? 144 | 145 | Short answer: Maybe, in some cases. 146 | 147 | Long answer: It won't have the same universal support as read-only access though. There is a `Content-Range` header for HTTP bodies - that is used in the response of an HTTP `GET` request that carries a `Range` header. The RFC does not say anything about this header being used for `PUT` requests. Most web server do not support it. Apache does support it if the WebDAV extensions are enabled. Maybe other servers support in specific configurations too. Support on public infrastructure servers, especially the low-cost ones, will likely be very rare. 148 | 149 | # Performance 150 | 151 | ## Prefetching and code-splitting 152 | 153 | This module has been designed for being bundled with a modern web bundler - which should correctly identify the chunks to be shared between main and the workers. 154 | 155 | Also, if you bundler supports preloading, preloading the WASM chunk can greatly improve the initial loading times. You should check [`webpack.config.cjs`](https://github.com/mmomtchev/sqlite-wasm-http/blob/main/webpack.config.cjs) for an example that uses preloading. 156 | 157 | ## Apache `httpd` configuration fragment 158 | 159 | These are all the options required for maximum performance: 160 | 161 | ``` 162 | Header always append Cross-Origin-Embedder-Policy "require-corp" 163 | Header always append Cross-Origin-Opener-Policy: "same-origin" 164 | AddOutputFilterByType DEFLATE application/wasm 165 | ``` 166 | 167 | They must be set on the origin - the main entry point as it is displayed in the user's URL bar. When using an `iframe`, the `iframe` must have them **as well as all of its parents up to the origin**, as well as the special `iframe` attribute: `