├── .gitattributes
├── .gitignore
├── .vscode
└── settings.json
├── LICENSE
├── README.md
├── create_db.sh
├── databench.txt
├── example
├── example.sqlite3
├── index.html
├── package.json
├── src
│ └── index.ts
├── tsconfig.json
└── webpack.config.js
├── index.html
├── package.json
├── sql.js
├── .eslintrc.js
├── .github
│ └── workflows
│ │ ├── CI.yml
│ │ └── release.yml
├── .gitignore
├── .gitrepo
├── .jsdoc.config.json
├── .nojekyll
├── .npmignore
├── AUTHORS
├── GUI
│ └── index.html
├── LICENSE
├── Makefile
├── README.md
├── dist
│ ├── .gitignore
│ └── .npmignore
├── documentation_index.md
├── examples
│ ├── GUI
│ │ ├── demo.css
│ │ ├── gui.js
│ │ └── index.html
│ ├── README.md
│ ├── persistent.html
│ ├── repl.html
│ ├── requireJS.html
│ ├── simple.html
│ └── start_local_server.py
├── index.html
├── logo.svg
├── package-lock.json
├── package.json
├── src
│ ├── api.js
│ ├── exported_functions.json
│ ├── exported_runtime_methods.json
│ ├── shell-post.js
│ ├── shell-pre.js
│ └── worker.js
└── test
│ ├── all.js
│ ├── disabled_test_memory_leak_on_error.js
│ ├── issue55.db
│ ├── load_sql_lib.js
│ ├── run.sh
│ ├── sql-requireJS.html
│ ├── test.sqlite
│ ├── test_blob.js
│ ├── test_database.js
│ ├── test_errors.js
│ ├── test_extension_functions.js
│ ├── test_functions.js
│ ├── test_functions_recreate.js
│ ├── test_issue128.js
│ ├── test_issue325.js
│ ├── test_issue55.js
│ ├── test_issue73.js
│ ├── test_issue76.js
│ ├── test_json1.js
│ ├── test_modularization.js
│ ├── test_node_file.js
│ ├── test_statement.js
│ ├── test_statement_iterator.js
│ ├── test_transactions.js
│ └── test_worker.js
├── src
├── db.ts
├── index.ts
├── lazyFile.ts
├── sqlite.worker.ts
├── types.d.ts
├── util.ts
└── vtab.ts
├── tsconfig.json
├── webpack.config.js
└── yarn.lock
/.gitattributes:
--------------------------------------------------------------------------------
1 | sql.js/** linguist-vendored
2 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | dist
3 | /data
4 | /example/package-lock.json
5 |
--------------------------------------------------------------------------------
/.vscode/settings.json:
--------------------------------------------------------------------------------
1 | {
2 | }
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
203 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # sql.js-httpvfs
2 |
3 | See my blog post for an introduction: https://phiresky.github.io/blog/2021/hosting-sqlite-databases-on-github-pages/
4 |
5 | sql.js is a light wrapper around SQLite compiled with EMScripten for use in the browser (client-side).
6 |
7 | This repo is a fork of and wrapper around sql.js to provide a read-only HTTP-Range-request based virtual file system for SQLite. It allows hosting an SQLite database on a static file hoster and querying that database from the browser without fully downloading it.
8 |
9 | The virtual file system is an emscripten filesystem with some "smart" logic to accelerate fetching with virtual read heads that speed up when sequential data is fetched. It could also be useful to other applications, the code is in [lazyFile.ts](./src/lazyFile.ts). It might also be useful to implement this lazy fetching as an [SQLite VFS](https://www.sqlite.org/vfs.html) since then SQLite could be compiled with e.g. WASI SDK without relying on all the emscripten OS emulation.
10 |
11 | Note that this whole thing only works well if your database and indexes are structured well.
12 |
13 | sql.js-httpvfs also provides a proof-of-concept level implementation of a DOM virtual table that allows interacting (read/write) with the browser DOM directly from within SQLite queries.
14 |
15 |
16 | ## Usage
17 |
18 |
19 | (optional) First, improve your SQLite database:
20 |
21 | ```sql
22 | -- first, add whatever indices you need. Note that here having many and correct indices is even more important than for a normal database.
23 | pragma journal_mode = delete; -- to be able to actually set page size
24 | pragma page_size = 1024; -- trade off of number of requests that need to be made vs overhead.
25 |
26 | insert into ftstable(ftstable) values ('optimize'); -- for every FTS table you have (if you have any)
27 |
28 | vacuum; -- reorganize database and apply changed page size
29 | ```
30 |
31 | (optional) Second, split the database into chunks and generate a json config using the [create_db.sh](./create_db.sh) script. This is needed if your hoster has a maximum file size. It can also be a good idea generally depending on your CDN since it allows selective CDN caching of the chunks your users actually use and reduces cache eviction.
32 |
33 | Finally, install sql.js-httpvfs from [npm](https://www.npmjs.com/package/sql.js-httpvfs) and use it in TypeScript / JS!
34 |
35 | Here's an example for people familiar with the JS / TS world. **At the bottom of this readme** there's a more complete example for those unfamiliar.
36 |
37 | ```ts
38 | import { createDbWorker } from "sql.js-httpvfs"
39 |
40 | // sadly there's no good way to package workers and wasm directly so you need a way to get these two URLs from your bundler.
41 | // This is the webpack5 way to create a asset bundle of the worker and wasm:
42 | const workerUrl = new URL(
43 | "sql.js-httpvfs/dist/sqlite.worker.js",
44 | import.meta.url,
45 | );
46 | const wasmUrl = new URL(
47 | "sql.js-httpvfs/dist/sql-wasm.wasm",
48 | import.meta.url,
49 | );
50 | // the legacy webpack4 way is something like `import wasmUrl from "file-loader!sql.js-httpvfs/dist/sql-wasm.wasm"`.
51 |
52 | // the config is either the url to the create_db script, or a inline configuration:
53 | const config = {
54 | from: "inline",
55 | config: {
56 | serverMode: "full", // file is just a plain old full sqlite database
57 | requestChunkSize: 4096, // the page size of the sqlite database (by default 4096)
58 | url: "/foo/bar/test.sqlite3" // url to the database (relative or full)
59 | }
60 | };
61 | // or:
62 | const config = {
63 | from: "jsonconfig",
64 | configUrl: "/foo/bar/config.json"
65 | }
66 |
67 |
68 | let maxBytesToRead = 10 * 1024 * 1024;
69 | const worker = await createDbWorker(
70 | [config],
71 | workerUrl.toString(),
72 | wasmUrl.toString(),
73 | maxBytesToRead // optional, defaults to Infinity
74 | );
75 | // you can also pass multiple config objects which can then be used as separate database schemas with `ATTACH virtualFilename as schemaname`, where virtualFilename is also set in the config object.
76 |
77 |
78 | // worker.db is a now SQL.js instance except that all functions return Promises.
79 |
80 | const result = await worker.db.exec(`select * from table where id = ?`, [123]);
81 |
82 | // worker.worker.bytesRead is a Promise for the number of bytes read by the worker.
83 | // if a request would cause it to exceed maxBytesToRead, that request will throw a SQLite disk I/O error.
84 | console.log(await worker.worker.bytesRead);
85 |
86 | // you can reset bytesRead by assigning to it:
87 | worker.worker.bytesRead = 0;
88 | ```
89 |
90 | ## Cachebusting
91 |
92 | Alongside the `url` or `urlPrefix`, config can take an optional `cacheBust` property whose value will be appended as a query parameter to URLs. If you set it to a random value when you update the database you can avoid caching-related database corruption.
93 |
94 | If using a remote config (`from: 'jsonconfig'`), don't forget to cachebust that too.
95 |
96 | ## Debugging data fetching
97 |
98 | If your query is fetching a lot of data and you're not sure why, try this:
99 |
100 | 1. Look at the output of `explain query plan select ......`
101 |
102 | - `SCAN TABLE t1` means the table t1 will have to be downloaded pretty much fully
103 | - `SCAN TABLE t1 USING INDEX i1 (a=?)` means direct index lookups to find a row, then table lookups by rowid
104 | - `SCAN TABLE t1 USING COVERING INDEX i1 (a)` direct index lookup _without_ a table lookup. This is the fastest.
105 |
106 | You want all the columns in your WHERE clause that significantly reduce the number of results to be part of an index, with the ones reducing the result count the most coming first.
107 |
108 | Another useful technique is to create an index containing exactly the rows filtered by and the rows selected, which SQLite reads as a COVERING INDEX in a sequential manner (no random access at all!). For example `create index i1 on tbl (filteredby1, filteredby2, selected1, selected2, selected3)`. This index is perfect for a query filtering by the `filteredby1` and `filteredby2` columns that only select the three columns at the back of the index.
109 |
110 | 2. You can look at the `dbstat` virtual table to find out exactly what the pages SQLite is reading contain. For example, if you have `[xhr of size 1 KiB @ 1484048 KiB]` in your logs that means it's reading page 1484048. You can get the full log of read pages by using `worker.getResetAccessedPages()`. Check the content of pages with `select * from dbstat where pageno = 1484048`. Do this in an SQLite3 shell not the browser because the `dbstat` vtable reads the whole database.
111 |
112 | ## Is this production ready?
113 |
114 | Note that this library was mainly written for small personal projects of mine and as a demonstration. I've received requests from many people for applications that are out of the scope of this library for me (Which is awesome, and I'm happy to have inspired so many interesting new idea).
115 |
116 | In general it works fine, but I'm not making any effort to support older or weird browsers. If the browser doesn't support WebAssembly and WebWorkers, this won't work. There's also no cache eviction, so the more data is fetched the more RAM it will use. Most of the complicated work is done by SQLite, which is well tested, but the virtual file system part doesn't have any tests.
117 |
118 | If you want to build something new that doesn't fit with this library exactly, I'd recommend you look into these discussions and libraries:
119 |
120 | * The general virtual file system discussion here: https://github.com/sql-js/sql.js/issues/447
121 | * [wa-sqlite](https://github.com/rhashimoto/wa-sqlite), which is a much simpler wasm wrapper for SQLite than sql.js a and has different VFSes that don't require an EMScripten dependency. sql.js-httpvfs could easily be reimplemented on top of this.
122 | * [absurd-sql](https://github.com/jlongster/absurd-sql), which is an implementation of a pretty efficient VFS that allows persistence / read/write queries by storing the DB in IndexedDB
123 |
124 | ## Inspiration
125 |
126 | This project is inspired by:
127 |
128 | * https://github.com/lmatteis/torrent-net and https://github.com/bittorrent/sqltorrent Torrent VFS for SQLite. In theory even more awesome than a httpvfs, but only works with native SQLite not in the browser (someone needs to make a baby of this and sqltorrent to get something that uses WebTorrent).
129 | * https://phiresky.github.io/tv-show-ratings/ a project of mine that fetches the backing data from a WebTorrent (and afterwards seeds it). Not SQLite though, just a torrent with a set of hashed file chunks containing protobufs.
130 | * https://phiresky.github.io/youtube-sponsorship-stats/?uploader=Adam+Ragusea what I originally built sql.js-httpvfs for
131 |
132 | The original code of lazyFile is based on the emscripten createLazyFile function, though not much of that code is remaining.
133 |
134 | ## Minimal example from scratch
135 |
136 | Here's an example of how to setup a project with sql.js-httpvfs completely from scratch, for people unfamiliar with JavaScript or NPM in general.
137 |
138 | First, You will need `node` and `npm`. Get this from your system package manager like `apt install nodejs npm`.
139 |
140 | Then, go to a new directory and add a few dependencies:
141 |
142 | ```sh
143 | mkdir example
144 | cd example
145 | echo '{}' > package.json
146 | npm install --save-dev webpack webpack-cli typescript ts-loader http-server
147 | npm install --save sql.js-httpvfs
148 | npx tsc --init
149 | ```
150 |
151 | Edit the generated tsconfig.json file to make it more modern:
152 | ```json
153 | ...
154 | "target": "es2020",
155 | "module": "es2020",
156 | "moduleResolution": "node",
157 | ...
158 | ```
159 |
160 | Create a webpack config, minimal index.html file and TypeScript entry point:
161 |
162 | * [example/webpack.config.js](./example/webpack.config.js)
163 | * [example/index.html](./example/index.html)
164 | * [example/src/index.ts](./example/src/index.ts)
165 |
166 | Finally, create a database:
167 |
168 | ```sh
169 | sqlite3 example.sqlite3 "create table mytable(foo, bar)"
170 | sqlite3 example.sqlite3 "insert into mytable values ('hello', 'world')"
171 | ```
172 |
173 | and build the JS bundle and start a webserver:
174 |
175 | ```
176 | ./node_modules/.bin/webpack --mode=development
177 | ./node_modules/.bin/http-server
178 | ```
179 |
180 | Then go to http://localhost:8080
181 |
182 | And you should see the output to the query `select * from mytable`.
183 |
184 | ```json
185 | [{"foo":"hello","bar":"world"}]
186 | ```
187 |
188 | The full code of this example is in [example/](./example/).
189 |
190 | ## Compiling
191 |
192 | To compile this project (only needed if you want to modify the library itself), make sure you have emscripten, then first compile sql.js, then sql.js-httpvfs:
193 |
194 | ```
195 | cd sql.js
196 | yarn build
197 | cd ..
198 | yarn build
199 | ```
200 |
--------------------------------------------------------------------------------
/create_db.sh:
--------------------------------------------------------------------------------
1 | set -eu
2 |
3 | indb="$1"
4 | outdir="$2"
5 |
6 | # for chunked mode, we need to know the database size in bytes beforehand
7 | bytes="$(stat --printf="%s" "$indb")"
8 | # set chunk size to 10MiB (needs to be a multiple of the `pragma page_size`!)
9 | serverChunkSize=$((10 * 1024 * 1024))
10 | suffixLength=3
11 | rm -f "$outdir/db.sqlite3"*
12 | split "$indb" --bytes=$serverChunkSize "$outdir/db.sqlite3." --suffix-length=$suffixLength --numeric-suffixes
13 |
14 | # set request chunk size to match page size
15 | requestChunkSize="$(sqlite3 "$indb" 'pragma page_size')"
16 |
17 | # write a json config
18 | echo '
19 | {
20 | "serverMode": "chunked",
21 | "requestChunkSize": '$requestChunkSize',
22 | "databaseLengthBytes": '$bytes',
23 | "serverChunkSize": '$serverChunkSize',
24 | "urlPrefix": "db.sqlite3.",
25 | "suffixLength": '$suffixLength'
26 | }
27 | ' > "$outdir/config.json"
28 |
--------------------------------------------------------------------------------
/databench.txt:
--------------------------------------------------------------------------------
1 | query: select * from videoData where author = 'Adam Ragusea' limit 20;
2 |
3 | without index:
4 |
5 | req size, pg
6 |
7 | req size 4096, youtube-metadata-pg4096.sqlite3 total bytes fetched: 581490 total requests: 142
8 | req size 512, youtube-metadata-pg4096.sqlite3 total bytes fetched: 580496 total requests: 1136
9 | req size 4096, youtube-metadata-pg512.sqlite3 total bytes fetched: 782145 total requests: 191
10 | req size 512, youtube-metadata-pg512.sqlite3 total bytes fetched: 761901 total requests: 1491
11 |
12 | req size 4096, youtube-metadata-pg32768.sqlite3 total bytes fetched: 622440 total requests: 152
13 | req size 32768, youtube-metadata-pg32768.sqlite3 total bytes fetched: 622573 total requests: 19
14 |
15 | req size 4096, youtube-metadata-pg16384.sqlite3 total bytes fetched: 606060 total requests: 148
16 | req size 16384, youtube-metadata-pg16384.sqlite3 total bytes fetched: 606171 total requests: 37
17 | req size 32768, youtube-metadata-pg16384.sqlite3 total bytes fetched: 688107 total requests: 21
18 |
19 | req size 16384, youtube-metadata-pg16384-aligned.sqlite3 total bytes fetched: 1736598 total requests: 106
20 |
21 | with index on videoData(author) (page size always equal to request size):
22 |
23 |
24 | youtube-metadata-pg16384-aligned.sqlite3 total bytes fetched: 98298 total requests: 6
25 |
26 | youtube-metadata-pg32768.sqlite3 total bytes fetched: 491505 total requests: 15
27 | youtube-metadata-pg16384.sqlite3 total bytes fetched: 327660 total requests: 20
28 | youtube-metadata-pg4096.sqlite3 total bytes fetched: 98280 total requests: 24
29 | youtube-metadata-pg512.sqlite3 total bytes fetched: 24017 total requests: 47
--------------------------------------------------------------------------------
/example/example.sqlite3:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/phiresky/sql.js-httpvfs/c64536d2acc78feeac17c34bfa1895df01050129/example/example.sqlite3
--------------------------------------------------------------------------------
/example/index.html:
--------------------------------------------------------------------------------
1 | <!DOCTYPE html>
2 |
3 | <title>Minimal sql.js-httpvfs demo</title>
4 | <script src="./dist/bundle.js"></script>
5 | <div>Hello World!</div>
6 |
--------------------------------------------------------------------------------
/example/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "dependencies": {
3 | "sql.js-httpvfs": "^0.8.8"
4 | },
5 | "devDependencies": {
6 | "http-server": "^0.12.3",
7 | "ts-loader": "^9.1.2",
8 | "typescript": "^4.2.4",
9 | "webpack": "^5.36.2",
10 | "webpack-cli": "^4.7.0"
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/example/src/index.ts:
--------------------------------------------------------------------------------
1 | import { createDbWorker } from "sql.js-httpvfs";
2 |
3 | const workerUrl = new URL(
4 | "sql.js-httpvfs/dist/sqlite.worker.js",
5 | import.meta.url
6 | );
7 | const wasmUrl = new URL("sql.js-httpvfs/dist/sql-wasm.wasm", import.meta.url);
8 |
9 | async function load() {
10 | const worker = await createDbWorker(
11 | [
12 | {
13 | from: "inline",
14 | config: {
15 | serverMode: "full",
16 | url: "/example.sqlite3",
17 | requestChunkSize: 4096,
18 | },
19 | },
20 | ],
21 | workerUrl.toString(),
22 | wasmUrl.toString()
23 | );
24 |
25 | const result = await worker.db.query(`select * from mytable`);
26 |
27 | document.body.textContent = JSON.stringify(result);
28 | }
29 |
30 | load();
31 |
--------------------------------------------------------------------------------
/example/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | /* Visit https://aka.ms/tsconfig.json to read more about this file */
4 |
5 | /* Basic Options */
6 | // "incremental": true, /* Enable incremental compilation */
7 | "target": "es2020", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
8 | "module": "es2020", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
9 | // "lib": [], /* Specify library files to be included in the compilation. */
10 | // "allowJs": true, /* Allow javascript files to be compiled. */
11 | // "checkJs": true, /* Report errors in .js files. */
12 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
13 | // "declaration": true, /* Generates corresponding '.d.ts' file. */
14 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
15 | // "sourceMap": true, /* Generates corresponding '.map' file. */
16 | // "outFile": "./", /* Concatenate and emit output to single file. */
17 | // "outDir": "./", /* Redirect output structure to the directory. */
18 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
19 | // "composite": true, /* Enable project compilation */
20 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
21 | // "removeComments": true, /* Do not emit comments to output. */
22 | // "noEmit": true, /* Do not emit outputs. */
23 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */
24 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
25 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
26 |
27 | /* Strict Type-Checking Options */
28 | "strict": true, /* Enable all strict type-checking options. */
29 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
30 | // "strictNullChecks": true, /* Enable strict null checks. */
31 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */
32 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
33 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
34 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
35 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
36 |
37 | /* Additional Checks */
38 | // "noUnusedLocals": true, /* Report errors on unused locals. */
39 | // "noUnusedParameters": true, /* Report errors on unused parameters. */
40 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
41 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
42 | // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
43 | // "noPropertyAccessFromIndexSignature": true, /* Require undeclared properties from index signatures to use element accesses. */
44 |
45 | /* Module Resolution Options */
46 | "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
47 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
48 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
49 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
50 | // "typeRoots": [], /* List of folders to include type definitions from. */
51 | // "types": [], /* Type declaration files to be included in compilation. */
52 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
53 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
54 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
55 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
56 |
57 | /* Source Map Options */
58 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
59 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
60 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
61 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
62 |
63 | /* Experimental Options */
64 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
65 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
66 |
67 | /* Advanced Options */
68 | "skipLibCheck": true, /* Skip type checking of declaration files. */
69 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
70 | }
71 | }
72 |
--------------------------------------------------------------------------------
/example/webpack.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | entry: "./src/index.ts",
3 | module: {
4 | rules: [
5 | {
6 | test: /\.tsx?$/,
7 | use: "ts-loader",
8 | exclude: /node_modules/,
9 | },
10 | ],
11 | },
12 | resolve: {
13 | extensions: [".tsx", ".ts", ".js"],
14 | },
15 | output: {
16 | filename: "bundle.js",
17 | },
18 | devServer: {
19 | publicPath: "/dist",
20 | },
21 | };
22 |
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
1 | <!DOCTYPE html>
2 |
3 | <meta charset="utf-8">
4 |
5 | <title>Sponsorship stat by uploader</title>
6 |
7 | <meta http-equiv="X-UA-Compatible" content="IE=edge">
8 | <meta name="viewport" content="width=device-width, initial-scale=1">
9 |
10 | <div id="root"></div>
11 | <script src="bundle.js"></script>
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "sql.js-httpvfs",
3 | "version": "0.8.12",
4 | "repository": "https://github.com/phiresky/sql.js-httpvfs",
5 | "license": "Apache-2.0",
6 | "scripts": {
7 | "dev": "webpack serve --mode=development",
8 | "build": "webpack build --mode=production"
9 | },
10 | "dependencies": {
11 | "comlink": "^4.3.0"
12 | },
13 | "devDependencies": {
14 | "@types/debounce-promise": "^3.1.3",
15 | "@types/react": "^17.0.3",
16 | "@types/react-dom": "^17.0.3",
17 | "@types/react-plotly.js": "^2.2.4",
18 | "@types/react-select": "^4.0.14",
19 | "@types/sql.js": "^1.4.2",
20 | "debounce-promise": "^3.1.2",
21 | "ts-loader": "^8.1.0",
22 | "ts-node": "^9.1.1",
23 | "typescript": "^4.3.0-dev.20210331",
24 | "webpack": "^5.28.0",
25 | "webpack-bundle-analyzer": "^4.4.1",
26 | "webpack-cli": "^4.6.0",
27 | "webpack-dev-server": "^3.11.2"
28 | },
29 | "main": "dist/index.js",
30 | "files": [
31 | "dist/*"
32 | ]
33 | }
34 |
--------------------------------------------------------------------------------
/sql.js/.eslintrc.js:
--------------------------------------------------------------------------------
1 | "use strict";
2 |
3 | module.exports = {
4 | env: {
5 | browser: true,
6 | es6: true,
7 | node: true
8 | },
9 | extends: [
10 | "airbnb-base"
11 | ],
12 | globals: {
13 | Atomics: "readonly",
14 | SharedArrayBuffer: "readonly"
15 | },
16 | ignorePatterns: [
17 | "/dist/",
18 | "/examples/",
19 | "/node_modules/",
20 | "/out/",
21 | "/src/shell-post.js",
22 | "/src/shell-pre.js",
23 | "/test/",
24 | "!/.eslintrc.js"
25 | ],
26 | parserOptions: {
27 | ecmaVersion: 5,
28 | sourceType: "script"
29 | },
30 | rules: {
31 | // reason - sqlite exposes functions with underscore-naming-convention
32 | camelcase: "off",
33 | // reason - They make it easier to add new elements to arrays
34 | // and parameters to functions, and make commit diffs clearer
35 | "comma-dangle": "off",
36 | // reason - string-notation needed to prevent closure-minifier
37 | // from mangling property-name
38 | "dot-notation": "off",
39 | // reason - enforce 4-space indent
40 | indent: ["error", 4, { SwitchCase: 1 }],
41 | // reason - enforce 80-column-width limit
42 | "max-len": ["error", { code: 80 }],
43 | // reason - src/api.js uses bitwise-operators
44 | "no-bitwise": "off",
45 | "no-cond-assign": ["error", "except-parens"],
46 | "no-param-reassign": "off",
47 | "no-throw-literal": "off",
48 | // reason - parserOptions is set to es5 language-syntax
49 | "no-var": "off",
50 | // reason - parserOptions is set to es5 language-syntax
51 | "object-shorthand": "off",
52 | // reason - parserOptions is set to es5 language-syntax
53 | "prefer-arrow-callback": "off",
54 | // reason - parserOptions is set to es5 language-syntax
55 | "prefer-destructuring": "off",
56 | // reason - parserOptions is set to es5 language-syntax
57 | "prefer-spread": "off",
58 | // reason - parserOptions is set to es5 language-syntax
59 | "prefer-template": "off",
60 | // reason - sql.js frequently use sql-query-strings containing
61 | // single-quotes
62 | quotes: ["error", "double"],
63 | // reason - allow top-level "use-strict" in commonjs-modules
64 | strict: ["error", "safe"],
65 | "vars-on-top": "off"
66 | }
67 | };
68 |
--------------------------------------------------------------------------------
/sql.js/.github/workflows/CI.yml:
--------------------------------------------------------------------------------
1 | name: CI
2 |
3 | on: [push, pull_request]
4 |
5 | jobs:
6 | build:
7 | runs-on: ubuntu-latest
8 | steps:
9 | - uses: actions/checkout@v2
10 | - uses: actions/cache@v1
11 | id: cache
12 | with:
13 | path: '.emsdk-cache'
14 | key: emscripten-2.0.6
15 | - uses: mymindstorm/setup-emsdk@ca33dc66a6b178f65393989c12e9465baf053352
16 | with:
17 | version: '2.0.6'
18 | actions-cache-folder: '.emsdk-cache'
19 | - name: make
20 | run: make
21 | - uses: actions/upload-artifact@v2
22 | with: {name: dist, path: dist}
23 | - name: test
24 | run: npm ci && npm test
25 | - name: generate documentation
26 | run: npm run doc
27 | - name: Update github pages
28 | if: github.event_name == 'push' && github.ref == 'refs/heads/master'
29 | uses: JamesIves/github-pages-deploy-action@3.6.2
30 | with:
31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
32 | BRANCH: gh-pages # The branch the action should deploy to.
33 | FOLDER: "." # The folder the action should deploy.
34 | CLEAN: false # Automatically remove deleted files from the deploy branch
35 |
--------------------------------------------------------------------------------
/sql.js/.github/workflows/release.yml:
--------------------------------------------------------------------------------
1 | on:
2 | push:
3 | # Sequence of patterns matched against refs/tags
4 | tags:
5 | - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
6 |
7 | name: Create a release
8 |
9 | jobs:
10 | build:
11 | name: Create a release
12 | runs-on: ubuntu-latest
13 | steps:
14 | - uses: actions/checkout@v2
15 | - uses: mymindstorm/setup-emsdk@ca33dc66a6b178f65393989c12e9465baf053352
16 | with: {version: '2.0.6'}
17 | - name: make
18 | run: make
19 | - name: Create Release
20 | id: create_release
21 | uses: actions/create-release@v1.0.0
22 | env:
23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
24 | with:
25 | tag_name: ${{ github.ref }}
26 | release_name: Release ${{ github.ref }}
27 | draft: false
28 | prerelease: false
29 | - run: cd dist && zip sqljs-wasm.zip sql-wasm.{js,wasm}
30 | - name: Upload Release Asset (wasm)
31 | uses: lovasoa/upload-release-asset@851d9cc59fe8113912edffbd8fddaa09470a5ac0
32 | env:
33 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
34 | with:
35 | upload_url: ${{ steps.create_release.outputs.upload_url }}
36 | asset_path: dist/sqljs-wasm.zip
37 | asset_name: sqljs-wasm.zip
38 | asset_label: wasm version, best runtime performance, smaller assets, requires configuration
39 | asset_content_type: application/zip
40 | - name: Upload Release Asset (asm)
41 | uses: lovasoa/upload-release-asset@851d9cc59fe8113912edffbd8fddaa09470a5ac0
42 | env:
43 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
44 | with:
45 | upload_url: ${{ steps.create_release.outputs.upload_url }}
46 | asset_path: dist/sql-asm.js
47 | asset_name: sql.js
48 | asset_label: asm.js version, slower, easy to integrate and compatible with old browsers
49 | asset_content_type: text/javascript
50 | - run: cd dist && zip sqljs-worker-wasm.zip worker.sql-wasm.js sql-wasm.wasm
51 | - name: Upload Release Asset (worker wasm)
52 | uses: lovasoa/upload-release-asset@851d9cc59fe8113912edffbd8fddaa09470a5ac0
53 | env:
54 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
55 | with:
56 | upload_url: ${{ steps.create_release.outputs.upload_url }}
57 | asset_path: dist/sqljs-worker-wasm.zip
58 | asset_name: sqljs-worker-wasm.zip
59 | asset_label: webworker wasm version, to be loaded as a web worker
60 | asset_content_type: application/zip
61 | - name: Upload Release Asset (worker asm)
62 | uses: lovasoa/upload-release-asset@851d9cc59fe8113912edffbd8fddaa09470a5ac0
63 | env:
64 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
65 | with:
66 | upload_url: ${{ steps.create_release.outputs.upload_url }}
67 | asset_path: dist/worker.sql-asm.js
68 | asset_name: worker.sql-asm.js
69 | asset_label: webworker asm version, to be loaded as a web worker
70 | asset_content_type: text/javascript
71 | - run: cd dist && zip sqljs-all.zip *.{js,wasm}
72 | - name: Upload Release Asset (all)
73 | uses: lovasoa/upload-release-asset@851d9cc59fe8113912edffbd8fddaa09470a5ac0
74 | env:
75 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
76 | with:
77 | upload_url: ${{ steps.create_release.outputs.upload_url }}
78 | asset_path: dist/sqljs-all.zip
79 | asset_name: sqljs-all.zip
80 | asset_label: all versions, including non-minified javascript
81 | asset_content_type: application/zip
82 | - name: publish the package to NPM
83 | run: |
84 | npm config set '//registry.npmjs.org/:_authToken' "${NPM_TOKEN}"
85 | npm publish
86 | env:
87 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
88 |
--------------------------------------------------------------------------------
/sql.js/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | *~
3 |
4 | # Intermediary files:
5 | cache/
6 | out/
7 | .emsdk-cache/
8 | sqlite-src/
9 | tmp/
10 | c/
11 | emsdk/
12 | sqljs.zip
13 |
--------------------------------------------------------------------------------
/sql.js/.gitrepo:
--------------------------------------------------------------------------------
1 | ; DO NOT EDIT (unless you know what you are doing)
2 | ;
3 | ; This subdirectory is a git "subrepo", and this file is maintained by the
4 | ; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme
5 | ;
6 | [subrepo]
7 | remote = git@github.com:phiresky/sql.js.git
8 | branch = master
9 | commit = 837d25db2fa248033016f08cd395989a0794ad18
10 | parent = 2eb5fdc1679729d99993ea0abdd05ba5de27b83a
11 | method = merge
12 | cmdver = 0.4.3
13 |
--------------------------------------------------------------------------------
/sql.js/.jsdoc.config.json:
--------------------------------------------------------------------------------
1 | {
2 | "plugins": [
3 | "plugins/markdown"
4 | ],
5 | "source": {
6 | "include": [
7 | "src/api.js"
8 | ]
9 | },
10 | "opts": {
11 | "encoding": "utf8",
12 | "destination": "./documentation/",
13 | "readme": "documentation_index.md",
14 | "template": "./node_modules/clean-jsdoc-theme",
15 | "theme_opts": {
16 | "title": "sql.js",
17 | "meta": [
18 | "<title>sql.js API documentation</title>",
19 | "<meta name=\"author\" content=\"Ophir Lojkine\">",
20 | "<meta name=\"description\" content=\"Documentation for sql.js: an in-memory SQL database for the browser based on SQLite.\">"
21 | ],
22 | "menu": [
23 | {
24 | "title": "Website",
25 | "link": "https://sql.js.org/"
26 | },
27 | {
28 | "title": "Github",
29 | "link": "https://github.com/sql-js/sql.js"
30 | },
31 | {
32 | "title": "Demo",
33 | "link": "https://sql.js.org/examples/GUI/"
34 | }
35 | ]
36 | }
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/sql.js/.nojekyll:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/phiresky/sql.js-httpvfs/c64536d2acc78feeac17c34bfa1895df01050129/sql.js/.nojekyll
--------------------------------------------------------------------------------
/sql.js/.npmignore:
--------------------------------------------------------------------------------
1 | test/
2 | c/
3 | gh-pages/
4 | node_modules/
5 | node-debug.log
6 | src/
7 | cache/
8 | out/
9 | examples/
10 | sqlite-src/
11 | .git/
12 | index.html
13 | .github
14 | Makefile
15 | emsdk_set_env.sh
16 | sqljs.zip
17 |
--------------------------------------------------------------------------------
/sql.js/AUTHORS:
--------------------------------------------------------------------------------
1 | Ophir LOJKINE <pere.jobs@gmail.com> (https://github.com/lovasoa)
2 | @kripken
3 | @hankinsoft
4 | @firien
5 | @dinedal
6 | @taytay
7 | @kaizhu256
8 | @brodybits
9 |
--------------------------------------------------------------------------------
/sql.js/GUI/index.html:
--------------------------------------------------------------------------------
1 | <html>
2 | <body>
3 | <!-- Add a link to the new page because there are likely links pointing here from elsewhere -->
4 | This page has moved to: <a href="../examples/GUI/index.html">../examples/GUI/index.html</a>
5 | </body>
6 | </html>
--------------------------------------------------------------------------------
/sql.js/LICENSE:
--------------------------------------------------------------------------------
1 | MIT license
2 | ===========
3 |
4 | Copyright (c) 2017 sql.js authors (see AUTHORS)
5 |
6 | Permission is hereby granted, free of charge, to any person obtaining a copy
7 | of this software and associated documentation files (the "Software"), to deal
8 | in the Software without restriction, including without limitation the rights
9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 | copies of the Software, and to permit persons to whom the Software is
11 | furnished to do so, subject to the following conditions:
12 |
13 | The above copyright notice and this permission notice shall be included in all
14 | copies or substantial portions of the Software.
15 |
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 | SOFTWARE.
23 |
24 |
25 |
26 | # Some portions of the Makefile taken from:
27 | Copyright 2017 Ryusei Yamaguchi
28 |
29 | Permission is hereby granted, free of charge, to any person obtaining a copy of
30 | this software and associated documentation files (the "Software"), to deal in
31 | the Software without restriction, including without limitation the rights to
32 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
33 | the Software, and to permit persons to whom the Software is furnished to do so,
34 | subject to the following conditions:
35 |
36 | The above copyright notice and this permission notice shall be included in all
37 | copies or substantial portions of the Software.
38 |
39 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
40 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
41 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
42 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
43 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
44 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--------------------------------------------------------------------------------
/sql.js/Makefile:
--------------------------------------------------------------------------------
1 | # Note: Last built with version 1.38.30 of Emscripten
2 |
3 | # TODO: Emit a file showing which version of emcc and SQLite was used to compile the emitted output.
4 | # TODO: Create a release on Github with these compiled assets rather than checking them in
5 | # TODO: Consider creating different files based on browser vs module usage: https://github.com/vuejs/vue/tree/dev/dist
6 |
7 | # I got this handy makefile syntax from : https://github.com/mandel59/sqlite-wasm (MIT License) Credited in LICENSE
8 | # To use another version of Sqlite, visit https://www.sqlite.org/download.html and copy the appropriate values here:
9 | SQLITE_AMALGAMATION = sqlite-amalgamation-3350000
10 | SQLITE_AMALGAMATION_ZIP_URL = https://www.sqlite.org/2021/sqlite-amalgamation-3350000.zip
11 | SQLITE_AMALGAMATION_ZIP_SHA1 = ba64bad885c9f51df765a9624700747e7bf21b79
12 |
13 | # Note that extension-functions.c hasn't been updated since 2010-02-06, so likely doesn't need to be updated
14 | EXTENSION_FUNCTIONS = extension-functions.c
15 | EXTENSION_FUNCTIONS_URL = https://www.sqlite.org/contrib/download/extension-functions.c?get=25
16 | EXTENSION_FUNCTIONS_SHA1 = c68fa706d6d9ff98608044c00212473f9c14892f
17 |
18 | EMCC=emcc
19 |
20 | CFLAGS = \
21 | -O2 \
22 | -DSQLITE_OMIT_LOAD_EXTENSION \
23 | -DSQLITE_DISABLE_LFS \
24 | -DSQLITE_ENABLE_FTS3 \
25 | -DSQLITE_ENABLE_FTS3_PARENTHESIS \
26 | -DSQLITE_ENABLE_FTS5 \
27 | -DSQLITE_ENABLE_JSON1 \
28 | -DSQLITE_THREADSAFE=0 \
29 | -DSQLITE_ENABLE_NORMALIZE
30 |
31 | # When compiling to WASM, enabling memory-growth is not expected to make much of an impact, so we enable it for all builds
32 | # Since tihs is a library and not a standalone executable, we don't want to catch unhandled Node process exceptions
33 | # So, we do : `NODEJS_CATCH_EXIT=0`, which fixes issue: https://github.com/sql-js/sql.js/issues/173 and https://github.com/sql-js/sql.js/issues/262
34 | EMFLAGS = \
35 | --memory-init-file 0 \
36 | -s RESERVED_FUNCTION_POINTERS=64 \
37 | -s ALLOW_TABLE_GROWTH=1 \
38 | -s EXPORTED_FUNCTIONS=@src/exported_functions.json \
39 | -s EXTRA_EXPORTED_RUNTIME_METHODS=@src/exported_runtime_methods.json \
40 | -s SINGLE_FILE=0 \
41 | -s NODEJS_CATCH_EXIT=0 \
42 | -s NODEJS_CATCH_REJECTION=0 \
43 | -s LEGACY_RUNTIME=1
44 |
45 | # -s ASYNCIFY=1 \
46 | # -s ASYNCIFY_IMPORTS='["sqlite3VdbeExec"]'
47 | EMFLAGS_ASM = \
48 | -s WASM=0
49 |
50 | EMFLAGS_ASM_MEMORY_GROWTH = \
51 | -s WASM=0 \
52 | -s ALLOW_MEMORY_GROWTH=1
53 |
54 | EMFLAGS_WASM = \
55 | -s WASM=1 \
56 | -s ALLOW_MEMORY_GROWTH=1
57 |
58 | EMFLAGS_OPTIMIZED= \
59 | -s INLINING_LIMIT=1 \
60 | -O3 \
61 | -flto
62 |
63 | EMFLAGS_DEBUG = \
64 | -s INLINING_LIMIT=1 \
65 | -s ASSERTIONS=1 \
66 | -s SAFE_HEAP=1 \
67 | -O0
68 |
69 | BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc
70 |
71 | OUTPUT_WRAPPER_FILES = src/shell-pre.js src/shell-post.js
72 |
73 | SOURCE_API_FILES = src/api.js
74 |
75 | EMFLAGS_PRE_JS_FILES = \
76 | --pre-js src/api.js
77 |
78 | EXPORTED_METHODS_JSON_FILES = src/exported_functions.json src/exported_runtime_methods.json
79 |
80 | all: optimized debug worker
81 |
82 | .PHONY: debug
83 | debug: dist/sql-asm-debug.js dist/sql-wasm-debug.js
84 |
85 | dist/sql-asm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
86 | $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
87 | mv $@ out/tmp-raw.js
88 | cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
89 | rm out/tmp-raw.js
90 |
91 | dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
92 | $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
93 | mv $@ out/tmp-raw.js
94 | cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
95 | rm out/tmp-raw.js
96 |
97 | .PHONY: optimized
98 | optimized: dist/sql-asm.js dist/sql-wasm.js dist/sql-asm-memory-growth.js
99 |
100 | dist/sql-asm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
101 | $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
102 | mv $@ out/tmp-raw.js
103 | cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
104 | rm out/tmp-raw.js
105 |
106 | dist/sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
107 | $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
108 | mv $@ out/tmp-raw.js
109 | cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
110 | rm out/tmp-raw.js
111 |
112 | dist/sql-asm-memory-growth.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
113 | $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
114 | mv $@ out/tmp-raw.js
115 | cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
116 | rm out/tmp-raw.js
117 |
118 | # Web worker API
119 | .PHONY: worker
120 | worker: dist/worker.sql-asm.js dist/worker.sql-asm-debug.js dist/worker.sql-wasm.js dist/worker.sql-wasm-debug.js
121 |
122 | dist/worker.sql-asm.js: dist/sql-asm.js src/worker.js
123 | cat $^ > $@
124 |
125 | dist/worker.sql-asm-debug.js: dist/sql-asm-debug.js src/worker.js
126 | cat $^ > $@
127 |
128 | dist/worker.sql-wasm.js: dist/sql-wasm.js src/worker.js
129 | cat $^ > $@
130 |
131 | dist/worker.sql-wasm-debug.js: dist/sql-wasm-debug.js src/worker.js
132 | cat $^ > $@
133 |
134 | # Building it this way gets us a wrapper that _knows_ it's in worker mode, which is nice.
135 | # However, since we can't tell emcc that we don't need the wasm generated, and just want the wrapper, we have to pay to have the .wasm generated
136 | # even though we would have already generated it with our sql-wasm.js target above.
137 | # This would be made easier if this is implemented: https://github.com/emscripten-core/emscripten/issues/8506
138 | # dist/worker.sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) src/api.js src/worker.js $(EXPORTED_METHODS_JSON_FILES) dist/sql-wasm-debug.wasm
139 | # $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) -s ENVIRONMENT=worker -s $(EMFLAGS_WASM) $(BITCODE_FILES) --pre-js src/api.js -o out/sql-wasm.js
140 | # mv out/sql-wasm.js out/tmp-raw.js
141 | # cat src/shell-pre.js out/tmp-raw.js src/shell-post.js src/worker.js > $@
142 | # #mv out/sql-wasm.wasm dist/sql-wasm.wasm
143 | # rm out/tmp-raw.js
144 |
145 | # dist/worker.sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) src/api.js src/worker.js $(EXPORTED_METHODS_JSON_FILES) dist/sql-wasm-debug.wasm
146 | # $(EMCC) -s ENVIRONMENT=worker $(EMFLAGS) $(EMFLAGS_DEBUG) -s ENVIRONMENT=worker -s WASM_BINARY_FILE=sql-wasm-foo.debug $(EMFLAGS_WASM) $(BITCODE_FILES) --pre-js src/api.js -o out/sql-wasm-debug.js
147 | # mv out/sql-wasm-debug.js out/tmp-raw.js
148 | # cat src/shell-pre.js out/tmp-raw.js src/shell-post.js src/worker.js > $@
149 | # #mv out/sql-wasm-debug.wasm dist/sql-wasm-debug.wasm
150 | # rm out/tmp-raw.js
151 |
152 | out/sqlite3.bc: sqlite-src/$(SQLITE_AMALGAMATION)
153 | mkdir -p out
154 | # Generate llvm bitcode
155 | $(EMCC) $(CFLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/sqlite3.c -o $@
156 |
157 | # Since the extension-functions.c includes other headers in the sqlite_amalgamation, we declare that this depends on more than just extension-functions.c
158 | out/extension-functions.bc: sqlite-src/$(SQLITE_AMALGAMATION)
159 | mkdir -p out
160 | # Generate llvm bitcode
161 | $(EMCC) $(CFLAGS) -s LINKABLE=1 -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@
162 |
163 | # TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder
164 | # module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js
165 | # tar --create --gzip $^ > $@
166 |
167 | ## cache
168 | cache/$(SQLITE_AMALGAMATION).zip:
169 | mkdir -p cache
170 | curl -LsSf '$(SQLITE_AMALGAMATION_ZIP_URL)' -o $@
171 |
172 | cache/$(EXTENSION_FUNCTIONS):
173 | mkdir -p cache
174 | curl -LsSf '$(EXTENSION_FUNCTIONS_URL)' -o $@
175 |
176 | ## sqlite-src
177 | .PHONY: sqlite-src
178 | sqlite-src: sqlite-src/$(SQLITE_AMALGAMATION) sqlite-src/$(SQLITE_AMALGAMATION)/$(EXTENSION_FUNCTIONS)
179 |
180 | sqlite-src/$(SQLITE_AMALGAMATION): cache/$(SQLITE_AMALGAMATION).zip sqlite-src/$(SQLITE_AMALGAMATION)/$(EXTENSION_FUNCTIONS)
181 | mkdir -p sqlite-src/$(SQLITE_AMALGAMATION)
182 | echo '$(SQLITE_AMALGAMATION_ZIP_SHA1) ./cache/$(SQLITE_AMALGAMATION).zip' > cache/check.txt
183 | sha1sum -c cache/check.txt
184 | # We don't delete the sqlite_amalgamation folder. That's a job for clean
185 | # Also, the extension functions get copied here, and if we get the order of these steps wrong,
186 | # this step could remove the extension functions, and that's not what we want
187 | unzip -u 'cache/$(SQLITE_AMALGAMATION).zip' -d sqlite-src/
188 | touch $@
189 |
190 | sqlite-src/$(SQLITE_AMALGAMATION)/$(EXTENSION_FUNCTIONS): cache/$(EXTENSION_FUNCTIONS)
191 | mkdir -p sqlite-src/$(SQLITE_AMALGAMATION)
192 | echo '$(EXTENSION_FUNCTIONS_SHA1) ./cache/$(EXTENSION_FUNCTIONS)' > cache/check.txt
193 | sha1sum -c cache/check.txt
194 | cp 'cache/$(EXTENSION_FUNCTIONS)' $@
195 |
196 |
197 | .PHONY: clean
198 | clean:
199 | rm -f out/* dist/* cache/*
200 | rm -rf sqlite-src/
201 |
--------------------------------------------------------------------------------
/sql.js/README.md:
--------------------------------------------------------------------------------
1 | <img src="https://user-images.githubusercontent.com/552629/76405509-87025300-6388-11ea-86c9-af882abb00bd.png" width="40" height="40" />
2 |
3 | # SQLite compiled to JavaScript
4 |
5 | [](https://github.com/sql-js/sql.js/actions)
6 | [](https://www.npmjs.com/package/sql.js)
7 | [](https://cdnjs.com/libraries/sql.js)
8 |
9 | *sql.js* is a javascript SQL database. It allows you to create a relational database and query it entirely in the browser. You can try it in [this online demo](https://sql.js.org/examples/GUI/). It uses a [virtual database file stored in memory](https://emscripten.org/docs/porting/files/file_systems_overview.html), and thus **doesn't persist the changes** made to the database. However, it allows you to **import** any existing sqlite file, and to **export** the created database as a [JavaScript typed array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Typed_arrays).
10 |
11 | *sql.js* uses [emscripten](https://emscripten.org/docs/introducing_emscripten/about_emscripten.html) to compile [SQLite](http://sqlite.org/about.html) to webassembly (or to javascript code for compatibility with older browsers). It includes [contributed math and string extension functions](https://www.sqlite.org/contrib?orderby=date).
12 |
13 | sql.js can be used like any traditional JavaScript library. If you are building a native application in JavaScript (using Electron for instance), or are working in node.js, you will likely prefer to use [a native binding of SQLite to JavaScript](https://www.npmjs.com/package/sqlite3). A native binding will not only be faster because it will run native code, but it will also be able to work on database files directly instead of having to load the entire database in memory, avoiding out of memory errors and further improving performances.
14 |
15 | SQLite is public domain, sql.js is MIT licensed.
16 |
17 | ## API documentation
18 | A [full API documentation](https://sql.js.org/documentation/) for all the available classes and methods is available.
19 | Is is generated from comments inside the source code, and is thus always up to date.
20 |
21 | ## Usage
22 |
23 | By default, *sql.js* uses [wasm](https://developer.mozilla.org/en-US/docs/WebAssembly), and thus needs to load a `.wasm` file in addition to the javascript library. You can find this file in `./node_modules/sql.js/dist/sql-wasm.wasm` after installing sql.js from npm, and instruct your bundler to add it to your static assets or load it from [a CDN](https://cdnjs.com/libraries/sql.js). Then use the [`locateFile`](https://emscripten.org/docs/api_reference/module.html#Module.locateFile) property of the configuration object passed to `initSqlJs` to indicate where the file is. If you use an asset builder such as webpack, you can automate this. See [this demo of how to integrate sql.js with webpack (and react)](https://github.com/sql-js/react-sqljs-demo).
24 |
25 | ```javascript
26 | const initSqlJs = require('sql.js');
27 | // or if you are in a browser:
28 | // var initSqlJs = window.initSqlJs;
29 |
30 | const SQL = await initSqlJs({
31 | // Required to load the wasm binary asynchronously. Of course, you can host it wherever you want
32 | // You can omit locateFile completely when running in node
33 | locateFile: file => `https://sql.js.org/dist/${file}`
34 | });
35 |
36 | // Create a database
37 | var db = new SQL.Database();
38 | // NOTE: You can also use new SQL.Database(data) where
39 | // data is an Uint8Array representing an SQLite database file
40 |
41 | // Prepare an sql statement
42 | var stmt = db.prepare("SELECT * FROM hello WHERE a=:aval AND b=:bval");
43 |
44 | // Bind values to the parameters and fetch the results of the query
45 | var result = stmt.getAsObject({':aval' : 1, ':bval' : 'world'});
46 | console.log(result); // Will print {a:1, b:'world'}
47 |
48 | // Bind other values
49 | stmt.bind([0, 'hello']);
50 | while (stmt.step()) console.log(stmt.get()); // Will print [0, 'hello']
51 | // free the memory used by the statement
52 | stmt.free();
53 | // You can not use your statement anymore once it has been freed.
54 | // But not freeing your statements causes memory leaks. You don't want that.
55 |
56 | // Execute a single SQL string that contains multiple statements
57 | sqlstr = "CREATE TABLE hello (a int, b char);";
58 | sqlstr += "INSERT INTO hello VALUES (0, 'hello');"
59 | sqlstr += "INSERT INTO hello VALUES (1, 'world');"
60 | db.run(sqlstr); // Run the query without returning anything
61 |
62 | var res = db.exec("SELECT * FROM hello");
63 | /*
64 | [
65 | {columns:['a','b'], values:[[0,'hello'],[1,'world']]}
66 | ]
67 | */
68 |
69 | // You can also use JavaScript functions inside your SQL code
70 | // Create the js function you need
71 | function add(a, b) {return a+b;}
72 | // Specifies the SQL function's name, the number of it's arguments, and the js function to use
73 | db.create_function("add_js", add);
74 | // Run a query in which the function is used
75 | db.run("INSERT INTO hello VALUES (add_js(7, 3), add_js('Hello ', 'world'));"); // Inserts 10 and 'Hello world'
76 |
77 | // Export the database to an Uint8Array containing the SQLite database file
78 | var binaryArray = db.export();
79 | ```
80 |
81 | ## Demo
82 | There are a few examples [available here](https://sql-js.github.io/sql.js/index.html). The most full-featured is the [Sqlite Interpreter](https://sql-js.github.io/sql.js/examples/GUI/index.html).
83 |
84 | ## Examples
85 | The test files provide up to date example of the use of the api.
86 | ### Inside the browser
87 | #### Example **HTML** file:
88 | ```html
89 | <meta charset="utf8" />
90 | <html>
91 | <script src='/dist/sql-wasm.js'></script>
92 | <script>
93 | config = {
94 | locateFile: filename => `/dist/${filename}`
95 | }
96 | // The `initSqlJs` function is globally provided by all of the main dist files if loaded in the browser.
97 | // We must specify this locateFile function if we are loading a wasm file from anywhere other than the current html page's folder.
98 | initSqlJs(config).then(function(SQL){
99 | //Create the database
100 | var db = new SQL.Database();
101 | // Run a query without reading the results
102 | db.run("CREATE TABLE test (col1, col2);");
103 | // Insert two rows: (1,111) and (2,222)
104 | db.run("INSERT INTO test VALUES (?,?), (?,?)", [1,111,2,222]);
105 |
106 | // Prepare a statement
107 | var stmt = db.prepare("SELECT * FROM test WHERE col1 BETWEEN $start AND $end");
108 | stmt.getAsObject({$start:1, $end:1}); // {col1:1, col2:111}
109 |
110 | // Bind new values
111 | stmt.bind({$start:1, $end:2});
112 | while(stmt.step()) { //
113 | var row = stmt.getAsObject();
114 | console.log('Here is a row: ' + JSON.stringify(row));
115 | }
116 | });
117 | </script>
118 | <body>
119 | Output is in Javascript console
120 | </body>
121 | </html>
122 | ```
123 |
124 | #### Creating a database from a file chosen by the user
125 | `SQL.Database` constructor takes an array of integer representing a database file as an optional parameter.
126 | The following code uses an HTML input as the source for loading a database:
127 | ```javascript
128 | dbFileElm.onchange = () => {
129 | var f = dbFileElm.files[0];
130 | var r = new FileReader();
131 | r.onload = function() {
132 | var Uints = new Uint8Array(r.result);
133 | db = new SQL.Database(Uints);
134 | }
135 | r.readAsArrayBuffer(f);
136 | }
137 | ```
138 | See : https://sql-js.github.io/sql.js/examples/GUI/gui.js
139 |
140 | #### Loading a database from a server
141 |
142 | ##### using fetch
143 |
144 | ```javascript
145 | const sqlPromise = initSqlJs({
146 | locateFile: file => `https://path/to/your/dist/folder/dist/${file}`
147 | });
148 | const dataPromise = fetch("/path/to/databse.sqlite").then(res => res.arrayBuffer());
149 | const [SQL, buf] = await Promise.all([sqlPromise, dataPromise])
150 | const db = new SQL.Database(new Uint8Array(buf));
151 | ```
152 |
153 | ##### using XMLHttpRequest
154 |
155 | ```javascript
156 | var xhr = new XMLHttpRequest();
157 | // For example: https://github.com/lerocha/chinook-database/raw/master/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite
158 | xhr.open('GET', '/path/to/database.sqlite', true);
159 | xhr.responseType = 'arraybuffer';
160 |
161 | xhr.onload = e => {
162 | var uInt8Array = new Uint8Array(xhr.response);
163 | var db = new SQL.Database(uInt8Array);
164 | var contents = db.exec("SELECT * FROM my_table");
165 | // contents is now [{columns:['col1','col2',...], values:[[first row], [second row], ...]}]
166 | };
167 | xhr.send();
168 | ```
169 | See: https://github.com/sql-js/sql.js/wiki/Load-a-database-from-the-server
170 |
171 |
172 | ### Use from node.js
173 |
174 | `sql.js` is [hosted on npm](https://www.npmjs.org/package/sql.js). To install it, you can simply run `npm install sql.js`.
175 | Alternatively, you can simply download `sql-wasm.js` and `sql-wasm.wasm`, from the download link below.
176 |
177 | #### read a database from the disk:
178 | ```javascript
179 | var fs = require('fs');
180 | var initSqlJs = require('sql-wasm.js');
181 | var filebuffer = fs.readFileSync('test.sqlite');
182 |
183 | initSqlJs().then(function(SQL){
184 | // Load the db
185 | var db = new SQL.Database(filebuffer);
186 | });
187 |
188 | ```
189 |
190 | #### write a database to the disk
191 | You need to convert the result of `db.export` to a buffer
192 | ```javascript
193 | var fs = require("fs");
194 | // [...] (create the database)
195 | var data = db.export();
196 | var buffer = new Buffer(data);
197 | fs.writeFileSync("filename.sqlite", buffer);
198 | ```
199 |
200 | See : https://github.com/sql-js/sql.js/blob/master/test/test_node_file.js
201 |
202 | ### Use as web worker
203 | If you don't want to run CPU-intensive SQL queries in your main application thread,
204 | you can use the *more limited* WebWorker API.
205 |
206 | You will need to download [dist/worker.sql-wasm.js](dist/worker.sql-wasm.js) [dist/worker.sql-wasm.wasm](dist/worker.sql-wasm.wasm).
207 |
208 | Example:
209 | ```html
210 | <script>
211 | var worker = new Worker("/dist/worker.sql-wasm.js");
212 | worker.onmessage = () => {
213 | console.log("Database opened");
214 | worker.onmessage = event => {
215 | console.log(event.data); // The result of the query
216 | };
217 |
218 | worker.postMessage({
219 | id: 2,
220 | action: "exec",
221 | sql: "SELECT age,name FROM test WHERE id=$id",
222 | params: { "$id": 1 }
223 | });
224 | };
225 |
226 | worker.onerror = e => console.log("Worker error: ", e);
227 | worker.postMessage({
228 | id:1,
229 | action:"open",
230 | buffer:buf, /*Optional. An ArrayBuffer representing an SQLite Database file*/
231 | });
232 | </script>
233 | ```
234 |
235 | See [examples/GUI/gui.js](examples/GUI/gui.js) for a full working example.
236 |
237 | ## Flavors/versions Targets/Downloads
238 |
239 | This library includes both WebAssembly and asm.js versions of Sqlite. (WebAssembly is the newer, preferred way to compile to JavaScript, and has superceded asm.js. It produces smaller, faster code.) Asm.js versions are included for compatibility.
240 |
241 | ## Upgrading from 0.x to 1.x
242 |
243 | Version 1.0 of sql.js must be loaded asynchronously, whereas asm.js was able to be loaded synchronously.
244 |
245 | So in the past, you would:
246 | ```html
247 | <script src='js/sql.js'></script>
248 | <script>
249 | var db = new SQL.Database();
250 | //...
251 | </script>
252 | ```
253 | or:
254 | ```javascript
255 | var SQL = require('sql.js');
256 | var db = new SQL.Database();
257 | //...
258 | ```
259 |
260 | Version 1.x:
261 | ```html
262 | <script src='dist/sql-wasm.js'></script>
263 | <script>
264 | initSqlJs({ locateFile: filename => `/dist/${filename}` }).then(function(SQL){
265 | var db = new SQL.Database();
266 | //...
267 | });
268 | </script>
269 | ```
270 | or:
271 | ```javascript
272 | var initSqlJs = require('sql-wasm.js');
273 | initSqlJs().then(function(SQL){
274 | var db = new SQL.Database();
275 | //...
276 | });
277 | ```
278 |
279 | `NOTHING` is now a reserved word in SQLite, whereas previously it was not. This could cause errors like `Error: near "nothing": syntax error`
280 |
281 | ### Downloading/Using: ###
282 | Although asm.js files were distributed as a single Javascript file, WebAssembly libraries are most efficiently distributed as a pair of files, the `.js` loader and the `.wasm` file, like `sql-wasm.js` and `sql-wasm.wasm`. The `.js` file is responsible for loading the `.wasm` file. You can find these files on our [release page](https://github.com/sql-js/sql.js/releases)
283 |
284 |
285 |
286 |
287 | ## Versions of sql.js included in the distributed artifacts
288 | You can always find the latest published artifacts on https://github.com/sql-js/sql.js/releases/latest.
289 |
290 | For each [release](https://github.com/sql-js/sql.js/releases/), you will find a file called `sqljs.zip` in the *release assets*. It will contain:
291 | - `sql-wasm.js` : The Web Assembly version of Sql.js. Minified and suitable for production. Use this. If you use this, you will need to include/ship `sql-wasm.wasm` as well.
292 | - `sql-wasm-debug.js` : The Web Assembly, Debug version of Sql.js. Larger, with assertions turned on. Useful for local development. You will need to include/ship `sql-wasm-debug.wasm` if you use this.
293 | - `sql-asm.js` : The older asm.js version of Sql.js. Slower and larger. Provided for compatibility reasons.
294 | - `sql-asm-memory-growth.js` : Asm.js doesn't allow for memory to grow by default, because it is slower and de-optimizes. If you are using sql-asm.js and you see this error (`Cannot enlarge memory arrays`), use this file.
295 | - `sql-asm-debug.js` : The _Debug_ asm.js version of Sql.js. Use this for local development.
296 | - `worker.*` - Web Worker versions of the above libraries. More limited API. See [examples/GUI/gui.js](examples/GUI/gui.js) for a good example of this.
297 |
298 | ## Compiling
299 |
300 | - Install the EMSDK, [as described here](https://emscripten.org/docs/getting_started/downloads.html)
301 | - Run `npm run rebuild`
302 |
303 | In order to enable extensions like FTS5, change the CFLAGS in the [Makefile](Makefile) and rebuild:
304 |
305 | ``` diff
306 | CFLAGS = \
307 | -O2 \
308 | -DSQLITE_OMIT_LOAD_EXTENSION \
309 | -DSQLITE_DISABLE_LFS \
310 | -DSQLITE_ENABLE_FTS3 \
311 | -DSQLITE_ENABLE_FTS3_PARENTHESIS \
312 | + -DSQLITE_ENABLE_FTS5 \
313 | -DSQLITE_ENABLE_JSON1 \
314 | -DSQLITE_THREADSAFE=0
315 | ```
316 |
--------------------------------------------------------------------------------
/sql.js/dist/.gitignore:
--------------------------------------------------------------------------------
1 | # Ignore everything in this directory
2 | *
3 | # Except this file
4 | !.gitignore
5 | !.npmignore
6 |
--------------------------------------------------------------------------------
/sql.js/dist/.npmignore:
--------------------------------------------------------------------------------
1 | .gitignore
--------------------------------------------------------------------------------
/sql.js/documentation_index.md:
--------------------------------------------------------------------------------
1 | # sql.js API documentation
2 |
3 | ## Introduction
4 |
5 | If you need a quick intoduction with code samples that you can copy-and-paste,
6 | head over to [sql.js.org](https://sql.js.org/)
7 |
8 | ## API
9 |
10 | ### The initSqlJs function
11 |
12 | The root object in the API is the [`initSqlJs`](./global.html#initSqlJs) function,
13 | that takes an [`SqlJsConfig`](./global.html#SqlJsConfig) parameter,
14 | and returns an [SqlJs](./global.html#SqlJs) object
15 |
16 | ### The SqlJs object
17 |
18 | `initSqlJs` returns the main sql.js object, the [**`SqlJs`**](./module-SqlJs.html) module, which contains :
19 |
20 | #### Database
21 |
22 | [**Database**](./Database.html) is the main class, that represents an SQLite database.
23 |
24 | #### Statement
25 |
26 | The [**Statement**](./Statement.html) class is used for prepared statements.
27 |
--------------------------------------------------------------------------------
/sql.js/examples/GUI/demo.css:
--------------------------------------------------------------------------------
1 | html {
2 | background:#222;
3 | margin:auto;
4 | width:80%;
5 | }
6 |
7 | body{
8 | background: linear-gradient(#aaa 0, #ddd 10px, #fff 55px);
9 | border: 1px solid black;
10 | padding: 10px 20px;
11 | box-shadow: 5px 0px 30px #000;
12 | border-radius: 8px;
13 | }
14 |
15 | h1 {
16 | text-align: center;
17 | color: #222;
18 | margin: 0 0 30px;
19 | }
20 |
21 | .button {
22 | color: #333;
23 | background: linear-gradient(#eee, #ddd);
24 | border: 1px solid #222;
25 | border-radius: 3px;
26 | padding: 7px;
27 | margin-right: 5px;
28 | transition: .3s;
29 | font-family: ubuntu, sans-serif;
30 | font-size: 1em;
31 | }
32 |
33 | .button:active {
34 | background: linear-gradient(#ddd, #eee);
35 | }
36 |
37 | .button:hover, button:focus {
38 | box-shadow: 0 0 2px #222;
39 | }
40 |
41 | #execute {
42 | margin-top: 5px;;
43 | width: 10%;
44 | min-width:100px;
45 | }
46 |
47 | .CodeMirror {
48 | border: 1px solid #222;
49 | height: auto;
50 | }
51 | .CodeMirror-scroll {
52 | overflow-y: hidden;
53 | overflow-x: auto;
54 | }
55 |
56 | .error {
57 | color:red;
58 | transition:.5s;
59 | overflow:hidden;
60 | margin: 15px;
61 | }
62 |
63 | #output {
64 | overflow: auto;
65 | }
66 |
67 | table {
68 | width:auto;
69 | margin:auto;
70 | border:1px solid black;
71 | border-collapse:collapse;
72 | margin-bottom:10px;
73 | }
74 |
75 | th, td {
76 | border:1px solid #777;
77 | }
78 |
79 | footer {
80 | font-size:.8em;
81 | color: #222;
82 | }
83 |
--------------------------------------------------------------------------------
/sql.js/examples/GUI/gui.js:
--------------------------------------------------------------------------------
1 | var execBtn = document.getElementById("execute");
2 | var outputElm = document.getElementById('output');
3 | var errorElm = document.getElementById('error');
4 | var commandsElm = document.getElementById('commands');
5 | var dbFileElm = document.getElementById('dbfile');
6 | var savedbElm = document.getElementById('savedb');
7 |
8 | // Start the worker in which sql.js will run
9 | var worker = new Worker("../../dist/worker.sql-wasm.js");
10 | worker.onerror = error;
11 |
12 | // Open a database
13 | worker.postMessage({ action: 'open' });
14 |
15 | // Connect to the HTML element we 'print' to
16 | function print(text) {
17 | outputElm.innerHTML = text.replace(/\n/g, '<br>');
18 | }
19 | function error(e) {
20 | console.log(e);
21 | errorElm.style.height = '2em';
22 | errorElm.textContent = e.message;
23 | }
24 |
25 | function noerror() {
26 | errorElm.style.height = '0';
27 | }
28 |
29 | // Run a command in the database
30 | function execute(commands) {
31 | tic();
32 | worker.onmessage = function (event) {
33 | var results = event.data.results;
34 | toc("Executing SQL");
35 | if (!results) {
36 | error({message: event.data.error});
37 | return;
38 | }
39 |
40 | tic();
41 | outputElm.innerHTML = "";
42 | for (var i = 0; i < results.length; i++) {
43 | outputElm.appendChild(tableCreate(results[i].columns, results[i].values));
44 | }
45 | toc("Displaying results");
46 | }
47 | worker.postMessage({ action: 'exec', sql: commands });
48 | outputElm.textContent = "Fetching results...";
49 | }
50 |
51 | // Create an HTML table
52 | var tableCreate = function () {
53 | function valconcat(vals, tagName) {
54 | if (vals.length === 0) return '';
55 | var open = '<' + tagName + '>', close = '</' + tagName + '>';
56 | return open + vals.join(close + open) + close;
57 | }
58 | return function (columns, values) {
59 | var tbl = document.createElement('table');
60 | var html = '<thead>' + valconcat(columns, 'th') + '</thead>';
61 | var rows = values.map(function (v) { return valconcat(v, 'td'); });
62 | html += '<tbody>' + valconcat(rows, 'tr') + '</tbody>';
63 | tbl.innerHTML = html;
64 | return tbl;
65 | }
66 | }();
67 |
68 | // Execute the commands when the button is clicked
69 | function execEditorContents() {
70 | noerror()
71 | execute(editor.getValue() + ';');
72 | }
73 | execBtn.addEventListener("click", execEditorContents, true);
74 |
75 | // Performance measurement functions
76 | var tictime;
77 | if (!window.performance || !performance.now) { window.performance = { now: Date.now } }
78 | function tic() { tictime = performance.now() }
79 | function toc(msg) {
80 | var dt = performance.now() - tictime;
81 | console.log((msg || 'toc') + ": " + dt + "ms");
82 | }
83 |
84 | // Add syntax highlihjting to the textarea
85 | var editor = CodeMirror.fromTextArea(commandsElm, {
86 | mode: 'text/x-mysql',
87 | viewportMargin: Infinity,
88 | indentWithTabs: true,
89 | smartIndent: true,
90 | lineNumbers: true,
91 | matchBrackets: true,
92 | autofocus: true,
93 | extraKeys: {
94 | "Ctrl-Enter": execEditorContents,
95 | "Ctrl-S": savedb,
96 | }
97 | });
98 |
99 | // Load a db from a file
100 | dbFileElm.onchange = function () {
101 | var f = dbFileElm.files[0];
102 | var r = new FileReader();
103 | r.onload = function () {
104 | worker.onmessage = function () {
105 | toc("Loading database from file");
106 | // Show the schema of the loaded database
107 | editor.setValue("SELECT `name`, `sql`\n FROM `sqlite_master`\n WHERE type='table';");
108 | execEditorContents();
109 | };
110 | tic();
111 | try {
112 | worker.postMessage({ action: 'open', buffer: r.result }, [r.result]);
113 | }
114 | catch (exception) {
115 | worker.postMessage({ action: 'open', buffer: r.result });
116 | }
117 | }
118 | r.readAsArrayBuffer(f);
119 | }
120 |
121 | // Save the db to a file
122 | function savedb() {
123 | worker.onmessage = function (event) {
124 | toc("Exporting the database");
125 | var arraybuff = event.data.buffer;
126 | var blob = new Blob([arraybuff]);
127 | var a = document.createElement("a");
128 | document.body.appendChild(a);
129 | a.href = window.URL.createObjectURL(blob);
130 | a.download = "sql.db";
131 | a.onclick = function () {
132 | setTimeout(function () {
133 | window.URL.revokeObjectURL(a.href);
134 | }, 1500);
135 | };
136 | a.click();
137 | };
138 | tic();
139 | worker.postMessage({ action: 'export' });
140 | }
141 | savedbElm.addEventListener("click", savedb, true);
142 |
--------------------------------------------------------------------------------
/sql.js/examples/GUI/index.html:
--------------------------------------------------------------------------------
1 | <!doctype html>
2 | <html>
3 |
4 | <head>
5 | <meta charset="utf8">
6 | <title>sql.js demo: Online SQL interpreter</title>
7 | <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.58.1/codemirror.css">
8 | <link rel="stylesheet" href="demo.css">
9 | <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.58.1/codemirror.js"></script>
10 |
11 | </head>
12 |
13 | <body>
14 | <!-- Github ribbon -->
15 | <a href="https://github.com/sql-js/sql.js"><img style="position: absolute; top: 0; left: 0; border: 0;"
16 | src="https://camo.githubusercontent.com/82b228a3648bf44fc1163ef44c62fcc60081495e/68747470733a2f2f73332e616d617a6f6e6177732e636f6d2f6769746875622f726962626f6e732f666f726b6d655f6c6566745f7265645f6161303030302e706e67"
17 | alt="Fork me on GitHub"
18 | data-canonical-src="https://s3.amazonaws.com/github/ribbons/forkme_left_red_aa0000.png"></a>
19 |
20 | <h1>Online SQL interpreter</h1>
21 |
22 | <main>
23 | <label for='commands'>Enter some SQL</label>
24 | <br>
25 |
26 | <textarea id="commands">DROP TABLE IF EXISTS employees;
27 | CREATE TABLE employees( id integer, name text,
28 | designation text, manager integer,
29 | hired_on date, salary integer,
30 | commission float, dept integer);
31 |
32 | INSERT INTO employees VALUES (1,'JOHNSON','ADMIN',6,'1990-12-17',18000,NULL,4);
33 | INSERT INTO employees VALUES (2,'HARDING','MANAGER',9,'1998-02-02',52000,300,3);
34 | INSERT INTO employees VALUES (3,'TAFT','SALES I',2,'1996-01-02',25000,500,3);
35 | INSERT INTO employees VALUES (4,'HOOVER','SALES I',2,'1990-04-02',27000,NULL,3);
36 | INSERT INTO employees VALUES (5,'LINCOLN','TECH',6,'1994-06-23',22500,1400,4);
37 | INSERT INTO employees VALUES (6,'GARFIELD','MANAGER',9,'1993-05-01',54000,NULL,4);
38 | INSERT INTO employees VALUES (7,'POLK','TECH',6,'1997-09-22',25000,NULL,4);
39 | INSERT INTO employees VALUES (8,'GRANT','ENGINEER',10,'1997-03-30',32000,NULL,2);
40 | INSERT INTO employees VALUES (9,'JACKSON','CEO',NULL,'1990-01-01',75000,NULL,4);
41 | INSERT INTO employees VALUES (10,'FILLMORE','MANAGER',9,'1994-08-09',56000,NULL,2);
42 | INSERT INTO employees VALUES (11,'ADAMS','ENGINEER',10,'1996-03-15',34000,NULL,2);
43 | INSERT INTO employees VALUES (12,'WASHINGTON','ADMIN',6,'1998-04-16',18000,NULL,4);
44 | INSERT INTO employees VALUES (13,'MONROE','ENGINEER',10,'2000-12-03',30000,NULL,2);
45 | INSERT INTO employees VALUES (14,'ROOSEVELT','CPA',9,'1995-10-12',35000,NULL,1);
46 |
47 | SELECT designation,COUNT(*) AS nbr, (AVG(salary)) AS avg_salary FROM employees GROUP BY designation ORDER BY avg_salary DESC;
48 | SELECT name,hired_on FROM employees ORDER BY hired_on;</textarea>
49 |
50 | <button id="execute" class="button">Execute</button>
51 | <button id='savedb' class="button">Save the db</button>
52 | <label class="button">Load an SQLite database file: <input type='file' id='dbfile'></label>
53 |
54 | <div id="error" class="error"></div>
55 |
56 | <pre id="output">Results will be displayed here</pre>
57 | </main>
58 |
59 | <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.58.1/mode/sql/sql.min.js"></script>
60 |
61 | <footer>
62 | Original work by kripken (<a href='https://github.com/sql-js/sql.js'>sql.js</a>).
63 | C to Javascript compiler by kripken (<a href='https://github.com/kripken/emscripten'>emscripten</a>).
64 | Project now maintained by <a href='https://github.com/lovasoa'>lovasoa</a>
65 | </footer>
66 |
67 | <script type="text/javascript" src="gui.js"></script>
68 | </body>
69 |
70 | </html>
--------------------------------------------------------------------------------
/sql.js/examples/README.md:
--------------------------------------------------------------------------------
1 |
2 | To show these examples locally, first run:
3 | ./start_local_server.py
4 |
5 | Then, open http://localhost:8081/index.html in a local browser.
6 |
7 |
--------------------------------------------------------------------------------
/sql.js/examples/persistent.html:
--------------------------------------------------------------------------------
1 | <!doctype html>
2 | <html>
3 |
4 | <head>
5 | <meta charset="utf8">
6 | <title>Persistent sqlite</title>
7 | <script src="../dist/sql-wasm.js"></script>
8 | </head>
9 |
10 | <body>
11 | <p>You have seen this page <span id="views">0</span> times.</p>
12 | <div>
13 | You have been here on the following dates: <ol id="dates"></ol>
14 | </div>
15 | <script>
16 | var baseUrl = '../dist/';
17 |
18 | function toBinArray(str) {
19 | var l = str.length,
20 | arr = new Uint8Array(l);
21 | for (var i = 0; i < l; i++) arr[i] = str.charCodeAt(i);
22 | return arr;
23 | }
24 |
25 | function toBinString(arr) {
26 | var uarr = new Uint8Array(arr);
27 | var strings = [], chunksize = 0xffff;
28 | // There is a maximum stack size. We cannot call String.fromCharCode with as many arguments as we want
29 | for (var i = 0; i * chunksize < uarr.length; i++) {
30 | strings.push(String.fromCharCode.apply(null, uarr.subarray(i * chunksize, (i + 1) * chunksize)));
31 | }
32 | return strings.join('');
33 | }
34 |
35 | // Normally Sql.js tries to load sql-wasm.wasm relative to the page, not relative to the javascript
36 | // doing the loading. So, we help it find the .wasm file with this function.
37 | var config = {
38 | locateFile: filename => `${baseUrl}/${filename}`
39 | }
40 | initSqlJs(config).then(function (SQL) {
41 | var dbstr = window.localStorage.getItem("viewcount.sqlite");
42 | if (dbstr) {
43 | var db = new SQL.Database(toBinArray(dbstr));
44 | } else {
45 | var db = new SQL.Database();
46 | db.run("CREATE TABLE views (date INTEGER PRIMARY KEY)");
47 | }
48 | db.run("INSERT INTO views(date) VALUES (?)", [Date.now()]);
49 |
50 | document.getElementById('views').textContent = db.exec("SELECT COUNT(*) FROM views")[0].values[0][0];
51 |
52 | var count = 0,
53 | dates = document.getElementById("dates");
54 |
55 | db.each("SELECT date FROM views ORDER BY date ASC",
56 | function callback(row) {
57 | var li = document.createElement("li");
58 | li.textContent = new Date(row.date);
59 | dates.appendChild(li);
60 | }, function done() {
61 | var dbstr = toBinString(db.export());
62 | window.localStorage.setItem("viewcount.sqlite", dbstr);
63 | }
64 | );
65 | });
66 |
67 | </script>
68 | </body>
69 |
70 | </html>
--------------------------------------------------------------------------------
/sql.js/examples/repl.html:
--------------------------------------------------------------------------------
1 | <!doctype html>
2 | <html>
3 | <!--Simple Read eval print loop for SQL-->
4 |
5 | <head>
6 | <meta charset="utf8">
7 | <title>SQL REPL</title>
8 | <script src="../dist/sql-wasm.js"></script>
9 | </head>
10 |
11 | <body>
12 | <input type='text' id='input' placeholder="ENTER SOME SQL" size='50'
13 | value="CREATE TABLE test(val);INSERT INTO test VALUES (666); SELECT * FROM test">
14 | <button id='submit'>Execute</button>
15 | <pre id='result'></pre>
16 | <pre id='error'></pre>
17 | <script>
18 |
19 | //Open a blank database
20 | var db;
21 | initSqlJs({ locateFile: filename => `../dist/${filename}` }).then(function (SQL) {
22 | db = new SQL.Database();
23 | });
24 |
25 | document.getElementById('submit').onclick = function () {
26 | var sql = document.getElementById('input').value;
27 | var result = '', error = '';
28 | try { result = db.exec(sql); }
29 | catch (e) { error = e; }
30 | document.getElementById('result').innerHTML = JSON.stringify(result, null, ' ');
31 | document.getElementById('error').innerHTML = error;
32 | };
33 | </script>
34 | </body>
35 |
--------------------------------------------------------------------------------
/sql.js/examples/requireJS.html:
--------------------------------------------------------------------------------
1 | <meta charset="utf8" />
2 | <html>
3 | <script src='https://requirejs.org/docs/release/2.3.6/minified/require.js'></script>
4 |
5 | <script>
6 | var baseUrl = '../dist'
7 | require.config({
8 | baseUrl: baseUrl
9 | });
10 |
11 | // Options: 'sql-wasm', 'sql-asm', 'sql-asm-memory-growth.js', 'sql-wasm-debug', 'sql-asm-debug'
12 | require(['sql-wasm'],
13 | function success(initSqlJs) {
14 | console.log(typeof initSqlJs);
15 | if (typeof initSqlJs !== 'function') {
16 | document.body.style.backgroundColor = 'red';
17 | console.log('initSqlJs returned: ', initSqlJs);
18 | alert("Failed to require sql.js through AMD");
19 | return;
20 | }
21 | // The `initSqlJs` function is globally provided by all of the main dist files if loaded in the browser.
22 | // We must specify this locateFile function if we are loading a wasm file from anywhere other than the current html page's folder.
23 |
24 | var config = {
25 | locateFile: filename => `${baseUrl}/${filename}`
26 | }
27 | initSqlJs(config).then(function (SQL) {
28 | //Create the database
29 | var db = new SQL.Database();
30 | // Run a query without reading the results
31 | db.run("CREATE TABLE test (col1, col2);");
32 | // Insert two rows: (1,111) and (2,222)
33 | db.run("INSERT INTO test VALUES (?,?), (?,?)", [1, 111, 2, 222]);
34 |
35 | // Prepare a statement
36 | var stmt = db.prepare("SELECT * FROM test WHERE col1 BETWEEN $start AND $end");
37 | stmt.getAsObject({ $start: 1, $end: 1 }); // {col1:1, col2:111}
38 |
39 | // Bind new values
40 | stmt.bind({ $start: 1, $end: 2 });
41 | while (stmt.step()) { //
42 | var row = stmt.getAsObject();
43 | // [...] do something with the row of result
44 | console.log('Here is a row: ', row);
45 | }
46 | });
47 | },
48 | function error(err) {
49 | document.body.style.backgroundColor = 'red';
50 | console.log(err);
51 | alert("Module load failed: " + err);
52 | }
53 | );
54 | </script>
55 |
56 | <body>
57 | Output is in Javscript console
58 | </body>
59 |
60 | </html>
61 |
--------------------------------------------------------------------------------
/sql.js/examples/simple.html:
--------------------------------------------------------------------------------
1 | <meta charset="utf8" />
2 | <html>
3 | <script src='../dist/sql-wasm-debug.js'></script>
4 | <script>
5 | config = {
6 | locateFile: (filename, prefix) => {
7 | console.log(`prefix is : ${prefix}`);
8 | return `../dist/${filename}`;
9 | }
10 | }
11 | // The `initSqlJs` function is globally provided by all of the main dist files if loaded in the browser.
12 | // We must specify this locateFile function if we are loading a wasm file from anywhere other than the current html page's folder.
13 | initSqlJs(config).then(function (SQL) {
14 | //Create the database
15 | var db = new SQL.Database();
16 | // Run a query without reading the results
17 | db.run("CREATE TABLE test (col1, col2);");
18 | // Insert two rows: (1,111) and (2,222)
19 | db.run("INSERT INTO test VALUES (?,?), (?,?)", [1, 111, 2, 222]);
20 |
21 | // Prepare a statement
22 | var stmt = db.prepare("SELECT * FROM test WHERE col1 BETWEEN $start AND $end");
23 | stmt.getAsObject({ $start: 1, $end: 1 }); // {col1:1, col2:111}
24 |
25 | // Bind new values
26 | stmt.bind({ $start: 1, $end: 2 });
27 | while (stmt.step()) { //
28 | var row = stmt.getAsObject();
29 | console.log('Here is a row: ' + JSON.stringify(row));
30 | }
31 | });
32 | </script>
33 |
34 | <body>
35 | Output is in Javscript console
36 | </body>
37 |
38 | </html>
--------------------------------------------------------------------------------
/sql.js/examples/start_local_server.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 |
3 | import http.server
4 | import os
5 |
6 | # We need to host from the root because we are going to be requesting files inside of dist
7 | os.chdir("../")
8 | port = 8081
9 | print("Running on port %d" % port)
10 |
11 | http.server.SimpleHTTPRequestHandler.extensions_map[".wasm"] = "application/wasm"
12 |
13 | httpd = http.server.HTTPServer(
14 | ("localhost", port), http.server.SimpleHTTPRequestHandler
15 | )
16 |
17 | print(
18 | 'Mapping ".wasm" to "%s"'
19 | % http.server.SimpleHTTPRequestHandler.extensions_map[".wasm"]
20 | )
21 | httpd.serve_forever()
22 |
--------------------------------------------------------------------------------
/sql.js/index.html:
--------------------------------------------------------------------------------
1 | <!DOCTYPE html>
2 | <html lang="en">
3 |
4 | <head>
5 | <meta charset="UTF-8">
6 | <title>sql.js</title>
7 | <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
8 | <meta name="description"
9 | content="sql.js is an SQL library for javascript containing a version of SQLite compiled for the web.">
10 | <meta name="viewport"
11 | content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
12 | <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/docsify/lib/themes/vue.css">
13 | </head>
14 |
15 | <body>
16 | <div id="app"></div>
17 | <script>window.$docsify = { name: 'SQL.js', repo: 'https://github.com/sql-js/sql.js/' }</script>
18 | <script src="//cdn.jsdelivr.net/npm/docsify/lib/docsify.min.js"></script>
19 | <script>
20 | let url = new URL(window.location);
21 | url.pathname = url.pathname.replace(/\/index.html$/, '/');
22 | if (window.location.toString() !== url.toString()) window.location = url;
23 | </script>
24 | </body>
25 |
26 | </html>
--------------------------------------------------------------------------------
/sql.js/logo.svg:
--------------------------------------------------------------------------------
1 | <svg xmlns="http://www.w3.org/2000/svg" height="400" width="400">
2 | <defs/>
3 | <defs>
4 | <path id="a" d="M6 10h139v93H6z"/>
5 | </defs>
6 | <rect ry="0" y="0" x="0" height="400" width="400" fill="#ffe70b" stroke="#000"/>
7 | <text y="170" x="29" style="line-height:1.25;-inkscape-font-specification:'Source Sans Pro Semi-Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-east-asian:normal" font-weight="600" font-size="200" font-family="Source Sans Pro">
8 | <tspan y="170" x="29">SQL</tspan>
9 | </text>
10 | <text style="line-height:1.25;-inkscape-font-specification:'Source Sans Pro Semi-Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-east-asian:normal" x="109" y="359" font-weight="600" font-size="200" font-family="Source Sans Pro">
11 | <tspan x="109" y="359">.JS</tspan>
12 | </text>
13 | </svg>
14 |
--------------------------------------------------------------------------------
/sql.js/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "sql.js",
3 | "version": "1.5.0",
4 | "description": "SQLite library with support for opening and writing databases, prepared statements, and more. This SQLite library is in pure javascript (compiled with emscripten).",
5 | "keywords": [
6 | "sql",
7 | "sqlite",
8 | "stand-alone",
9 | "relational",
10 | "database",
11 | "RDBMS",
12 | "data",
13 | "query",
14 | "statement",
15 | "emscripten",
16 | "asm",
17 | "asm.js"
18 | ],
19 | "license": "MIT",
20 | "main": "./dist/sql-wasm.js",
21 | "scripts": {
22 | "build": "make",
23 | "rebuild": "make clean && make",
24 | "test": "npm run lint && npm run test-asm && npm run test-asm-debug && npm run test-wasm && npm run test-wasm-debug && npm run test-asm-memory-growth",
25 | "lint": "eslint .",
26 | "prettify": "eslint . --fix",
27 | "test-asm": "node test/all.js asm",
28 | "test-asm-debug": "node test/all.js asm-debug",
29 | "test-asm-memory-growth": "node test/all.js asm-memory-growth",
30 | "test-wasm": "node test/all.js wasm",
31 | "test-wasm-debug": "node test/all.js wasm-debug",
32 | "doc": "jsdoc -c .jsdoc.config.json"
33 | },
34 | "homepage": "http://github.com/sql-js/sql.js",
35 | "repository": {
36 | "type": "git",
37 | "url": "http://github.com/sql-js/sql.js.git"
38 | },
39 | "bugs": {
40 | "url": "https://github.com/sql-js/sql.js/issues"
41 | },
42 | "devDependencies": {
43 | "clean-jsdoc-theme": "^2.2.15",
44 | "eslint": "^6.8.0",
45 | "eslint-config-airbnb-base": "^14.2.1",
46 | "eslint-plugin-import": "^2.22.1",
47 | "jsdoc": "^3.6.6",
48 | "puppeteer": "^2.1.1",
49 | "test": ">=0.6"
50 | },
51 | "dependencies": {}
52 | }
53 |
--------------------------------------------------------------------------------
/sql.js/src/exported_functions.json:
--------------------------------------------------------------------------------
1 | [
2 | "_malloc",
3 | "_free",
4 | "_sqlite3_open",
5 | "_sqlite3_exec",
6 | "_sqlite3_free",
7 | "_sqlite3_errmsg",
8 | "_sqlite3_changes",
9 | "_sqlite3_prepare_v2",
10 | "_sqlite3_sql",
11 | "_sqlite3_normalized_sql",
12 | "_sqlite3_bind_text",
13 | "_sqlite3_bind_blob",
14 | "_sqlite3_bind_double",
15 | "_sqlite3_bind_int",
16 | "_sqlite3_bind_parameter_index",
17 | "_sqlite3_step",
18 | "_sqlite3_column_count",
19 | "_sqlite3_data_count",
20 | "_sqlite3_column_double",
21 | "_sqlite3_column_text",
22 | "_sqlite3_column_blob",
23 | "_sqlite3_column_bytes",
24 | "_sqlite3_column_type",
25 | "_sqlite3_column_name",
26 | "_sqlite3_reset",
27 | "_sqlite3_clear_bindings",
28 | "_sqlite3_finalize",
29 | "_sqlite3_close_v2",
30 | "_sqlite3_create_function_v2",
31 | "_sqlite3_create_module_v2",
32 | "_sqlite3_value_bytes",
33 | "_sqlite3_value_type",
34 | "_sqlite3_value_text",
35 | "_sqlite3_value_int",
36 | "_sqlite3_value_blob",
37 | "_sqlite3_value_double",
38 | "_sqlite3_result_double",
39 | "_sqlite3_result_null",
40 | "_sqlite3_result_text",
41 | "_sqlite3_result_blob",
42 | "_sqlite3_result_int",
43 | "_sqlite3_result_int64",
44 | "_sqlite3_result_error",
45 | "_sqlite3_declare_vtab",
46 | "_sqlite3_malloc",
47 | "_RegisterExtensionFunctions"
48 | ]
49 |
--------------------------------------------------------------------------------
/sql.js/src/exported_runtime_methods.json:
--------------------------------------------------------------------------------
1 | [
2 | "cwrap",
3 | "stackAlloc",
4 | "stackSave",
5 | "stackRestore",
6 | "UTF8ToString",
7 | "stringToUTF8",
8 | "lengthBytesUTF8",
9 | "ccall",
10 | "setValue",
11 | "getValue",
12 | "addFunction"
13 | ]
14 |
--------------------------------------------------------------------------------
/sql.js/src/shell-post.js:
--------------------------------------------------------------------------------
1 |
2 |
3 | // The shell-pre.js and emcc-generated code goes above
4 | return Module;
5 | }); // The end of the promise being returned
6 |
7 | return initSqlJsPromise;
8 | } // The end of our initSqlJs function
9 |
10 | // This bit below is copied almost exactly from what you get when you use the MODULARIZE=1 flag with emcc
11 | // However, we don't want to use the emcc modularization. See shell-pre.js
12 | if (typeof exports === 'object' && typeof module === 'object'){
13 | module.exports = initSqlJs;
14 | // This will allow the module to be used in ES6 or CommonJS
15 | module.exports.default = initSqlJs;
16 | }
17 | else if (typeof define === 'function' && define['amd']) {
18 | define([], function() { return initSqlJs; });
19 | }
20 | else if (typeof exports === 'object'){
21 | exports["Module"] = initSqlJs;
22 | }
23 |
--------------------------------------------------------------------------------
/sql.js/src/shell-pre.js:
--------------------------------------------------------------------------------
1 |
2 | // We are modularizing this manually because the current modularize setting in Emscripten has some issues:
3 | // https://github.com/kripken/emscripten/issues/5820
4 | // In addition, When you use emcc's modularization, it still expects to export a global object called `Module`,
5 | // which is able to be used/called before the WASM is loaded.
6 | // The modularization below exports a promise that loads and resolves to the actual sql.js module.
7 | // That way, this module can't be used before the WASM is finished loading.
8 |
9 | // We are going to define a function that a user will call to start loading initializing our Sql.js library
10 | // However, that function might be called multiple times, and on subsequent calls, we don't actually want it to instantiate a new instance of the Module
11 | // Instead, we want to return the previously loaded module
12 |
13 | // TODO: Make this not declare a global if used in the browser
14 | var initSqlJsPromise = undefined;
15 |
16 | var initSqlJs = function (moduleConfig) {
17 |
18 | if (initSqlJsPromise){
19 | return initSqlJsPromise;
20 | }
21 | // If we're here, we've never called this function before
22 | initSqlJsPromise = new Promise(function (resolveModule, reject) {
23 |
24 | // We are modularizing this manually because the current modularize setting in Emscripten has some issues:
25 | // https://github.com/kripken/emscripten/issues/5820
26 |
27 | // The way to affect the loading of emcc compiled modules is to create a variable called `Module` and add
28 | // properties to it, like `preRun`, `postRun`, etc
29 | // We are using that to get notified when the WASM has finished loading.
30 | // Only then will we return our promise
31 |
32 | // If they passed in a moduleConfig object, use that
33 | // Otherwise, initialize Module to the empty object
34 | var Module = typeof moduleConfig !== 'undefined' ? moduleConfig : {};
35 |
36 | // EMCC only allows for a single onAbort function (not an array of functions)
37 | // So if the user defined their own onAbort function, we remember it and call it
38 | var originalOnAbortFunction = Module['onAbort'];
39 | Module['onAbort'] = function (errorThatCausedAbort) {
40 | reject(new Error(errorThatCausedAbort));
41 | if (originalOnAbortFunction){
42 | originalOnAbortFunction(errorThatCausedAbort);
43 | }
44 | };
45 |
46 | Module['postRun'] = Module['postRun'] || [];
47 | Module['postRun'].push(function () {
48 | // When Emscripted calls postRun, this promise resolves with the built Module
49 | resolveModule(Module);
50 | });
51 |
52 | // There is a section of code in the emcc-generated code below that looks like this:
53 | // (Note that this is lowercase `module`)
54 | // if (typeof module !== 'undefined') {
55 | // module['exports'] = Module;
56 | // }
57 | // When that runs, it's going to overwrite our own modularization export efforts in shell-post.js!
58 | // The only way to tell emcc not to emit it is to pass the MODULARIZE=1 or MODULARIZE_INSTANCE=1 flags,
59 | // but that carries with it additional unnecessary baggage/bugs we don't want either.
60 | // So, we have three options:
61 | // 1) We undefine `module`
62 | // 2) We remember what `module['exports']` was at the beginning of this function and we restore it later
63 | // 3) We write a script to remove those lines of code as part of the Make process.
64 | //
65 | // Since those are the only lines of code that care about module, we will undefine it. It's the most straightforward
66 | // of the options, and has the side effect of reducing emcc's efforts to modify the module if its output were to change in the future.
67 | // That's a nice side effect since we're handling the modularization efforts ourselves
68 | module = undefined;
69 |
70 | // The emcc-generated code and shell-post.js code goes below,
71 | // meaning that all of it runs inside of this promise. If anything throws an exception, our promise will abort
72 |
--------------------------------------------------------------------------------
/sql.js/src/worker.js:
--------------------------------------------------------------------------------
1 | /* global initSqlJs */
2 | /* eslint-env worker */
3 | /* eslint no-restricted-globals: ["error"] */
4 |
5 | "use strict";
6 |
7 | var db;
8 |
9 | function onModuleReady(SQL) {
10 | function createDb(data) {
11 | if (db != null) db.close();
12 | db = new SQL.Database(data);
13 | return db;
14 | }
15 |
16 | var buff; var data; var result;
17 | data = this["data"];
18 | switch (data && data["action"]) {
19 | case "open":
20 | buff = data["buffer"];
21 | createDb(buff && new Uint8Array(buff));
22 | return postMessage({
23 | id: data["id"],
24 | ready: true
25 | });
26 | case "exec":
27 | if (db === null) {
28 | createDb();
29 | }
30 | if (!data["sql"]) {
31 | throw "exec: Missing query string";
32 | }
33 | return postMessage({
34 | id: data["id"],
35 | results: db.exec(data["sql"], data["params"])
36 | });
37 | case "each":
38 | if (db === null) {
39 | createDb();
40 | }
41 | var callback = function callback(row) {
42 | return postMessage({
43 | id: data["id"],
44 | row: row,
45 | finished: false
46 | });
47 | };
48 | var done = function done() {
49 | return postMessage({
50 | id: data["id"],
51 | finished: true
52 | });
53 | };
54 | return db.each(data["sql"], data["params"], callback, done);
55 | case "export":
56 | buff = db["export"]();
57 | result = {
58 | id: data["id"],
59 | buffer: buff
60 | };
61 | try {
62 | return postMessage(result, [result]);
63 | } catch (error) {
64 | return postMessage(result);
65 | }
66 | case "close":
67 | if (db) {
68 | db.close();
69 | }
70 | return postMessage({
71 | id: data["id"]
72 | });
73 | default:
74 | throw new Error("Invalid action : " + (data && data["action"]));
75 | }
76 | }
77 |
78 | function onError(err) {
79 | return postMessage({
80 | id: this["data"]["id"],
81 | error: err["message"]
82 | });
83 | }
84 |
85 | if (typeof importScripts === "function") {
86 | db = null;
87 | var sqlModuleReady = initSqlJs();
88 | self.onmessage = function onmessage(event) {
89 | return sqlModuleReady
90 | .then(onModuleReady.bind(event))
91 | .catch(onError.bind(event));
92 | };
93 | }
94 |
--------------------------------------------------------------------------------
/sql.js/test/all.js:
--------------------------------------------------------------------------------
1 | var fs = require("fs");
2 | Error.stackTraceLimit = 200;
3 | var sqlLibType = process.argv[2];
4 | const sqlLibLoader = require('./load_sql_lib');
5 |
6 | sqlLibLoader(sqlLibType).then((sql)=>{
7 | var files = fs.readdirSync(__dirname);
8 | for (var i=0; i<files.length; i++) {
9 | var file = files[i];
10 | var m = /^test_(.+)\.js$/.exec(file);
11 | if (m !== null) {
12 | var name = m[1];
13 | var testModule = require("./" + file);
14 | if (testModule.test) {
15 | exports['test ' + name] = testModule.test.bind(null, sql);
16 | }
17 |
18 | }
19 | }
20 |
21 | if (module == require.main) require('test').run(exports);
22 | })
23 | .catch((e)=>{
24 | console.error(e);
25 | });
26 |
--------------------------------------------------------------------------------
/sql.js/test/disabled_test_memory_leak_on_error.js:
--------------------------------------------------------------------------------
1 | // See: https://github.com/sql-js/sql.js/issues/306
2 | exports.test = function(sql, assert) {
3 | var errors = 0, runs=10000;
4 | for (var i=0; i<runs; i++) {
5 | var db = new sql.Database()
6 | try {
7 | db.exec("CREATE TABLE cats (name TEXT NOT NULL, age INTEGER NULL)")
8 | db.exec("INSERT INTO cats (name, age) VALUES (NULL, 3)")
9 | } catch (e) {
10 | errors += e.toString() === "Error: NOT NULL constraint failed: cats.name";
11 | }
12 | db.close()
13 | }
14 | assert.equal(errors, runs, "Multiple constraint violation errors do not trigger an OOM error");
15 | };
16 |
17 | if (module == require.main) {
18 | const target_file = process.argv[2];
19 | const sql_loader = require('./load_sql_lib');
20 | sql_loader(target_file).then((sql)=>{
21 | require('test').run({
22 | 'test memory leak on error': function(assert){
23 | exports.test(sql, assert);
24 | }
25 | });
26 | })
27 | .catch((e)=>{
28 | console.error(e);
29 | assert.fail(e);
30 | });
31 | }
32 |
--------------------------------------------------------------------------------
/sql.js/test/issue55.db:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/phiresky/sql.js-httpvfs/c64536d2acc78feeac17c34bfa1895df01050129/sql.js/test/issue55.db
--------------------------------------------------------------------------------
/sql.js/test/load_sql_lib.js:
--------------------------------------------------------------------------------
1 | module.exports = function(sqlLibraryType){
2 | // Use sql-wasm.js by default
3 | var sqlJsLib = sqlLibraryType ? "../dist/sql-"+sqlLibraryType+".js" : "../dist/sql-wasm.js";
4 | begin = new Date();
5 | var initSqlJs = require(sqlJsLib);
6 | return initSqlJs().then((sql)=>{
7 | end = new Date();
8 | console.log(`Loaded and inited ${sqlJsLib} in ${end -begin}ms`);
9 | return sql;
10 | });
11 | }
12 |
--------------------------------------------------------------------------------
/sql.js/test/run.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | passed=0
4 | total=0
5 | for f in $(dirname $0)/test_*.js
6 | do
7 | total=$((total+1))
8 | echo -ne "Testing $f...\t"
9 | node "$f" > /tmp/sqljstest
10 | if [ $? = 0 ]
11 | then
12 | echo "Passed."
13 | passed=$((passed+1))
14 | else
15 | echo -e "\033[31mFail!\e[0m"
16 | cat /tmp/sqljstest
17 | fi
18 | done
19 |
20 | if [ $passed = $total ]
21 | then
22 | echo -e "\033[32mAll $total tests passed\e[0m"
23 | exit 0
24 | else
25 | echo -e "\033[31mWarning\e[0m : $passed tests passed out of $total"
26 | exit 1
27 | fi
28 |
--------------------------------------------------------------------------------
/sql.js/test/sql-requireJS.html:
--------------------------------------------------------------------------------
1 | <meta charset="utf8" />
2 | <script src='http://requirejs.org/docs/release/2.1.14/minified/require.js'></script>
3 |
4 | <script>
5 | require.config({
6 | baseUrl: '../js'
7 | });
8 |
9 | require(['sql'], function success (SQL){
10 | if (typeof SQL !== 'object') {
11 | document.body.style.backgroundColor = 'red';
12 | alert("Failed to require sql.js through AMD");
13 | } else {
14 | document.body.style.backgroundColor = 'green';
15 | alert("sql.js successfully loaded with AMD");
16 | }
17 | }, function error(err) {
18 | console.log(err);
19 | alert("Module load failed");
20 | });
21 | </script>
22 |
--------------------------------------------------------------------------------
/sql.js/test/test.sqlite:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/phiresky/sql.js-httpvfs/c64536d2acc78feeac17c34bfa1895df01050129/sql.js/test/test.sqlite
--------------------------------------------------------------------------------
/sql.js/test/test_blob.js:
--------------------------------------------------------------------------------
1 | exports.test = function(SQL, assert){
2 | var db = new SQL.Database();
3 | db.exec("CREATE TABLE test (data); INSERT INTO test VALUES (x'6162ff'),(x'00')"); // Insert binary data. This is invalid UTF8 on purpose
4 |
5 | var stmt = db.prepare("INSERT INTO test VALUES (?)");
6 | var bigArray = new Uint8Array(1e6);
7 | bigArray[500] = 0x42
8 | stmt.run([ bigArray ]);
9 |
10 | var stmt = db.prepare("SELECT * FROM test ORDER BY length(data) DESC");
11 |
12 | stmt.step();
13 | var array = stmt.get()[0];
14 | assert.equal(array.length, bigArray.length, "BLOB read from the database should be the same size as the one that was inserted");
15 | for (var i=0; i<array.length; i++) {
16 | // Avoid doing 1e6 assert, to not pollute the console
17 | if (array[i]!==bigArray[i])
18 | assert.fail(array[i], bigArray[i] , "The blob stored in the database should be exactly the same as the one that was inserted");
19 | }
20 |
21 | stmt.step();
22 | var res = stmt.get();
23 | assert.deepEqual(res, [new Uint8Array([0x61, 0x62, 0xff])], "Reading BLOB");
24 |
25 | stmt.step();
26 | var res = stmt.get();
27 | assert.deepEqual(res, [new Uint8Array([0x00])], "Reading BLOB with a null byte");
28 |
29 | assert.strictEqual(stmt.step(), false, "stmt.step() should return false after all values were read");
30 | db.close();
31 | };
32 |
33 | if (module == require.main) {
34 | const target_file = process.argv[2];
35 | const sql_loader = require('./load_sql_lib');
36 | sql_loader(target_file).then((sql)=>{
37 | require('test').run({
38 | 'test blob': function(assert){
39 | exports.test(sql, assert);
40 | }
41 | });
42 | })
43 | .catch((e)=>{
44 | console.error(e);
45 | });
46 | }
47 |
--------------------------------------------------------------------------------
/sql.js/test/test_database.js:
--------------------------------------------------------------------------------
1 | exports.test = function(SQL, assert, done) {
2 | assert.notEqual(SQL.Database, undefined, "Should export a Database object");
3 |
4 | // Create a database
5 | var db = new SQL.Database();
6 | assert.equal(Object.getPrototypeOf(db), SQL.Database.prototype, "Creating a database object");
7 |
8 | // Execute some sql
9 | sqlstr = "CREATE TABLE test (a, b, c, d, e);";
10 | var res = db.exec(sqlstr);
11 | assert.deepEqual(res, [], "Creating a table should not return anything");
12 |
13 | db.run("INSERT INTO test VALUES (NULL, 42, 4.2, 'fourty two', x'42');");
14 |
15 | //Retrieving values
16 | sqlstr = "SELECT * FROM test;";
17 | var res = db.exec(sqlstr);
18 | var expectedResult = [{
19 | columns : ['a','b','c','d','e'],
20 | values : [
21 | [null,42,4.2,'fourty two', new Uint8Array([0x42])]
22 | ]
23 | }];
24 | assert.deepEqual(res, expectedResult, "db.exec() return value");
25 |
26 |
27 | // Export the database to an Uint8Array containing the SQLite database file
28 | var binaryArray = db.export();
29 | assert.strictEqual(String.fromCharCode.apply(null,binaryArray.subarray(0,6)), 'SQLite',
30 | "The first 6 bytes of an SQLite database should form the word 'SQLite'");
31 | db.close();
32 |
33 | var db2 = new SQL.Database(binaryArray);
34 | result = db2.exec("SELECT * FROM test");
35 | assert.deepEqual(result, expectedResult,
36 | "Exporting and re-importing the database should lead to the same database");
37 | db2.close();
38 |
39 | db = new SQL.Database();
40 | assert.deepEqual(db.exec("SELECT * FROM sqlite_master"),
41 | [],
42 | "Newly created databases should be empty");
43 | // Testing db.each
44 | db.run("CREATE TABLE test (a,b); INSERT INTO test VALUES (1,'a'),(2,'b')");
45 | var count = 0, finished = false;
46 | db.each("SELECT * FROM test ORDER BY a", function callback (row){
47 | count++;
48 | if (count === 1) assert.deepEqual(row, {a:1,b:'a'}, 'db.each returns the correct 1st row');
49 | if (count === 2) assert.deepEqual(row, {a:2,b:'b'}, 'db.each returns the correct 2nd row');
50 | }, function last () {
51 | finished = true;
52 | assert.strictEqual(count, 2, "db.each returns the right number of rows");
53 | // No need to wait for this timeout anymore
54 | // In fact, if we do keep waiting for this, we'll get an error when it fires because we've already called done
55 | clearTimeout(testTimeoutId);
56 | done();
57 | });
58 | var testTimeoutId = setTimeout(function timeout(){
59 | if (!finished) {
60 | assert.fail("db.each should call its last callback after having returned the rows");
61 | done();
62 | }
63 | }, 3000);
64 | };
65 |
66 | if (module == require.main) {
67 | const target_file = process.argv[2];
68 | const sql_loader = require('./load_sql_lib');
69 | sql_loader(target_file).then((sql)=>{
70 | require('test').run({
71 | 'test database': function(assert, done){
72 | exports.test(sql, assert, done);
73 | }
74 | });
75 | })
76 | .catch((e)=>{
77 | console.error(e);
78 | assert.fail(e);
79 | });
80 | }
81 |
--------------------------------------------------------------------------------
/sql.js/test/test_errors.js:
--------------------------------------------------------------------------------
1 | exports.test = function(sql, assert) {
2 |
3 | assert.throws(function(){
4 | var db = new sql.Database([1,2,3]);
5 | db.exec("SELECT * FROM sqlite_master");
6 | },
7 | /not a database/,
8 | "Querying an invalid database should throw an error");
9 |
10 | // Create a database
11 | var db = new sql.Database();
12 |
13 | // Execute some sql
14 | var res = db.exec("CREATE TABLE test (a INTEGER PRIMARY KEY, b, c, d, e);");
15 |
16 | assert.throws(function(){
17 | db.exec("I ain't be no valid sql ...");
18 | },
19 | /syntax error/,
20 | "Executing invalid SQL should throw an error");
21 |
22 | assert.throws(function(){
23 | db.run("INSERT INTO test (a) VALUES (1)");
24 | db.run("INSERT INTO test (a) VALUES (1)");
25 | },
26 | /UNIQUE constraint failed/,
27 | "Inserting two rows with the same primary key should fail");
28 |
29 | var stmt = db.prepare("INSERT INTO test (a) VALUES (?)");
30 |
31 |
32 | assert.throws(function(){
33 | stmt.bind([1,2,3]);
34 | },
35 | /out of range/,
36 | "Binding too many parameters should throw an exception");
37 |
38 | assert.throws(function(){
39 | db.run("CREATE TABLE test (this,wont,work)");
40 | },
41 | /table .+ already exists/,
42 | "Trying to create a table with a name that is already used should throw an error");
43 |
44 | stmt.run([2]);
45 | assert.deepEqual(db.exec("SELECT a,b FROM test WHERE a=2"),
46 | [{columns:['a', 'b'],values:[[2, null]]}],
47 | "Previous errors should not have spoiled the statement");
48 |
49 | db.close();
50 |
51 | assert.throws(function(){
52 | stmt.run([3]);
53 | }, "Statements shouldn't be able to execute after the database is closed");
54 | };
55 |
56 | if (module == require.main) {
57 | const target_file = process.argv[2];
58 | const sql_loader = require('./load_sql_lib');
59 | sql_loader(target_file).then((sql)=>{
60 | require('test').run({
61 | 'test errors': function(assert){
62 | exports.test(sql, assert);
63 | }
64 | });
65 | })
66 | .catch((e)=>{
67 | console.error(e);
68 | });
69 | }
70 |
--------------------------------------------------------------------------------
/sql.js/test/test_extension_functions.js:
--------------------------------------------------------------------------------
1 | exports.test = function(sql, assert) {
2 | var db = new sql.Database();
3 | var res = db.exec("CREATE TABLE test (str_data, data);");
4 |
5 | db.run("INSERT INTO test VALUES ('Hello World!', 1);");
6 | db.run("INSERT INTO test VALUES ('', 2);");
7 | db.run("INSERT INTO test VALUES ('', 2);");
8 | db.run("INSERT INTO test VALUES ('', 4);");
9 | db.run("INSERT INTO test VALUES ('', 5);");
10 | db.run("INSERT INTO test VALUES ('', 6);");
11 | db.run("INSERT INTO test VALUES ('', 7);");
12 | db.run("INSERT INTO test VALUES ('', 8);");
13 | db.run("INSERT INTO test VALUES ('', 9);");
14 |
15 | var res = db.exec("SELECT mode(data) FROM test;");
16 | var expectedResult = [{
17 | columns : ['mode(data)'],
18 | values : [
19 | [2]
20 | ]
21 | }];
22 | assert.deepEqual(res, expectedResult, "mode() function works");
23 |
24 | var res = db.exec("SELECT lower_quartile(data) FROM test;");
25 | var expectedResult = [{
26 | columns : ['lower_quartile(data)'],
27 | values : [
28 | [2]
29 | ]
30 | }];
31 | assert.deepEqual(res, expectedResult, "upper_quartile() function works");
32 |
33 | var res = db.exec("SELECT upper_quartile(data) FROM test;");
34 | var expectedResult = [{
35 | columns : ['upper_quartile(data)'],
36 | values : [
37 | [7]
38 | ]
39 | }];
40 | assert.deepEqual(res, expectedResult, "upper_quartile() function works");
41 |
42 | var res = db.exec("SELECT variance(data) FROM test;");
43 | assert.equal(res[0]['values'][0][0].toFixed(2), 8.11, "variance() function works");
44 |
45 | var res = db.exec("SELECT stdev(data) FROM test;");
46 | assert.equal(res[0]['values'][0][0].toFixed(2), 2.85, "stdev() function works");
47 |
48 | var res = db.exec("SELECT acos(data) FROM test;");
49 | assert.equal(res[0]['values'][0][0].toFixed(2), 0, "acos() function works");
50 |
51 | var res = db.exec("SELECT asin(data) FROM test;");
52 | assert.equal(res[0]['values'][0][0].toFixed(2), 1.57, "asin() function works");
53 |
54 | var res = db.exec("SELECT atan2(data, 1) FROM test;");
55 | assert.equal(res[0]['values'][0][0].toFixed(2), 0.79, "atan2() function works");
56 |
57 | var res = db.exec("SELECT difference(str_data, 'ello World!') FROM test;");
58 | assert.equal(res[0]['values'][0][0], 3, "difference() function works");
59 |
60 | var res = db.exec("SELECT ceil(4.1)");
61 | assert.equal(res[0]['values'][0][0], 5, "ceil() function works");
62 |
63 | var res = db.exec("SELECT floor(4.1)");
64 | assert.equal(res[0]['values'][0][0], 4, "floor() function works");
65 |
66 | var res = db.exec("SELECT pi()");
67 | assert.equal(res[0]['values'][0][0].toFixed(5), 3.14159, "pi() function works");
68 |
69 | var res = db.exec("SELECT reverse(str_data) FROM test;");
70 | assert.equal(res[0]['values'][0][0], "!dlroW olleH", "reverse() function works");
71 |
72 | };
73 |
74 | if (module == require.main) {
75 | const target_file = process.argv[2];
76 | const sql_loader = require('./load_sql_lib');
77 | sql_loader(target_file).then((sql)=>{
78 | require('test').run({
79 | 'test extension functions': function(assert){
80 | exports.test(sql, assert);
81 | }
82 | });
83 | })
84 | .catch((e)=>{
85 | console.error(e);
86 | assert.fail(e);
87 | });
88 | }
89 |
--------------------------------------------------------------------------------
/sql.js/test/test_functions.js:
--------------------------------------------------------------------------------
1 | exports.test = function(SQL, assert){
2 | var db = new SQL.Database();
3 | db.exec("CREATE TABLE test (data); INSERT INTO test VALUES ('Hello World');");
4 |
5 | // Simple function, appends extra text on a string.
6 | function test_function(string_arg) {
7 | return "Function called with: " + string_arg;
8 | };
9 |
10 | // Register with SQLite.
11 | db.create_function("TestFunction", test_function);
12 |
13 | // Use in a query, check expected result.
14 | var result = db.exec("SELECT TestFunction(data) FROM test;");
15 | var result_str = result[0]["values"][0][0];
16 | assert.equal(result_str, "Function called with: Hello World", "Named functions can be registered");
17 |
18 | // 2 arg function, adds two ints together.
19 | db.exec("CREATE TABLE test2 (int1, int2); INSERT INTO test2 VALUES (456, 789);");
20 |
21 | function test_add(int1, int2) {
22 | return int1 + int2;
23 | };
24 |
25 | db.create_function("TestAdd", test_add);
26 | result = db.exec("SELECT TestAdd(int1, int2) FROM test2;");
27 | result_int = result[0]["values"][0][0];
28 | assert.equal(result_int, 1245, "Multiple argument functions can be registered");
29 |
30 | // Binary data function, tests which byte in a column is set to 0
31 | db.exec("CREATE TABLE test3 (data); INSERT INTO test3 VALUES (x'6100ff'), (x'ffffff00ffff');");
32 |
33 | function test_zero_byte_index(data) {
34 | // Data is a Uint8Array
35 | for (var i=0; i<data.length; i++) {
36 | if (data[i] === 0) {
37 | return i;
38 | }
39 | }
40 | return -1;
41 | };
42 |
43 | db.create_function("TestZeroByteIndex", test_zero_byte_index);
44 | result = db.exec("SELECT TestZeroByteIndex(data) FROM test3;");
45 | result_int0 = result[0]["values"][0][0];
46 | result_int1 = result[0]["values"][1][0];
47 | assert.equal(result_int0, 1, "Binary data works inside functions");
48 | assert.equal(result_int1, 3, "Binary data works inside functions");
49 |
50 | db.create_function("addOne", function (x) { return x + 1;} );
51 | result = db.exec("SELECT addOne(1);");
52 | assert.equal(result[0]["values"][0][0], 2, "Accepts anonymous functions");
53 |
54 | // Test api support of different sqlite types and special values
55 | db.create_function("identityFunction", function (x) { return x;} );
56 | var verbose=false;
57 | function canHandle(testData)
58 | {
59 | let result={};
60 | let ok=true;
61 | let sql_value=("sql_value" in testData)?testData.sql_value:(""+testData.value);
62 | function simpleEqual(a, b) {return a===b;}
63 | let value_equal=("equal" in testData)?testData.equal:simpleEqual;
64 | db.create_function("CheckTestValue", function (x) {return value_equal(testData.value,x)?12345:5678;});
65 | db.create_function("GetTestValue", function () {return testData.value; });
66 | // Check sqlite to js value conversion
67 | result = db.exec("SELECT CheckTestValue("+sql_value+")==12345");
68 | if(result[0]["values"][0][0]!=1)
69 | {
70 | if(verbose)
71 | assert.ok(false, "Can accept "+testData.info);
72 | ok=false;
73 | }
74 | // Check js to sqlite value conversion
75 | result = db.exec("SELECT GetTestValue()");
76 | if(!value_equal(result[0]["values"][0][0],testData.value))
77 | {
78 | if(verbose)
79 | assert.ok(false, "Can return "+testData.info);
80 | ok=false;
81 | }
82 | // Check sqlite to sqlite value conversion (identityFunction(x)==x)
83 | if(sql_value!=="null")
84 | {
85 | result = db.exec("SELECT identityFunction("+sql_value+")="+sql_value);
86 | }else
87 | {
88 | result = db.exec("SELECT identityFunction("+sql_value+") is null");
89 | }
90 | if(result[0]["values"][0][0]!=1)
91 | {
92 | if(verbose)
93 | assert.ok(false, "Can pass "+testData.info);
94 | ok=false;
95 | }
96 | return ok;
97 | }
98 |
99 | function numberEqual(a, b) {
100 | return (+a)===(+b);
101 | }
102 |
103 | function blobEqual(a, b) {
104 | if(((typeof a)!="object")||(!a)||((typeof b)!="object")||(!b)) return false;
105 | if (a.byteLength !== b.byteLength) return false;
106 | return a.every((val, i) => val === b[i]);
107 | }
108 |
109 | [
110 | {info:"null",value:null}, // sqlite special value null
111 | {info:"false",value:false,sql_value:"0",equal:numberEqual}, // sqlite special value (==false)
112 | {info:"true", value:true,sql_value:"1",equal:numberEqual}, // sqlite special value (==true)
113 | {info:"integer 0",value:0}, // sqlite special value (==false)
114 | {info:"integer 1",value:1}, // sqlite special value (==true)
115 | {info:"integer -1",value:-1},
116 | {info:"long integer 5e+9",value:5000000000}, // int64
117 | {info:"long integer -5e+9",value:-5000000000}, // negative int64
118 | {info:"double",value:0.5},
119 | {info:"string",value:"Test",sql_value:"'Test'"},
120 | {info:"empty string",value:"",sql_value:"''"},
121 | {info:"unicode string",value:"\uC7B8",sql_value:"CAST(x'EC9EB8' AS TEXT)"}, // unicode-hex: C7B8 utf8-hex: EC9EB8
122 | {info:"blob",value:new Uint8Array([0xC7,0xB8]),sql_value:"x'C7B8'",equal:blobEqual},
123 | {info:"empty blob",value:new Uint8Array([]),sql_value:"x''",equal:blobEqual}
124 | ].forEach(function(testData)
125 | {
126 | assert.ok(canHandle(testData),"Can handle "+testData.info);
127 | });
128 |
129 | db.create_function("throwFunction", function () { throw "internal exception"; return 5;} );
130 | assert.throws(function(){db.exec("SELECT throwFunction()");},/internal exception/, "Can handle internal exceptions");
131 |
132 | db.create_function("customeObjectFunction", function () { return {test:123};} );
133 | assert.throws(function(){db.exec("SELECT customeObjectFunction()");},/Wrong API use/, "Reports wrong API use");
134 |
135 | db.close();
136 | };
137 |
138 | if (module == require.main) {
139 | const target_file = process.argv[2];
140 | const sql_loader = require('./load_sql_lib');
141 | sql_loader(target_file).then((sql)=>{
142 | require('test').run({
143 | 'test functions': function(assert, done){
144 | exports.test(sql, assert, done);
145 | }
146 | });
147 | })
148 | .catch((e)=>{
149 | console.error(e);
150 | assert.fail(e);
151 | });
152 | }
153 |
154 |
--------------------------------------------------------------------------------
/sql.js/test/test_functions_recreate.js:
--------------------------------------------------------------------------------
1 | exports.test = function(sql, assert) {
2 | // Test 1: Create a database, Register single function, close database, repeat 1000 times
3 |
4 | for (var i = 1; i <= 1000; i++)
5 | {
6 | let lastStep=(i==1000);
7 | let db = new sql.Database();
8 | function add() {return i;}
9 | try
10 | {
11 | db.create_function("TestFunction"+i, add)
12 | }catch(e)
13 | {
14 | assert.ok(false,"Test 1: Recreate database "+i+"th times and register function failed with exception:"+e);
15 | db.close();
16 | break;
17 | }
18 | var result = db.exec("SELECT TestFunction"+i+"()");
19 | var result_str = result[0]["values"][0][0];
20 | if((result_str!=i)||lastStep)
21 | {
22 | assert.equal(result_str, i, "Test 1: Recreate database "+i+"th times and register function");
23 | db.close();
24 | break;
25 | }
26 | db.close();
27 | }
28 |
29 | // Test 2: Create a database, Register same function 1000 times, close database
30 | {
31 | let db = new sql.Database();
32 | for (var i = 1; i <= 1000; i++)
33 | {
34 | let lastStep=(i==1000);
35 | function add() {return i;}
36 | try
37 | {
38 | db.create_function("TestFunction", add);
39 | }catch(e)
40 | {
41 | assert.ok(false,"Test 2: Reregister function "+i+"th times failed with exception:"+e);
42 | break;
43 | }
44 | var result = db.exec("SELECT TestFunction()");
45 | var result_str = result[0]["values"][0][0];
46 | if((result_str!=i)||lastStep)
47 | {
48 | assert.equal(result_str, i, "Test 2: Reregister function "+i+"th times");
49 | break;
50 | }
51 | }
52 | db.close();
53 | }
54 | };
55 |
56 |
57 | if (module == require.main) {
58 | const target_file = process.argv[2];
59 | const sql_loader = require('./load_sql_lib');
60 | sql_loader(target_file).then((sql)=>{
61 | require('test').run({
62 | 'test creating multiple functions': function(assert){
63 | exports.test(sql, assert);
64 | }
65 | });
66 | })
67 | .catch((e)=>{
68 | console.error(e);
69 | assert.fail(e);
70 | });
71 | }
72 |
--------------------------------------------------------------------------------
/sql.js/test/test_issue128.js:
--------------------------------------------------------------------------------
1 | exports.test = function(sql, assert) {
2 | // Create a database
3 | var db = new sql.Database();
4 |
5 | db.run("CREATE TABLE test (data TEXT);");
6 |
7 | db.exec("SELECT * FROM test;");
8 | assert.deepEqual(db.getRowsModified(), 0, "getRowsModified returns 0 at first");
9 |
10 | db.exec("INSERT INTO test VALUES ('Hello1');");
11 | db.exec("INSERT INTO test VALUES ('Hello');");
12 | db.exec("INSERT INTO test VALUES ('Hello');");
13 | db.exec("INSERT INTO test VALUES ('World4');");
14 | assert.deepEqual(db.getRowsModified(), 1, "getRowsModified works for inserts");
15 |
16 | db.exec("UPDATE test SET data = 'World4' where data = 'Hello';");
17 | assert.deepEqual(db.getRowsModified(), 2, "getRowsModified works for updates");
18 |
19 | db.exec("DELETE FROM test;");
20 | assert.deepEqual(db.getRowsModified(), 4, "getRowsModified works for deletes");
21 |
22 | db.exec("SELECT * FROM test;");
23 | assert.deepEqual(db.getRowsModified(), 4, "getRowsModified unmodified by queries");
24 |
25 | };
26 |
27 | if (module == require.main) {
28 | const target_file = process.argv[2];
29 | const sql_loader = require('./load_sql_lib');
30 | sql_loader(target_file).then((sql)=>{
31 | require('test').run({
32 | 'test issue 128': function(assert){
33 | exports.test(sql, assert);
34 | }
35 | });
36 | })
37 | .catch((e)=>{
38 | console.error(e);
39 | assert.fail(e);
40 | });
41 | }
42 |
--------------------------------------------------------------------------------
/sql.js/test/test_issue325.js:
--------------------------------------------------------------------------------
1 |
2 | exports.test = function(sql, assert){
3 | "use strict";
4 | // Create a database
5 | var db = new sql.Database();
6 |
7 | // binding a large number
8 | assert.strictEqual(
9 | db.exec("SELECT ?", [1.7976931348623157e+308])[0].values[0][0],
10 | 1.7976931348623157e+308,
11 | "binding 1.7976931348623159e+308 as a parameter"
12 | );
13 |
14 | // inline result value test
15 | assert.strictEqual(
16 | db.exec("SELECT 1.7976931348623157e+308")[0].values[0][0],
17 | 1.7976931348623157e+308,
18 | "SELECT 1.7976931348623157e+308 is 1.7976931348623157e+308"
19 | );
20 |
21 | // Close the database and all associated statements
22 | db.close();
23 | };
24 |
25 | if (module == require.main) {
26 | const target_file = process.argv[2];
27 | const sql_loader = require('./load_sql_lib');
28 | sql_loader(target_file).then((sql)=>{
29 | require('test').run({
30 | 'test issue325': function(assert){
31 | exports.test(sql, assert);
32 | }
33 | });
34 | })
35 | .catch((e)=>{
36 | console.error(e);
37 | assert.fail(e);
38 | });
39 | }
40 |
--------------------------------------------------------------------------------
/sql.js/test/test_issue55.js:
--------------------------------------------------------------------------------
1 | exports.test = function(SQL, assert) {
2 | var fs = require('fs');
3 | var path = require('path');
4 |
5 | var filebuffer = fs.readFileSync(path.join(__dirname, 'issue55.db'));
6 |
7 | //Works
8 | var db = new SQL.Database(filebuffer);
9 |
10 | var origCount = db.prepare("SELECT COUNT(*) AS count FROM networklocation").getAsObject({}).count;
11 |
12 | db.run("INSERT INTO networklocation (x, y, network_id, floor_id) VALUES (?, ?, ?, ?)", [123, 123, 1, 1]);
13 |
14 | var count = db.prepare("SELECT COUNT(*) AS count FROM networklocation").getAsObject({}).count;
15 |
16 | assert.equal(count, origCount + 1, "The row has been inserted");
17 | var dbCopy = new SQL.Database(db.export());
18 | var newCount = dbCopy.prepare("SELECT COUNT(*) AS count FROM networklocation").getAsObject({}).count;
19 | assert.equal(newCount, count, "export and reimport copies all the data");
20 | };
21 |
22 | if (module == require.main) {
23 | const target_file = process.argv[2];
24 | const sql_loader = require('./load_sql_lib');
25 | sql_loader(target_file).then((sql)=>{
26 | require('test').run({
27 | 'test issue 55': function(assert){
28 | exports.test(sql, assert);
29 | }
30 | });
31 | })
32 | .catch((e)=>{
33 | console.error(e);
34 | assert.fail(e);
35 | });
36 | }
37 |
38 |
--------------------------------------------------------------------------------
/sql.js/test/test_issue73.js:
--------------------------------------------------------------------------------
1 | exports.test = function(sql, assert) {
2 | // Create a database
3 | var db = new sql.Database();
4 |
5 | // Execute some sql
6 | sqlstr = "CREATE TABLE COMPANY("+
7 | " ID INT PRIMARY KEY NOT NULL,"+
8 | " NAME TEXT NOT NULL,"+
9 | " AGE INT NOT NULL,"+
10 | " ADDRESS CHAR(50),"+
11 | " SALARY REAL"+
12 | " );"+
13 | " CREATE TABLE AUDIT("+
14 | " EMP_ID INT NOT NULL,"+
15 | " ENTRY_DATE TEXT NOT NULL"+
16 | " );"+
17 | " CREATE TRIGGER audit_log AFTER INSERT"+
18 | " ON COMPANY"+
19 | " BEGIN"+
20 | " INSERT INTO AUDIT"+
21 | " (EMP_ID, ENTRY_DATE)"+
22 | " VALUES"+
23 | " (new.ID, '2014-11-10');"+
24 | " END;"+
25 | " INSERT INTO COMPANY VALUES (73,'A',8,'',1200);"+
26 | " SELECT * FROM AUDIT;"+
27 | " INSERT INTO COMPANY VALUES (42,'B',8,'',1600);"+
28 | " SELECT EMP_ID FROM AUDIT ORDER BY EMP_ID";
29 | var res = db.exec(sqlstr);
30 | var expectedResult = [
31 | {
32 | columns : ['EMP_ID','ENTRY_DATE'],
33 | values : [
34 | [73, '2014-11-10']
35 | ]
36 | },
37 | {
38 | columns : ['EMP_ID'],
39 | values : [
40 | [42],[73]
41 | ]
42 | }
43 | ];
44 | assert.deepEqual(res, expectedResult,
45 | "db.exec with a statement that contains a ';'");
46 | };
47 |
48 | if (module == require.main) {
49 | const target_file = process.argv[2];
50 | const sql_loader = require('./load_sql_lib');
51 | sql_loader(target_file).then((sql)=>{
52 | require('test').run({
53 | 'test issue 73': function(assert){
54 | exports.test(sql, assert);
55 | }
56 | });
57 | })
58 | .catch((e)=>{
59 | console.error(e);
60 | assert.fail(e);
61 | });
62 | }
63 |
--------------------------------------------------------------------------------
/sql.js/test/test_issue76.js:
--------------------------------------------------------------------------------
1 | exports.test = function(sql, assert) {
2 | // Create a database
3 | var db = new sql.Database();
4 | // Ultra-simple query
5 | var stmt = db.prepare("VALUES (?)");
6 | // Bind null to the parameter and get the result
7 | assert.deepEqual(stmt.get([null]), [null],
8 | "binding a null value to a statement parameter");
9 | db.close();
10 | };
11 |
12 | if (module == require.main) {
13 | const target_file = process.argv[2];
14 | const sql_loader = require('./load_sql_lib');
15 | sql_loader(target_file).then((sql)=>{
16 | require('test').run({
17 | 'test issue 76': function(assert){
18 | exports.test(sql, assert);
19 | }
20 | });
21 | })
22 | .catch((e)=>{
23 | console.error(e);
24 | assert.fail(e);
25 | });
26 | }
27 |
--------------------------------------------------------------------------------
/sql.js/test/test_json1.js:
--------------------------------------------------------------------------------
1 | exports.test = function(sql, assert) {
2 | var db = new sql.Database();
3 | // tests taken from https://www.sqlite.org/json1.html#jmini
4 | [
5 | // The json() function
6 | `json(' { "this" : "is", "a": [ "test" ] } ') = '{"this":"is","a":["test"]}'`,
7 |
8 | // The json_array_length() function
9 | `json_array(1,2,'3',4) = '[1,2,"3",4]'`,
10 | `json_array('[1,2]') = '["[1,2]"]'`,
11 | `json_array(json_array(1,2)) = '[[1,2]]'`,
12 | `json_array(1,null,'3','[4,5]','{"six":7.7}') = '[1,null,"3","[4,5]","{\\"six\\":7.7}"]'`,
13 | `json_array(1,null,'3',json('[4,5]'),json('{"six":7.7}')) = '[1,null,"3",[4,5],{"six":7.7}]'`,
14 | `json_array_length('[1,2,3,4]') = 4`,
15 | `json_array_length('[1,2,3,4]', '#39;) = 4`,
16 | `json_array_length('[1,2,3,4]', '$[2]') = 0`,
17 | `json_array_length('{"one":[1,2,3]}') = 0`,
18 | `json_array_length('{"one":[1,2,3]}', '$.one') = 3`,
19 | `json_array_length('{"one":[1,2,3]}', '$.two') = null`,
20 |
21 | // The json_extract() function
22 | `json_extract('{"a":2,"c":[4,5,{"f":7}]}', '#39;) = '{"a":2,"c":[4,5,{"f":7}]}'`,
23 | `json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c') = '[4,5,{"f":7}]'`,
24 | `json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]') = '{"f":7}'`,
25 | `json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f') = 7`,
26 | `json_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a') = '[[4,5],2]'`,
27 | `json_extract('{"a":2,"c":[4,5],"f":7}','$.c[#-1]') = 5`,
28 | `json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x') = null`,
29 | `json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x', '$.a') = '[null,2]'`,
30 |
31 | // The json_insert(), json_replace, and json_set() functions
32 | `json_insert('[1,2,3,4]','$[#]',99) = '[1,2,3,4,99]'`,
33 | `json_insert('[1,[2,3],4]','$[1][#]',99) = '[1,[2,3,99],4]'`,
34 | `json_insert('{"a":2,"c":4}', '$.a', 99) = '{"a":2,"c":4}'`,
35 | `json_insert('{"a":2,"c":4}', '$.e', 99) = '{"a":2,"c":4,"e":99}'`,
36 | `json_replace('{"a":2,"c":4}', '$.a', 99) = '{"a":99,"c":4}'`,
37 | `json_replace('{"a":2,"c":4}', '$.e', 99) = '{"a":2,"c":4}'`,
38 | `json_set('{"a":2,"c":4}', '$.a', 99) = '{"a":99,"c":4}'`,
39 | `json_set('{"a":2,"c":4}', '$.e', 99) = '{"a":2,"c":4,"e":99}'`,
40 | `json_set('{"a":2,"c":4}', '$.c', '[97,96]') = '{"a":2,"c":"[97,96]"}'`,
41 | `json_set('{"a":2,"c":4}', '$.c', json('[97,96]')) = '{"a":2,"c":[97,96]}'`,
42 | `json_set('{"a":2,"c":4}', '$.c', json_array(97,96)) = '{"a":2,"c":[97,96]}'`,
43 |
44 | // The json_object() function
45 | `json_object('a',2,'c',4) = '{"a":2,"c":4}'`,
46 | `json_object('a',2,'c','{e:5}') = '{"a":2,"c":"{e:5}"}'`,
47 | `json_object('a',2,'c',json_object('e',5)) = '{"a":2,"c":{"e":5}}'`,
48 |
49 | // The json_patch() function
50 | `json_patch('{"a":1,"b":2}','{"c":3,"d":4}') = '{"a":1,"b":2,"c":3,"d":4}'`,
51 | `json_patch('{"a":[1,2],"b":2}','{"a":9}') = '{"a":9,"b":2}'`,
52 | `json_patch('{"a":[1,2],"b":2}','{"a":null}') = '{"b":2}'`,
53 | `json_patch('{"a":1,"b":2}','{"a":9,"b":null,"c":8}') = '{"a":9,"c":8}'`,
54 | `json_patch('{"a":{"x":1,"y":2},"b":3}','{"a":{"y":9},"c":8}') = '{"a":{"x":1,"y":9},"b":3,"c":8}'`,
55 |
56 | // The json_remove() function
57 | `json_remove('[0,1,2,3,4]','$[2]') = '[0,1,3,4]'`,
58 | `json_remove('[0,1,2,3,4]','$[2]','$[0]') = '[1,3,4]'`,
59 | `json_remove('[0,1,2,3,4]','$[0]','$[2]') = '[1,2,4]'`,
60 | `json_remove('[0,1,2,3,4]','$[#-1]','$[0]') = '[1,2,3]'`,
61 | `json_remove('{"x":25,"y":42}') = '{"x":25,"y":42}'`,
62 | `json_remove('{"x":25,"y":42}','$.z') = '{"x":25,"y":42}'`,
63 | `json_remove('{"x":25,"y":42}','$.y') = '{"x":25}'`,
64 | `json_remove('{"x":25,"y":42}','#39;) = null`,
65 |
66 | // The json_type() function
67 | `json_type('{"a":[2,3.5,true,false,null,"x"]}') = 'object'`,
68 | `json_type('{"a":[2,3.5,true,false,null,"x"]}','#39;) = 'object'`,
69 | `json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a') = 'array'`,
70 | `json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[0]') = 'integer'`,
71 | `json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[1]') = 'real'`,
72 | `json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[2]') = 'true'`,
73 | `json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[3]') = 'false'`,
74 | `json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[4]') = 'null'`,
75 | `json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[5]') = 'text'`,
76 | `json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[6]') = null`,
77 |
78 | // The json_valid() function
79 | `json_valid('{"x":35}') = 1`,
80 | `json_valid('{"x":35') = 0`,
81 |
82 | // The json_quote() function
83 | `json_quote(3.14159) = 3.14159`,
84 | `json_quote('verdant') = "verdant"`
85 | ].forEach(function (sql) {
86 | assert.equal(
87 | String(db.exec(
88 | "SELECT " + sql.split(" = ")[0] + " AS val;"
89 | )[0].values[0][0]),
90 | String(sql.split(" = ")[1].replace(/'/g, ""))
91 | );
92 | });
93 | };
94 |
95 | if (module == require.main) {
96 | const target_file = process.argv[2];
97 | const sql_loader = require('./load_sql_lib');
98 | sql_loader(target_file).then((sql)=>{
99 | require('test').run({
100 | 'test extension functions': function(assert){
101 | exports.test(sql, assert);
102 | }
103 | });
104 | })
105 | .catch((e)=>{
106 | console.error(e);
107 | assert.fail(e);
108 | });
109 | }
110 |
--------------------------------------------------------------------------------
/sql.js/test/test_modularization.js:
--------------------------------------------------------------------------------
1 |
2 | exports.test = function (SQL, assert, done, sqlLibFilename) {
3 | if (!sqlLibFilename){
4 | // Whew - this is ugly and fragile and makes way too many assumptions about how these tests are run from all.js
5 | // However, this is the quickest way to make sure that we are testing the lib that is requested
6 | const targetFile = process.argv[2];
7 | sqlLibFilename = targetFile ? "../dist/sql-"+targetFile+".js" : "../dist/sql-wasm.js";
8 | }
9 |
10 | var initSqlJsLib1 = require(sqlLibFilename);
11 | initSqlJsLib1().then((sqlModule1) => {
12 | var initSqlJsLib2 = require(sqlLibFilename);
13 | initSqlJsLib2().then((sqlModule2) => {
14 | assert.equal(SQL, sqlModule1, "Initializing the module multiple times only creates it once");
15 | assert.equal(sqlModule1, sqlModule2, "Initializing the module multiple times only creates it once");
16 | var db1 = new sqlModule1.Database();
17 | assert.equal(Object.getPrototypeOf(db1), SQL.Database.prototype, "sqlModule1 has a Database object that has the same prototype as the originally loaded SQL module");
18 | assert.equal(Object.getPrototypeOf(db1), sqlModule2.Database.prototype, "sqlModule1 has a Database object that has the same prototype as the sqlModule2");
19 |
20 |
21 | var db2 = new sqlModule2.Database();
22 | assert.equal(Object.getPrototypeOf(db2), sqlModule1.Database.prototype, "sqlModule2 has a Database object that has the same prototype as the sqlModule1");
23 |
24 | done();
25 | });
26 | });
27 | };
28 |
29 | if (module == require.main) {
30 | const targetFile = process.argv[2];
31 | const loadSqlLib = require('./load_sql_lib');
32 | loadSqlLib(targetFile).then((sql) => {
33 | require('test').run({
34 | 'test modularization': function (assert, done) {
35 | // TODO: Dry this up so that this code isn't duped between here and load_sql_lib.js
36 | var sqlJsLibFilename = targetFile ? "../dist/sql-"+targetFile+".js" : "../dist/sql-wasm.js";
37 | exports.test(sql, assert, done, sqlJsLibFilename);
38 | }
39 | })
40 | })
41 | .catch((e) => {
42 | console.error(e);
43 | });
44 | }
45 |
--------------------------------------------------------------------------------
/sql.js/test/test_node_file.js:
--------------------------------------------------------------------------------
1 | exports.test = function(SQL, assert) {
2 | //Node filesystem module - You know that.
3 | var fs = require('fs');
4 |
5 | //Ditto, path module
6 | var path = require('path');
7 |
8 | var filebuffer = fs.readFileSync(path.join(__dirname, 'test.sqlite'));
9 |
10 | //Works
11 | var db = new SQL.Database(filebuffer);
12 |
13 | //[{"columns":["id","content"],"values":[["0","hello"],["1","world"]]}]
14 | var res = db.exec("SELECT * FROM test WHERE id = 0");
15 | assert.deepEqual(res,
16 | [{"columns":["id","content"],"values":[[0,"hello"]]}],
17 | "One should be able to read the contents of an SQLite database file read from disk");
18 | db.close();
19 | }
20 |
21 | if (module == require.main) {
22 | const target_file = process.argv[2];
23 | const sql_loader = require('./load_sql_lib');
24 | sql_loader(target_file).then((sql)=>{
25 | require('test').run({
26 | 'test node file': function(assert){
27 | exports.test(sql, assert);
28 | }
29 | });
30 | })
31 | .catch((e)=>{
32 | console.error(e);
33 | assert.fail(e);
34 | });
35 | }
36 |
--------------------------------------------------------------------------------
/sql.js/test/test_statement.js:
--------------------------------------------------------------------------------
1 |
2 | exports.test = function(sql, assert){
3 | // Create a database
4 | var db = new sql.Database();
5 |
6 | // Execute some sql
7 | sqlstr = "CREATE TABLE alphabet (letter, code);";
8 | db.exec(sqlstr);
9 |
10 | var result = db.exec("SELECT name FROM sqlite_master WHERE type='table'");
11 | assert.deepEqual(result, [{columns:['name'], values:[['alphabet']]}],
12 | "Table properly created");
13 |
14 | // Prepare a statement to insert values in tha database
15 | var stmt = db.prepare("INSERT INTO alphabet (letter,code) VALUES (?,?)");
16 | // Execute the statement several times
17 | stmt.run(['a',1]);
18 | stmt.run(['b',2.2]);
19 | stmt.run(['c']); // The second parameter will be bound to NULL
20 |
21 | // Free the statement
22 | stmt.free();
23 |
24 | result = db.exec("SELECT * FROM alphabet");
25 | assert.deepEqual(result,
26 | [{columns:['letter', 'code'], values:[['a',1],['b',2.2],['c',null]]}],
27 | "Statement.run() should have added data to the database");
28 |
29 | db.run("CREATE TABLE data (nbr, str, no_value); INSERT INTO data VALUES (5, '粵語😄', NULL);");
30 | stmt = db.prepare("SELECT * FROM data");
31 | stmt.step(); // Run the statement
32 | assert.deepEqual(stmt.getColumnNames(), ['nbr','str','no_value'], 'Statement.GetColumnNames()');
33 | var res = stmt.getAsObject();
34 | assert.strictEqual(res.nbr, 5, 'Read number');
35 | assert.strictEqual(res.str, '粵語😄', "Read string");
36 | assert.strictEqual(res.no_value, null, "Read null");
37 | assert.deepEqual(res, {nbr:5, str:'粵語😄', no_value:null}, "Statement.getAsObject()");
38 | stmt.free();
39 |
40 | // getColumnNames() should work even if query returns no data
41 | stmt = db.prepare("SELECT * FROM data WHERE nbr = -1");
42 | assert.deepEqual(stmt.getColumnNames(), ['nbr','str','no_value'], 'Statement.GetColumnNames()');
43 | stmt.free();
44 |
45 | stmt = db.prepare("SELECT str FROM data WHERE str=?");
46 | assert.deepEqual(stmt.getAsObject(['粵語😄']), {'str':'粵語😄'}, "UTF8 support in prepared statements");
47 |
48 | // Prepare an sql statement
49 | stmt = db.prepare("SELECT * FROM alphabet WHERE code BETWEEN :start AND :end ORDER BY code");
50 | // Bind values to the parameters
51 | stmt.bind([0, 256]);
52 | // Execute the statement
53 | stmt.step();
54 | // Get one row of result
55 | result = stmt.get();
56 | assert.deepEqual(result, ['a',1], "Binding named parameters by their position");
57 |
58 | // Fetch the next row of result
59 | result = stmt.step();
60 | assert.equal(result, true);
61 | result = stmt.get();
62 | assert.deepEqual(result, ['b',2.2], "Fetching the next row of result");
63 |
64 | // Reset and reuse at once
65 | result = stmt.get([0, 1]);
66 | assert.deepEqual(result, ['a',1], "Reset and reuse at once");
67 |
68 | // Pass objects to get() and bind() to use named parameters
69 | result = stmt.get({':start':1, ':end':1});
70 | assert.deepEqual(result, ['a',1], "Binding named parameters");
71 |
72 | // Prepare statement, pass null to bind() and check that it works
73 | stmt = db.prepare("SELECT 'bind-with-null'");
74 | result = stmt.bind(null);
75 | assert.equal(result, true);
76 | stmt.step();
77 | result = stmt.get();
78 | assert.equal(result,"bind-with-null")
79 |
80 | // Close the database and all associated statements
81 | db.close();
82 | };
83 |
84 | if (module == require.main) {
85 | const target_file = process.argv[2];
86 | const sql_loader = require('./load_sql_lib');
87 | sql_loader(target_file).then((sql)=>{
88 | require('test').run({
89 | 'test statement': function(assert){
90 | exports.test(sql, assert);
91 | }
92 | });
93 | })
94 | .catch((e)=>{
95 | console.error(e);
96 | assert.fail(e);
97 | });
98 | }
99 |
100 |
--------------------------------------------------------------------------------
/sql.js/test/test_statement_iterator.js:
--------------------------------------------------------------------------------
1 | exports.test = function (SQL, assert) {
2 | // Create a database
3 | var db = new SQL.Database();
4 |
5 | // Multiline SQL
6 | var sqlstr = "CREATE TABLE test (x text, y integer);\n"
7 | + "INSERT INTO test\n"
8 | + "VALUES ('hello', 42), ('goodbye', 17);\n"
9 | + "SELECT * FROM test;\n"
10 | + " -- nothing here";
11 | var sqlstart = "CREATE TABLE test (x text, y integer);"
12 |
13 | // Manual iteration
14 | // Get an iterator
15 | var it = db.iterateStatements(sqlstr);
16 |
17 | // Get first item
18 | var x = it.next();
19 | assert.equal(x.done, false, "Valid iterator object produced");
20 | assert.equal(x.value.getSQL(), sqlstart, "Statement is for first query only");
21 | assert.equal(it.getRemainingSQL(), sqlstr.slice(sqlstart.length), "Remaining sql retrievable");
22 |
23 | // execute the first query
24 | x.value.step();
25 |
26 | // get and execute the second query
27 | x = it.next();
28 | assert.equal(x.done, false, "Second query found");
29 | x.value.step();
30 |
31 | // get and execute the third query
32 | x = it.next();
33 | assert.equal(x.done, false, "Third query found");
34 | x.value.step();
35 | assert.deepEqual(x.value.getColumnNames(), ['x', 'y'], "Third query is SELECT");
36 |
37 | // check for additional queries
38 | x = it.next();
39 | assert.deepEqual(x, { done: true }, "Done reported after last query");
40 |
41 | // additional iteration does nothing
42 | x = it.next();
43 | assert.deepEqual(x, { done: true }, "Done reported when iterating past completion");
44 |
45 | db.run("DROP TABLE test;");
46 |
47 | // for...of
48 | var count = 0;
49 | for (let statement of db.iterateStatements(sqlstr)) {
50 | statement.step();
51 | count = count + 1;
52 | }
53 | assert.equal(count, 3, "For loop iterates correctly");
54 |
55 | var badsql = "SELECT 1 as x;garbage in, garbage out";
56 |
57 | // bad sql will stop iteration
58 | it = db.iterateStatements(badsql);
59 | x = it.next();
60 | x.value.step();
61 | assert.deepEqual(x.value.getAsObject(), { x: 1 }, "SQL before bad statement executes successfully");
62 | assert.throws(function () { it.next() }, /syntax error/, "Bad SQL stops iteration with exception");
63 | assert.deepEqual(it.next(), { done: true }, "Done reported when iterating after exception");
64 |
65 | // valid SQL executes, remaining SQL accessible after exception
66 | it = db.iterateStatements(badsql);
67 | var remains = '';
68 | try {
69 | for (let statement of it) {
70 | statement.step();
71 | }
72 | } catch {
73 | remains = it.getRemainingSQL();
74 | }
75 | assert.equal(remains, "garbage in, garbage out", "Remaining SQL accessible after exception");
76 |
77 | // From the doc example on the iterateStatements method
78 | const results = [];
79 | const sql_queries = "SELECT 1 AS x; SELECT '2' as y";
80 | for (const statement of db.iterateStatements(sql_queries)) {
81 | const sql = statement.getSQL();
82 | const result = statement.getAsObject({});
83 | results.push({ sql, result });
84 | }
85 | assert.deepEqual(results, [
86 | { sql: 'SELECT 1 AS x;', result: { x: 1 } },
87 | { sql: " SELECT '2' as y", result: { y: '2' } }
88 | ], "The code example from the documentation works");
89 | };
90 |
91 | if (module == require.main) {
92 | const target_file = process.argv[2];
93 | const sql_loader = require('./load_sql_lib');
94 | sql_loader(target_file).then((sql) => {
95 | require('test').run({
96 | 'test statement iterator': function (assert) {
97 | exports.test(sql, assert);
98 | }
99 | });
100 | })
101 | .catch((e) => {
102 | console.error(e);
103 | assert.fail(e);
104 | });
105 | }
106 |
--------------------------------------------------------------------------------
/sql.js/test/test_transactions.js:
--------------------------------------------------------------------------------
1 | exports.test = function(SQL, assert){
2 | var db = new SQL.Database();
3 | db.exec("CREATE TABLE test (data); INSERT INTO test VALUES (1);");
4 |
5 | // Open a transaction
6 | db.exec("BEGIN TRANSACTION;");
7 |
8 | // Insert a row
9 | db.exec("INSERT INTO test VALUES (4);")
10 |
11 | // Rollback
12 | db.exec("ROLLBACK;");
13 |
14 | var res = db.exec("SELECT data FROM test WHERE data = 4;");
15 | var expectedResult = [];
16 | assert.deepEqual(res, expectedResult, "transaction rollbacks work");
17 |
18 | // Open a transaction
19 | db.exec("BEGIN TRANSACTION;");
20 |
21 | // Insert a row
22 | db.exec("INSERT INTO test VALUES (4);")
23 |
24 | // Commit
25 | db.exec("COMMIT;");
26 |
27 | var res = db.exec("SELECT data FROM test WHERE data = 4;");
28 | var expectedResult = [{
29 | columns : ['data'],
30 | values : [
31 | [4]
32 | ]
33 | }];
34 | assert.deepEqual(res, expectedResult, "transaction commits work");
35 |
36 | // Open a transaction
37 | db.exec("BEGIN TRANSACTION;");
38 |
39 | // Insert a row
40 | db.exec("INSERT INTO test VALUES (5);")
41 |
42 | // Rollback
43 | db.exec("ROLLBACK;");
44 |
45 | var res = db.exec("SELECT data FROM test WHERE data IN (4,5);");
46 | var expectedResult = [{
47 | columns : ['data'],
48 | values : [
49 | [4]
50 | ]
51 | }];
52 | assert.deepEqual(res, expectedResult, "transaction rollbacks after commits work");
53 |
54 | db.close();
55 | };
56 |
57 | if (module == require.main) {
58 | const target_file = process.argv[2];
59 | const sql_loader = require('./load_sql_lib');
60 | sql_loader(target_file).then((sql)=>{
61 | require('test').run({
62 | 'test transactions': function(assert){
63 | exports.test(sql, assert);
64 | }
65 | });
66 | })
67 | .catch((e)=>{
68 | console.error(e);
69 | assert.fail(e);
70 | });
71 | }
72 |
--------------------------------------------------------------------------------
/sql.js/test/test_worker.js:
--------------------------------------------------------------------------------
1 | // TODO: Instead of using puppeteer, we could use the new Node 11 workers via
2 | // node --experimental-worker test/all.js
3 | // Then we could do this:
4 | //const { Worker } = require('worker_threads');
5 | // But it turns out that the worker_threads interface is just different enough not to work.
6 | var puppeteer = require("puppeteer");
7 | var path = require("path");
8 | var fs = require("fs");
9 |
10 | class Worker {
11 | constructor(handle) {
12 | this.handle = handle;
13 | }
14 | static async fromFile(file) {
15 | const browser = await puppeteer.launch();
16 | const page = await browser.newPage();
17 | const source = fs.readFileSync(file, 'utf8');
18 | const worker = await page.evaluateHandle(x => {
19 | const url = URL.createObjectURL(new Blob([x]), { type: 'application/javascript; charset=utf-8' });
20 | return new Worker(url);
21 | }, source);
22 | return new Worker(worker);
23 | }
24 | async postMessage(msg) {
25 | return await this.handle.evaluate((worker, msg) => {
26 | return new Promise((accept, reject) => {
27 | setTimeout(reject, 20000, new Error("time out"));
28 | worker.onmessage = evt => accept(evt.data);
29 | worker.onerror = reject;
30 | worker.postMessage(msg);
31 | })
32 | }, msg);
33 | }
34 | }
35 |
36 | exports.test = async function test(SQL, assert) {
37 | var target = process.argv[2];
38 | var file = target ? "sql-" + target : "sql-wasm";
39 | if (file.indexOf('wasm') > -1 || file.indexOf('memory-growth') > -1) {
40 | console.error("Skipping worker test for " + file + ". Not implemented yet");
41 | return;
42 | };
43 | // If we use puppeteer, we need to pass in this new cwd as the root of the file being loaded:
44 | const filename = "../dist/worker." + file + ".js";
45 | var worker = await Worker.fromFile(path.join(__dirname, filename));
46 | var data = await worker.postMessage({ id: 1, action: 'open' });
47 | assert.strictEqual(data.id, 1, "Return the given id in the correct format");
48 | assert.deepEqual(data, { id: 1, ready: true }, 'Correct data answered to the "open" query');
49 |
50 | data = await worker.postMessage({
51 | id: 2,
52 | action: 'exec',
53 | params: {
54 | ":num2": 2,
55 | "@str2": 'b',
56 | // test_worker.js has issue message-passing Uint8Array
57 | // but it works fine in real-world browser-usage
58 | // "$hex2": new Uint8Array([0x00, 0x42]),
59 | ":num3": 3,
60 | "@str3": 'c'
61 | // "$hex3": new Uint8Array([0x00, 0x44])
62 | },
63 | sql: "CREATE TABLE test (num, str, hex);" +
64 | "INSERT INTO test VALUES (1, 'a', x'0042');" +
65 | "INSERT INTO test VALUES (:num2, @str2, x'0043');" +
66 | // test passing params split across multi-statement "exec"
67 | "INSERT INTO test VALUES (:num3, @str3, x'0044');" +
68 | "SELECT * FROM test;"
69 | });
70 | assert.strictEqual(data.id, 2, "Correct id");
71 | // debug error
72 | assert.strictEqual(data.error, undefined, data.error);
73 | var results = data.results;
74 | assert.ok(Array.isArray(results), 'Correct result type');
75 | assert.strictEqual(results.length, 1, 'Expected exactly 1 table');
76 | var table = results[0];
77 | assert.strictEqual(typeof table, 'object', 'Type of the returned table');
78 | assert.deepEqual(table.columns, ['num', 'str', 'hex'], 'Reading column names');
79 | assert.strictEqual(table.values[0][0], 1, 'Reading number');
80 | assert.strictEqual(table.values[0][1], 'a', 'Reading string');
81 | assert.deepEqual(obj2array(table.values[0][2]), [0x00, 0x42], 'Reading BLOB byte');
82 | assert.strictEqual(table.values[1][0], 2, 'Reading number');
83 | assert.strictEqual(table.values[1][1], 'b', 'Reading string');
84 | assert.deepEqual(obj2array(table.values[1][2]), [0x00, 0x43], 'Reading BLOB byte');
85 | assert.strictEqual(table.values[2][0], 3, 'Reading number');
86 | assert.strictEqual(table.values[2][1], 'c', 'Reading string');
87 | assert.deepEqual(obj2array(table.values[2][2]), [0x00, 0x44], 'Reading BLOB byte');
88 |
89 | data = await worker.postMessage({ action: 'export' });
90 | var header = "SQLite format 3\0";
91 | var actual = "";
92 | for (let i = 0; i < header.length; i++) actual += String.fromCharCode(data.buffer[i]);
93 | assert.equal(actual, header, 'Data returned is an SQLite database file');
94 |
95 | // test worker properly opens db after closing
96 | await worker.postMessage({ action: "close" });
97 | await worker.postMessage({ action: "open" });
98 | data = await worker.postMessage({ action: "exec", sql: "SELECT 1" });
99 | assert.deepEqual(data.results, [{"columns":["1"],"values":[[1]]}]);
100 | }
101 |
102 | function obj2array(obj) {
103 | var buffer = []
104 | for (var p in obj) { buffer[p] = obj[p] }
105 | return buffer;
106 | }
107 |
108 | if (module == require.main) {
109 | process.on('unhandledRejection', r => console.log(r));
110 |
111 | require('test').run({
112 | 'test worker': function (assert, done) {
113 | exports.test(null, assert).then(done);
114 | }
115 | });
116 |
117 | }
118 |
--------------------------------------------------------------------------------
/src/db.ts:
--------------------------------------------------------------------------------
1 | /// <reference lib="dom" />
2 | /// <reference lib="dom.iterable" />
3 |
4 | // TODO: using comlink for all this is a pretty ugly hack
5 | import * as Comlink from "comlink";
6 |
7 | import {
8 | LazyHttpDatabase,
9 | SplitFileConfig,
10 | SqliteComlinkMod,
11 | } from "./sqlite.worker";
12 |
13 | import { DomRow, MainThreadRequest } from "./vtab";
14 |
15 | Comlink.transferHandlers.set("WORKERSQLPROXIES", {
16 | canHandle: (obj): obj is unknown => false,
17 | serialize(obj) {
18 | throw Error("no");
19 | },
20 | deserialize: (port: MessagePort) => {
21 | port.start();
22 | return Comlink.wrap(port);
23 | },
24 | });
25 | export type SqliteWorker = Comlink.Remote<SqliteComlinkMod>;
26 | export interface WorkerHttpvfs {
27 | db: Comlink.Remote<LazyHttpDatabase>;
28 | worker: Comlink.Remote<SqliteComlinkMod>;
29 | configs: SplitFileConfig[];
30 | release: () => void;
31 | }
32 | export async function createDbWorker(
33 | configs: SplitFileConfig[],
34 | workerUrl: string,
35 | wasmUrl: string,
36 | maxBytesToRead: number = Infinity
37 | ): Promise<WorkerHttpvfs> {
38 | const worker: Worker = new Worker(workerUrl);
39 | const sqlite = Comlink.wrap<SqliteComlinkMod>(worker);
40 |
41 | const db = ((await sqlite.SplitFileHttpDatabase(
42 | wasmUrl,
43 | configs,
44 | undefined,
45 | maxBytesToRead
46 | )) as unknown) as Comlink.Remote<LazyHttpDatabase>;
47 | const release = () => {
48 | db[Comlink.releaseProxy]();
49 | sqlite[Comlink.releaseProxy]();
50 | worker.terminate();
51 | }
52 |
53 | worker.addEventListener("message", handleAsyncRequestFromWorkerThread);
54 | return { db, worker: sqlite, configs, release };
55 | }
56 |
57 | async function handleAsyncRequestFromWorkerThread(ev: MessageEvent) {
58 | if (ev.data && ev.data.action === "eval") {
59 | const metaArray = new Int32Array(ev.data.notify, 0, 2);
60 | const dataArray = new Uint8Array(ev.data.notify, 2 * 4);
61 | let response;
62 | try {
63 | response = { ok: await handleDomVtableRequest(ev.data.request) };
64 | } catch (e) {
65 | console.error("worker request error", ev.data.request, e);
66 | response = { err: String(e) };
67 | }
68 | const text = new TextEncoder().encode(JSON.stringify(response));
69 | dataArray.set(text, 0); // need to copy here because:
70 | // sadly TextEncoder.encodeInto: Argument 2 can't be a SharedArrayBuffer or an ArrayBufferView backed by a SharedArrayBuffer [AllowShared]
71 | // otherwise this would be better:
72 | /*const encodeRes = new TextEncoder().encodeInto(response, data);
73 | if (encodeRes.read !== response.length) {
74 | console.log(encodeRes, response.length)
75 | throw Error(`not enough space for response: ${response.length} > ${data.byteLength}`);
76 | }*/
77 | metaArray[1] = text.length;
78 | Atomics.notify(metaArray, 0);
79 | }
80 | }
81 | function getUniqueSelector(elm: Element) {
82 | if (elm.tagName === "BODY") return "body";
83 | const names = [];
84 | while (elm.parentElement && elm.tagName !== "BODY") {
85 | if (elm.id) {
86 | // assume id is unique (which it isn't)
87 | names.unshift("#" + elm.id);
88 | break;
89 | } else {
90 | let c = 1;
91 | let e = elm;
92 | while (e.previousElementSibling) {
93 | e = e.previousElementSibling;
94 | c++;
95 | }
96 | names.unshift(elm.tagName.toLowerCase() + ":nth-child(" + c + ")");
97 | }
98 | elm = elm.parentElement;
99 | }
100 | return names.join(" > ");
101 | }
102 |
103 | function keys<T>(o: T): (keyof T)[] {
104 | return Object.keys(o) as (keyof T)[];
105 | }
106 | async function handleDomVtableRequest(
107 | req: MainThreadRequest
108 | ): Promise<Partial<DomRow>[] | null> {
109 | console.log("dom vtable request", req);
110 | if (req.type === "select") {
111 | return [...document.querySelectorAll(req.selector)].map((e) => {
112 | const out: Partial<DomRow> = {};
113 | for (const column of req.columns) {
114 | if (column === "selector") {
115 | out.selector = getUniqueSelector(e);
116 | } else if (column === "parent") {
117 | if (e.parentElement) {
118 | out.parent = e.parentElement
119 | ? getUniqueSelector(e.parentElement)
120 | : null;
121 | }
122 | } else if (column === "idx") {
123 | // ignore
124 | } else {
125 | out[column] = e[column] as string;
126 | }
127 | }
128 | return out;
129 | });
130 | } else if (req.type === "insert") {
131 | if (!req.value.parent)
132 | throw Error(`"parent" column must be set when inserting`);
133 | const target = document.querySelectorAll(req.value.parent);
134 | if (target.length === 0)
135 | throw Error(`Parent element ${req.value.parent} could not be found`);
136 | if (target.length > 1)
137 | throw Error(
138 | `Parent element ${req.value.parent} ambiguous (${target.length} results)`
139 | );
140 | const parent = target[0];
141 | if (!req.value.tagName) throw Error(`tagName must be set for inserting`);
142 | const ele = document.createElement(req.value.tagName);
143 | const cantSet = ["idx"];
144 | for (const i of keys(req.value)) {
145 | if (req.value[i] !== null) {
146 | if (i === "tagName" || i === "parent") continue;
147 | if (i === "idx" || i === "selector") throw Error(`${i} can't be set`);
148 |
149 | ele[i] = req.value[i] as any;
150 | }
151 | }
152 | parent.appendChild(ele);
153 | return null;
154 | } else if (req.type === "update") {
155 | const targetElement = document.querySelector(req.value.selector!);
156 | if (!targetElement) throw Error(`Element ${req.value.selector} not found!`);
157 | const toSet: (
158 | | "innerHTML"
159 | | "id"
160 | | "textContent"
161 | | "innerHTML"
162 | | "outerHTML"
163 | | "className"
164 | )[] = [];
165 | for (const k of keys(req.value)) {
166 | const v = req.value[k];
167 | if (k === "parent") {
168 | if (v !== getUniqueSelector(targetElement.parentElement!)) {
169 | const targetParent = document.querySelectorAll(v as string);
170 | if (targetParent.length !== 1)
171 | throw Error(
172 | `Invalid target parent: found ${targetParent.length} matches`
173 | );
174 | targetParent[0].appendChild(targetElement);
175 | }
176 | continue;
177 | }
178 | if (k === "idx" || k === "selector") continue;
179 | if (v !== targetElement[k]) {
180 | console.log("SETTING ", k, targetElement[k], "->", v);
181 | if (k === "tagName") throw Error("can't change tagName");
182 | toSet.push(k); // defer setting to prevent setting multiple interdependent values (e.g. textContent and innerHTML)
183 | }
184 | }
185 | for (const k of toSet) {
186 | targetElement[k] = req.value[k] as string;
187 | }
188 | return null;
189 | } else {
190 | throw Error(`unknown request ${req.type}`);
191 | }
192 | }
193 |
--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
1 | export * from "./db";
2 | export type { SqliteStats } from "./sqlite.worker";
3 | export type { PageReadLog } from "./lazyFile";
4 |
--------------------------------------------------------------------------------
/src/lazyFile.ts:
--------------------------------------------------------------------------------
1 | // adapted from https://github.com/emscripten-core/emscripten/blob/cbc974264e0b0b3f0ce8020fb2f1861376c66545/src/library_fs.js
2 | // flexible chunk size parameter
3 | // Creates a file record for lazy-loading from a URL. XXX This requires a synchronous
4 | // XHR, which is not possible in browsers except in a web worker!
5 |
6 | export type RangeMapper = (
7 | fromByte: number,
8 | toByte: number
9 | ) => { url: string; fromByte: number; toByte: number };
10 |
11 | export type RequestLimiter = (bytes: number) => void;
12 |
13 | export type LazyFileConfig = {
14 | /** function to map a read request to an url with read request */
15 | rangeMapper: RangeMapper;
16 | /** must be known beforehand if there's multiple server chunks (i.e. rangeMapper returns different urls) */
17 | fileLength?: number;
18 | /** chunk size for random access requests (should be same as sqlite page size) */
19 | requestChunkSize: number;
20 | /** number of virtual read heads. default: 3 */
21 | maxReadHeads?: number;
22 | /** max read speed for sequential access. default: 5 MiB */
23 | maxReadSpeed?: number;
24 | /** if true, log all read pages into the `readPages` field for debugging */
25 | logPageReads?: boolean;
26 | /** if defined, this is called once per request and passed the number of bytes about to be requested **/
27 | requestLimiter?: RequestLimiter;
28 | };
29 | export type PageReadLog = {
30 | pageno: number;
31 | // if page was already loaded
32 | wasCached: boolean;
33 | // how many pages were prefetched
34 | prefetch: number;
35 | };
36 |
37 | type ReadHead = { startChunk: number; speed: number };
38 | export class LazyUint8Array {
39 | private serverChecked = false;
40 | private readonly chunks: Uint8Array[] = []; // Loaded chunks. Index is the chunk number
41 | totalFetchedBytes = 0;
42 | totalRequests = 0;
43 | readPages: PageReadLog[] = [];
44 | private _length?: number;
45 |
46 | // LRU list of read heds, max length = maxReadHeads. first is most recently used
47 | private readonly readHeads: ReadHead[] = [];
48 | private readonly _chunkSize: number;
49 | private readonly rangeMapper: RangeMapper;
50 | private readonly maxSpeed: number;
51 | private readonly maxReadHeads: number;
52 | private readonly logPageReads: boolean;
53 | private readonly requestLimiter: RequestLimiter;
54 |
55 | constructor(config: LazyFileConfig) {
56 | this._chunkSize = config.requestChunkSize;
57 | this.maxSpeed = Math.round(
58 | (config.maxReadSpeed || 5 * 1024 * 1024) / this._chunkSize
59 | ); // max 5MiB at once
60 | this.maxReadHeads = config.maxReadHeads ?? 3;
61 | this.rangeMapper = config.rangeMapper;
62 | this.logPageReads = config.logPageReads ?? false;
63 | if (config.fileLength) {
64 | this._length = config.fileLength;
65 | }
66 | this.requestLimiter = config.requestLimiter == null ? ((ignored) => {}) : config.requestLimiter;
67 | }
68 | /**
69 | * efficiently copy the range [start, start + length) from the http file into the
70 | * output buffer at position [outOffset, outOffest + length)
71 | * reads from cache or synchronously fetches via HTTP if needed
72 | */
73 | copyInto(
74 | buffer: Uint8Array,
75 | outOffset: number,
76 | length: number,
77 | start: number
78 | ): number {
79 | if (start >= this.length) return 0;
80 | length = Math.min(this.length - start, length);
81 | const end = start + length;
82 | let i = 0;
83 | while (i < length) {
84 | // {idx: 24, chunkOffset: 24, chunkNum: 0, wantedSize: 16}
85 | const idx = start + i;
86 | const chunkOffset = idx % this.chunkSize;
87 | const chunkNum = (idx / this.chunkSize) | 0;
88 | const wantedSize = Math.min(this.chunkSize, end - idx);
89 | let inChunk = this.getChunk(chunkNum);
90 | if (chunkOffset !== 0 || wantedSize !== this.chunkSize) {
91 | inChunk = inChunk.subarray(chunkOffset, chunkOffset + wantedSize);
92 | }
93 | buffer.set(inChunk, outOffset + i);
94 | i += inChunk.length;
95 | }
96 | return length;
97 | }
98 |
99 | private lastGet = -1;
100 | /* find the best matching existing read head to get the given chunk or create a new one */
101 | private moveReadHead(wantedChunkNum: number): ReadHead {
102 | for (const [i, head] of this.readHeads.entries()) {
103 | const fetchStartChunkNum = head.startChunk + head.speed;
104 | const newSpeed = Math.min(this.maxSpeed, head.speed * 2);
105 | const wantedIsInNextFetchOfHead =
106 | wantedChunkNum >= fetchStartChunkNum &&
107 | wantedChunkNum < fetchStartChunkNum + newSpeed;
108 | if (wantedIsInNextFetchOfHead) {
109 | head.speed = newSpeed;
110 | head.startChunk = fetchStartChunkNum;
111 | if (i !== 0) {
112 | // move head to front
113 | this.readHeads.splice(i, 1);
114 | this.readHeads.unshift(head);
115 | }
116 | return head;
117 | }
118 | }
119 | const newHead: ReadHead = {
120 | startChunk: wantedChunkNum,
121 | speed: 1,
122 | };
123 | this.readHeads.unshift(newHead);
124 | while (this.readHeads.length > this.maxReadHeads) this.readHeads.pop();
125 | return newHead;
126 | }
127 | /** get the given chunk from cache or fetch it from remote */
128 | private getChunk(wantedChunkNum: number): Uint8Array {
129 | let wasCached = true;
130 | if (typeof this.chunks[wantedChunkNum] === "undefined") {
131 | wasCached = false;
132 | // double the fetching chunk size if the wanted chunk would be within the next fetch request
133 | const head = this.moveReadHead(wantedChunkNum);
134 |
135 | const chunksToFetch = head.speed;
136 | const startByte = head.startChunk * this.chunkSize;
137 | let endByte = (head.startChunk + chunksToFetch) * this.chunkSize - 1; // including this byte
138 | endByte = Math.min(endByte, this.length - 1); // if datalength-1 is selected, this is the last block
139 |
140 | const buf = this.doXHR(startByte, endByte);
141 | for (let i = 0; i < chunksToFetch; i++) {
142 | const curChunk = head.startChunk + i;
143 | if (i * this.chunkSize >= buf.byteLength) break; // past end of file
144 | const curSize =
145 | (i + 1) * this.chunkSize > buf.byteLength
146 | ? buf.byteLength - i * this.chunkSize
147 | : this.chunkSize;
148 | // console.log("constructing chunk", buf.byteLength, i * this.chunkSize, curSize);
149 | this.chunks[curChunk] = new Uint8Array(
150 | buf,
151 | i * this.chunkSize,
152 | curSize
153 | );
154 | }
155 | }
156 | if (typeof this.chunks[wantedChunkNum] === "undefined")
157 | throw new Error("doXHR failed (bug)!");
158 | const boring = !this.logPageReads || this.lastGet == wantedChunkNum;
159 | if (!boring) {
160 | this.lastGet = wantedChunkNum;
161 | this.readPages.push({
162 | pageno: wantedChunkNum,
163 | wasCached,
164 | prefetch: wasCached ? 0 : this.readHeads[0].speed - 1,
165 | });
166 | }
167 | return this.chunks[wantedChunkNum];
168 | }
169 | /** verify the server supports range requests and find out file length */
170 | private checkServer() {
171 | var xhr = new XMLHttpRequest();
172 | const url = this.rangeMapper(0, 0).url;
173 | // can't set Accept-Encoding header :( https://stackoverflow.com/questions/41701849/cannot-modify-accept-encoding-with-fetch
174 | xhr.open("HEAD", url, false);
175 | // // maybe this will help it not use compression?
176 | // xhr.setRequestHeader("Range", "bytes=" + 0 + "-" + 1e12);
177 | xhr.send(null);
178 | if (!((xhr.status >= 200 && xhr.status < 300) || xhr.status === 304))
179 | throw new Error("Couldn't load " + url + ". Status: " + xhr.status);
180 | var datalength: number | null = Number(
181 | xhr.getResponseHeader("Content-length")
182 | );
183 |
184 | var hasByteServing = xhr.getResponseHeader("Accept-Ranges") === "bytes";
185 | const encoding = xhr.getResponseHeader("Content-Encoding");
186 | var usesCompression = encoding && encoding !== "identity";
187 |
188 | if (!hasByteServing) {
189 | const msg =
190 | "Warning: The server did not respond with Accept-Ranges=bytes. It either does not support byte serving or does not advertise it (`Accept-Ranges: bytes` header missing), or your database is hosted on CORS and the server doesn't mark the accept-ranges header as exposed. This may lead to incorrect results.";
191 | console.warn(
192 | msg,
193 | "(seen response headers:",
194 | xhr.getAllResponseHeaders(),
195 | ")"
196 | );
197 | // throw Error(msg);
198 | }
199 | if (usesCompression) {
200 | console.warn(
201 | `Warning: The server responded with ${encoding} encoding to a HEAD request. Ignoring since it may not do so for Range HTTP requests, but this will lead to incorrect results otherwise since the ranges will be based on the compressed data instead of the uncompressed data.`
202 | );
203 | }
204 | if (usesCompression) {
205 | // can't use the given data length if there's compression
206 | datalength = null;
207 | }
208 |
209 | if (!this._length) {
210 | if (!datalength) {
211 | console.error("response headers", xhr.getAllResponseHeaders());
212 | throw Error("Length of the file not known. It must either be supplied in the config or given by the HTTP server.");
213 | }
214 | this._length = datalength;
215 | }
216 | this.serverChecked = true;
217 | }
218 | get length() {
219 | if (!this.serverChecked) {
220 | this.checkServer();
221 | }
222 | return this._length!;
223 | }
224 |
225 | get chunkSize() {
226 | if (!this.serverChecked) {
227 | this.checkServer();
228 | }
229 | return this._chunkSize!;
230 | }
231 | private doXHR(absoluteFrom: number, absoluteTo: number) {
232 | console.log(
233 | `[xhr of size ${(absoluteTo + 1 - absoluteFrom) / 1024} KiB @ ${
234 | absoluteFrom / 1024
235 | } KiB]`
236 | );
237 | this.requestLimiter(absoluteTo - absoluteFrom);
238 | this.totalFetchedBytes += absoluteTo - absoluteFrom;
239 | this.totalRequests++;
240 | if (absoluteFrom > absoluteTo)
241 | throw new Error(
242 | "invalid range (" +
243 | absoluteFrom +
244 | ", " +
245 | absoluteTo +
246 | ") or no bytes requested!"
247 | );
248 | if (absoluteTo > this.length - 1)
249 | throw new Error(
250 | "only " + this.length + " bytes available! programmer error!"
251 | );
252 | const {
253 | fromByte: from,
254 | toByte: to,
255 | url,
256 | } = this.rangeMapper(absoluteFrom, absoluteTo);
257 |
258 | // TODO: Use mozResponseArrayBuffer, responseStream, etc. if available.
259 | var xhr = new XMLHttpRequest();
260 | xhr.open("GET", url, false);
261 | if (this.length !== this.chunkSize)
262 | xhr.setRequestHeader("Range", "bytes=" + from + "-" + to);
263 |
264 | // Some hints to the browser that we want binary data.
265 | xhr.responseType = "arraybuffer";
266 | if (xhr.overrideMimeType) {
267 | xhr.overrideMimeType("text/plain; charset=x-user-defined");
268 | }
269 |
270 | xhr.send(null);
271 | if (!((xhr.status >= 200 && xhr.status < 300) || xhr.status === 304))
272 | throw new Error("Couldn't load " + url + ". Status: " + xhr.status);
273 | if (xhr.response !== undefined) {
274 | return xhr.response as ArrayBuffer;
275 | } else {
276 | throw Error("xhr did not return uint8array");
277 | }
278 | }
279 | }
280 | /** create the actual file object for the emscripten file system */
281 | export function createLazyFile(
282 | FS: any,
283 | parent: string,
284 | name: string,
285 | canRead: boolean,
286 | canWrite: boolean,
287 | lazyFileConfig: LazyFileConfig
288 | ) {
289 | var lazyArray = new LazyUint8Array(lazyFileConfig);
290 | var properties = { isDevice: false, contents: lazyArray };
291 |
292 | var node = FS.createFile(parent, name, properties, canRead, canWrite);
293 | node.contents = lazyArray;
294 | // Add a function that defers querying the file size until it is asked the first time.
295 | Object.defineProperties(node, {
296 | usedBytes: {
297 | get: /** @this {FSNode} */ function () {
298 | return this.contents.length;
299 | },
300 | },
301 | });
302 | // override each stream op with one that tries to force load the lazy file first
303 | var stream_ops: any = {};
304 | var keys = Object.keys(node.stream_ops);
305 | keys.forEach(function (key) {
306 | var fn = node.stream_ops[key];
307 | stream_ops[key] = function forceLoadLazyFile() {
308 | FS.forceLoadFile(node);
309 | return fn.apply(null, arguments);
310 | };
311 | });
312 | // use a custom read function
313 | stream_ops.read = function stream_ops_read(
314 | stream: { node: { contents: LazyUint8Array } },
315 | buffer: Uint8Array,
316 | offset: number,
317 | length: number,
318 | position: number
319 | ) {
320 | FS.forceLoadFile(node);
321 |
322 | const contents = stream.node.contents;
323 |
324 | return contents.copyInto(buffer, offset, length, position);
325 | };
326 | node.stream_ops = stream_ops;
327 | return node;
328 | }
329 |
--------------------------------------------------------------------------------
/src/sqlite.worker.ts:
--------------------------------------------------------------------------------
1 | /// <reference path="./types.d.ts" />
2 |
3 | import * as Comlink from "comlink";
4 | import initSqlJs from "../sql.js/dist/sql-wasm.js";
5 | import wasmUrl from "../sql.js/dist/sql-wasm.wasm";
6 | import { createLazyFile, LazyUint8Array, PageReadLog, RangeMapper } from "./lazyFile";
7 | import { Database, QueryExecResult } from "sql.js";
8 | import { SeriesVtab, sqlite3_module, SqljsEmscriptenModuleType } from "./vtab";
9 |
10 | wasmUrl;
11 |
12 | // https://gist.github.com/frankier/4bbc85f65ad3311ca5134fbc744db711
13 | function initTransferHandlers(sql: typeof import("sql.js")) {
14 | Comlink.transferHandlers.set("WORKERSQLPROXIES", {
15 | canHandle: (obj): obj is unknown => {
16 | let isDB = obj instanceof sql.Database;
17 | let hasDB =
18 | obj && (obj as any).db && (obj as any).db instanceof sql.Database; // prepared statements
19 | return isDB || hasDB;
20 | },
21 | serialize(obj) {
22 | const { port1, port2 } = new MessageChannel();
23 | Comlink.expose(obj, port1);
24 | return [port2, [port2]];
25 | },
26 | deserialize: (port: MessagePort) => {},
27 | });
28 | }
29 |
30 | async function init(wasmfile: string) {
31 | const sql = await initSqlJs({
32 | locateFile: (_file: string) => wasmfile,
33 | });
34 | initTransferHandlers(sql);
35 | return sql;
36 | }
37 |
38 | export function toObjects<T>(res: QueryExecResult[]): T[] {
39 | return res.flatMap(r => r.values.map((v) => {
40 | const o: any = {};
41 | for (let i = 0; i < r.columns.length; i++) {
42 | o[r.columns[i]] = v[i];
43 | }
44 | return o as T;
45 | }));
46 | }
47 |
48 | export type SplitFileConfig =
49 | | SplitFileConfigPure
50 | | {
51 | virtualFilename?: string;
52 | from: "jsonconfig";
53 | configUrl: string;
54 | };
55 | export type SplitFileConfigPure = {
56 | virtualFilename?: string;
57 | from: "inline";
58 | config: SplitFileConfigInner;
59 | };
60 | export type SplitFileConfigInner = {
61 | requestChunkSize: number;
62 | cacheBust?: string;
63 | } & (
64 | | {
65 | serverMode: "chunked";
66 | urlPrefix: string;
67 | serverChunkSize: number;
68 | databaseLengthBytes: number;
69 | suffixLength: number;
70 | }
71 | | {
72 | serverMode: "full";
73 | url: string;
74 | }
75 | );
76 | export interface LazyHttpDatabase extends Database {
77 | lazyFiles: Map<string, { contents: LazyUint8Array }>;
78 | filename: string;
79 | query: <T = any>(query: string, ...params: any[]) => T[];
80 | create_vtab: (cons: {
81 | new (sqljs: SqljsEmscriptenModuleType, db: Database): sqlite3_module;
82 | }) => void;
83 | }
84 | export type SqliteStats = {
85 | filename: string;
86 | totalBytes: number;
87 | totalFetchedBytes: number;
88 | totalRequests: number;
89 | };
90 |
91 | async function fetchConfigs(
92 | configsOrUrls: SplitFileConfig[]
93 | ): Promise<SplitFileConfigPure[]> {
94 | const configs = configsOrUrls.map(async (config) => {
95 | if (config.from === "jsonconfig") {
96 | const configUrl = new URL(config.configUrl, location.href);
97 | const req = await fetch(configUrl.toString());
98 |
99 | if (!req.ok) {
100 | console.error("httpvfs config error", await req.text());
101 | throw Error(
102 | `Could not load httpvfs config: ${req.status}: ${req.statusText}`
103 | );
104 | }
105 | const configOut: SplitFileConfigInner = await req.json();
106 | return {
107 | from: "inline",
108 | // resolve url relative to config file
109 | config:
110 | configOut.serverMode === "chunked"
111 | ? {
112 | ...configOut,
113 | urlPrefix: new URL(configOut.urlPrefix, configUrl).toString(),
114 | }
115 | : {
116 | ...configOut,
117 | url: new URL(configOut.url, configUrl).toString(),
118 | },
119 | virtualFilename: config.virtualFilename,
120 | } as SplitFileConfigPure;
121 | } else {
122 | return config;
123 | }
124 | });
125 | return Promise.all(configs);
126 | }
127 | const mod = {
128 | db: null as null | LazyHttpDatabase,
129 | inited: false,
130 | sqljs: null as null | Promise<any>,
131 | bytesRead: 0,
132 | async SplitFileHttpDatabase(
133 | wasmUrl: string,
134 | configs: SplitFileConfig[],
135 | mainVirtualFilename?: string,
136 | maxBytesToRead: number = Infinity,
137 | ): Promise<LazyHttpDatabase> {
138 | if (this.inited) throw Error(`sorry, only one db is supported right now`);
139 | this.inited = true;
140 | if (!this.sqljs) {
141 | this.sqljs = init(wasmUrl);
142 | }
143 | const sql = await this.sqljs;
144 |
145 | this.bytesRead = 0;
146 | let requestLimiter = (bytes: number) => {
147 | if (this.bytesRead + bytes > maxBytesToRead) {
148 | this.bytesRead = 0;
149 | // I couldn't figure out how to get ERRNO_CODES included
150 | // so just hardcode the actual value
151 | // https://github.com/emscripten-core/emscripten/blob/565fb3651ed185078df1a13b8edbcb6b2192f29e/system/include/wasi/api.h#L146
152 | // https://github.com/emscripten-core/emscripten/blob/565fb3651ed185078df1a13b8edbcb6b2192f29e/system/lib/libc/musl/arch/emscripten/bits/errno.h#L13
153 | throw new sql.FS.ErrnoError(6 /* EAGAIN */);
154 | }
155 | this.bytesRead += bytes;
156 | };
157 |
158 | const lazyFiles = new Map();
159 | const hydratedConfigs = await fetchConfigs(configs);
160 | let mainFileConfig;
161 | for (const { config, virtualFilename } of hydratedConfigs) {
162 | const id =
163 | config.serverMode === "chunked" ? config.urlPrefix : config.url;
164 | console.log("constructing url database", id);
165 | let rangeMapper: RangeMapper;
166 | let suffix = config.cacheBust ? "?cb=" + config.cacheBust : "";
167 | if (config.serverMode == "chunked") {
168 | rangeMapper = (from: number, to: number) => {
169 | const serverChunkId = (from / config.serverChunkSize) | 0;
170 | const serverFrom = from % config.serverChunkSize;
171 | const serverTo = serverFrom + (to - from);
172 | return {
173 | url: config.urlPrefix + String(serverChunkId).padStart(config.suffixLength, "0") + suffix,
174 | fromByte: serverFrom,
175 | toByte: serverTo,
176 | };
177 | };
178 | } else {
179 | rangeMapper = (fromByte, toByte) => ({
180 | url: config.url + suffix,
181 | fromByte,
182 | toByte,
183 | });
184 | }
185 |
186 | const filename = virtualFilename || id.replace(/\//g, "_");
187 |
188 | if (!mainVirtualFilename) {
189 | mainVirtualFilename = filename;
190 | mainFileConfig = config
191 | }
192 | console.log("filename", filename);
193 | console.log("constructing url database", id, "filename", filename);
194 | const lazyFile = createLazyFile(sql.FS, "/", filename, true, true, {
195 | rangeMapper,
196 | requestChunkSize: config.requestChunkSize,
197 | fileLength:
198 | config.serverMode === "chunked"
199 | ? config.databaseLengthBytes
200 | : undefined,
201 | logPageReads: true,
202 | maxReadHeads: 3,
203 | requestLimiter
204 | });
205 | lazyFiles.set(filename, lazyFile);
206 | }
207 |
208 | this.db = new sql.CustomDatabase(mainVirtualFilename) as LazyHttpDatabase;
209 | if (mainFileConfig) {
210 | // verify page size and disable cache (since we hold everything in memory anyways)
211 | const pageSizeResp = await this.db.exec("pragma page_size; pragma cache_size=0");
212 | const pageSize = pageSizeResp[0].values[0][0];
213 | if (pageSize !== mainFileConfig.requestChunkSize)
214 | console.warn(
215 | `Chunk size does not match page size: pragma page_size = ${pageSize} but chunkSize = ${mainFileConfig.requestChunkSize}`
216 | );
217 | }
218 |
219 | this.db.lazyFiles = lazyFiles;
220 | this.db.create_vtab(SeriesVtab);
221 | this.db.query = (...args) => toObjects(this.db!.exec(...args));
222 | return this.db!;
223 | },
224 | getResetAccessedPages(virtualFilename?: string): PageReadLog[] {
225 | if (!this.db) return [];
226 | const lazyFile = this.db.lazyFiles.get(virtualFilename || this.db.filename);
227 | if (!lazyFile) throw Error("unknown lazy file");
228 | const pages = [...lazyFile.contents.readPages];
229 | lazyFile.contents.readPages = [];
230 | return pages;
231 | },
232 | getStats(virtualFilename?: string): SqliteStats | null {
233 | const db = this.db;
234 | if (!db) return null;
235 | const lazyFile = db.lazyFiles.get(virtualFilename || db.filename);
236 | if (!lazyFile) throw Error("unknown lazy file");
237 | const res = {
238 | filename: db.filename,
239 | totalBytes: lazyFile.contents.length,
240 | totalFetchedBytes: lazyFile.contents.totalFetchedBytes,
241 | totalRequests: lazyFile.contents.totalRequests,
242 | };
243 | return res;
244 | },
245 | async evalCode(code: string) {
246 | return await eval(`(async function (db) {
247 | ${code}
248 | })`)(this.db);
249 | },
250 | };
251 | export type SqliteComlinkMod = typeof mod;
252 | Comlink.expose(mod);
253 |
--------------------------------------------------------------------------------
/src/types.d.ts:
--------------------------------------------------------------------------------
1 | declare module "*";
2 | declare module "worker-loader!*";
3 |
4 | declare module "*.worker";
--------------------------------------------------------------------------------
/src/util.ts:
--------------------------------------------------------------------------------
1 |
2 | export type SegmentUUID = string & { __segmentUUIDBrand: unknown };
3 | export type VideoID = string & { __videoIDBrand: unknown };
4 | export type VideoDuration = number & { __videoDurationBrand: unknown };
5 | export type Category = string & { __categoryBrand: unknown };
6 | export type VideoIDHash = VideoID;
7 | export type IPAddress = string & { __ipAddressBrand: unknown };
8 | export type HashedIP = IPAddress;
9 | type SBRecord<K extends string, T> = {
10 | [P in string | K]: T;
11 | };
12 | // Uncomment as needed
13 | export enum Service {
14 | YouTube = 'YouTube',
15 | PeerTube = 'PeerTube',
16 | // Twitch = 'Twitch',
17 | // Nebula = 'Nebula',
18 | // RSS = 'RSS',
19 | // Corridor = 'Corridor',
20 | // Lbry = 'Lbry'
21 | }
22 |
23 | export interface IncomingSegment {
24 | category: Category;
25 | segment: string[];
26 | }
27 |
28 | export interface Segment {
29 | category: Category;
30 | segment: number[];
31 | UUID: SegmentUUID;
32 | videoDuration: VideoDuration;
33 | }
34 |
35 | export enum Visibility {
36 | VISIBLE = 0,
37 | HIDDEN = 1
38 | }
39 |
40 | export interface DBSegment {
41 | category: Category;
42 | startTime: number;
43 | endTime: number;
44 | UUID: SegmentUUID;
45 | votes: number;
46 | locked: boolean;
47 | shadowHidden: Visibility;
48 | videoID: VideoID;
49 | videoDuration: VideoDuration;
50 | hashedVideoID: VideoIDHash;
51 | }
52 |
53 | export interface OverlappingSegmentGroup {
54 | segments: DBSegment[],
55 | votes: number;
56 | locked: boolean; // Contains a locked segment
57 | }
58 |
59 | export interface VotableObject {
60 | votes: number;
61 | }
62 |
63 | export interface VotableObjectWithWeight extends VotableObject {
64 | weight: number;
65 | }
66 |
67 | export interface VideoData {
68 | hash: VideoIDHash;
69 | segments: Segment[];
70 | }
71 |
72 | export interface SegmentCache {
73 | shadowHiddenSegmentIPs: SBRecord<VideoID, {hashedIP: HashedIP}[]>,
74 | userHashedIP?: HashedIP
75 | }
76 |
77 |
78 | //gets a weighted random choice from the choices array based on their `votes` property.
79 | //amountOfChoices specifies the maximum amount of choices to return, 1 or more.
80 | //choices are unique
81 | function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOfChoices: number): T[] {
82 | //trivial case: no need to go through the whole process
83 | if (amountOfChoices >= choices.length) {
84 | return choices;
85 | }
86 |
87 | type TWithWeight = T & {
88 | weight: number
89 | }
90 |
91 | //assign a weight to each choice
92 | let totalWeight = 0;
93 | let choicesWithWeights: TWithWeight[] = choices.map(choice => {
94 | //The 3 makes -2 the minimum votes before being ignored completely
95 | //this can be changed if this system increases in popularity.
96 | const weight = Math.exp((choice.votes + 3));
97 | totalWeight += weight;
98 |
99 | return {...choice, weight};
100 | });
101 |
102 | //iterate and find amountOfChoices choices
103 | const chosen = [];
104 | while (amountOfChoices-- > 0) {
105 | //weighted random draw of one element of choices
106 | const randomNumber = Math.random() * totalWeight;
107 | let stackWeight = choicesWithWeights[0].weight;
108 | let i = 0;
109 | while (stackWeight < randomNumber) {
110 | stackWeight += choicesWithWeights[++i].weight;
111 | }
112 |
113 | //add it to the chosen ones and remove it from the choices before the next iteration
114 | chosen.push(choicesWithWeights[i]);
115 | totalWeight -= choicesWithWeights[i].weight;
116 | choicesWithWeights.splice(i, 1);
117 | }
118 |
119 | return chosen;
120 | }
121 |
122 | //This function will find segments that are contained inside of eachother, called similar segments
123 | //Only one similar time will be returned, randomly generated based on the sqrt of votes.
124 | //This allows new less voted items to still sometimes appear to give them a chance at getting votes.
125 | //Segments with less than -1 votes are already ignored before this function is called
126 | export function chooseSegments(segments: DBSegment[]): DBSegment[] {
127 | //Create groups of segments that are similar to eachother
128 | //Segments must be sorted by their startTime so that we can build groups chronologically:
129 | //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
130 | //2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall
131 | // inside that group (because they're sorted) so we can create a new one
132 | const overlappingSegmentsGroups: OverlappingSegmentGroup[] = [];
133 | let currentGroup: OverlappingSegmentGroup;
134 | let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
135 | segments.forEach(segment => {
136 | if (segment.startTime > cursor) {
137 | currentGroup = {segments: [], votes: 0, locked: false};
138 | overlappingSegmentsGroups.push(currentGroup);
139 | }
140 |
141 | currentGroup.segments.push(segment);
142 | //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
143 | if (segment.votes > 0) {
144 | currentGroup.votes += segment.votes;
145 | }
146 |
147 | if (segment.locked) {
148 | currentGroup.locked = true;
149 | }
150 |
151 | cursor = Math.max(cursor, segment.endTime);
152 | });
153 |
154 | overlappingSegmentsGroups.forEach((group) => {
155 | if (group.locked) {
156 | group.segments = group.segments.filter((segment) => segment.locked);
157 | }
158 | });
159 |
160 | //if there are too many groups, find the best 8
161 | return getWeightedRandomChoice(overlappingSegmentsGroups, 32).map(
162 | //randomly choose 1 good segment per group and return them
163 | group => getWeightedRandomChoice(group.segments, 1)[0],
164 | );
165 | }
166 |
--------------------------------------------------------------------------------
/src/vtab.ts:
--------------------------------------------------------------------------------
1 | /// <reference lib="webworker" />
2 | import { Database } from "sql.js";
3 | /*
4 | * This file implements the virtual table that makes interacting with the DOM as a virtual SQLite table possible.
5 | * It is not required at all for the httpvfs functionality.
6 | *
7 | * don't look at it
8 | *
9 | *
10 | *
11 | *
12 | *
13 | *
14 | * plz
15 | *
16 | *
17 | *
18 | * if this is ever to be used for a purpose other than to make people say 'wat'
19 | * it needs to be rewritten
20 | *
21 | */
22 |
23 | // these types are just to make it easier to understand
24 | type Ptr<T> = number;
25 | type int = number;
26 | interface sqlite3_vtab {
27 | pModule: Ptr<sqlite3_module>;
28 | nRef: int;
29 | zErrMsg: Ptr<string>;
30 | }
31 | type SqliteStatus = int;
32 |
33 | interface sqlite3_index_info {}
34 | interface sqlite3_vtab_cursor {
35 | pVtab: Ptr<sqlite3_vtab>;
36 | }
37 | interface sqlite3_context {}
38 | interface sqlite3_value {}
39 | export interface sqlite3_module {
40 | iVersion: int;
41 | xCreate?(
42 | conn: Ptr<"sqliteconn">,
43 | pAux: Ptr<void>,
44 | argc: int,
45 | argv: Ptr<string[]>,
46 | ppVTab: Ptr<sqlite3_vtab>,
47 | pzErr: Ptr<string>
48 | ): SqliteStatus;
49 | xConnect(
50 | conn: Ptr<"sqliteconn">,
51 | pAux: Ptr<void>,
52 | argc: int,
53 | argv: Ptr<string[]>,
54 | ppVTab: Ptr<sqlite3_vtab[]>,
55 | pzErr: Ptr<string[]>
56 | ): SqliteStatus;
57 | xBestIndex(
58 | pVTab: Ptr<sqlite3_vtab>,
59 | sqlite3_index_info: Ptr<sqlite3_index_info>
60 | ): SqliteStatus;
61 | xDisconnect(pVTab: Ptr<sqlite3_vtab>): SqliteStatus;
62 | xDestroy?(pVTab: Ptr<sqlite3_vtab>): SqliteStatus;
63 | xOpen(
64 | pVTab: Ptr<sqlite3_vtab>,
65 | ppCursor: Ptr<sqlite3_vtab_cursor>
66 | ): SqliteStatus;
67 | xClose(sqlite3_vtab_cursor: Ptr<sqlite3_vtab_cursor>): SqliteStatus;
68 | xFilter(
69 | sqlite3_vtab_cursor: Ptr<sqlite3_vtab_cursor>,
70 | idxNum: int,
71 | idxStr: Ptr<string>,
72 | argc: int,
73 | argv: Ptr<sqlite3_value[]>
74 | ): SqliteStatus;
75 | xNext(sqlite3_vtab_cursor: Ptr<sqlite3_vtab_cursor>): SqliteStatus;
76 | xEof(sqlite3_vtab_cursor: Ptr<sqlite3_vtab_cursor>): SqliteStatus;
77 | xColumn(
78 | sqlite3_vtab_cursor: Ptr<sqlite3_vtab_cursor>,
79 | sqlite3_context: Ptr<sqlite3_context[]>,
80 | int: int
81 | ): SqliteStatus;
82 | xRowid(
83 | sqlite3_vtab_cursor: Ptr<sqlite3_vtab_cursor>,
84 | pRowid: Ptr<int>
85 | ): SqliteStatus;
86 | xUpdate?(
87 | vtab: Ptr<sqlite3_vtab>,
88 | argc: int,
89 | argv: Ptr<sqlite3_value[]>,
90 | pRowid: Ptr<int>
91 | ): SqliteStatus;
92 | xBegin?(pVTab: Ptr<sqlite3_vtab>): SqliteStatus;
93 | xSync?(pVTab: Ptr<sqlite3_vtab>): SqliteStatus;
94 | xCommit?(pVTab: Ptr<sqlite3_vtab>): SqliteStatus;
95 | xRollback?(pVTab: Ptr<sqlite3_vtab>): SqliteStatus;
96 | xFindFunction?(
97 | pVtab: Ptr<sqlite3_vtab>,
98 | nArg: int,
99 | zName: Ptr<string>,
100 | pxFunc: Ptr<
101 | (
102 | sqlite3_context: Ptr<sqlite3_context[]>,
103 | argc: int,
104 | argv: Ptr<sqlite3_value[]>
105 | ) => void
106 | >,
107 | ppArg: Ptr<void>
108 | ): SqliteStatus;
109 |
110 | xRename?(pVtab: Ptr<sqlite3_vtab>, zNew: Ptr<string>): SqliteStatus;
111 | xSavepoint?(pVTab: Ptr<sqlite3_vtab>, int: int): SqliteStatus;
112 | xRelease?(pVTab: Ptr<sqlite3_vtab>, int: int): SqliteStatus;
113 | xRollbackTo?(pVTab: Ptr<sqlite3_vtab>, int: int): SqliteStatus;
114 | xShadowName?(str: Ptr<string>): SqliteStatus;
115 | }
116 | /*const seriesVfs: sqlite3_module = {
117 | iVersion: 0,
118 | xConnect()
119 | }
120 | */
121 |
122 | const SQLITE_OK = 0;
123 | const SQLITE_MISUSE = 21;
124 |
125 | // see exported_runtime_methods.json
126 | export interface SqljsEmscriptenModuleType extends EmscriptenModule {
127 | ccall: typeof ccall;
128 | setValue: typeof setValue;
129 | getValue: typeof getValue;
130 | UTF8ToString: typeof UTF8ToString;
131 | stringToUTF8: typeof stringToUTF8;
132 | lengthBytesUTF8: typeof lengthBytesUTF8;
133 | addFunction: typeof addFunction;
134 | extract_value: (ptr: Ptr<sqlite3_value>) => null | string | number;
135 | set_return_value: (
136 | ptr: Ptr<sqlite3_context>,
137 | value: string | number | boolean | null
138 | ) => void;
139 | sqlite3_malloc: (size: int) => Ptr<void>,
140 | }
141 |
142 | type Cursor = {
143 | elements: ArrayLike<Element>;
144 | querySelector: string;
145 | index: number;
146 | };
147 | enum Columns {
148 | idx,
149 | id,
150 | tagName,
151 | textContent,
152 | innerHTML,
153 | outerHTML,
154 | className,
155 | parent,
156 | selector,
157 | querySelector,
158 | }
159 | const columnNames = Object.keys(Columns)
160 | .map((key) => Columns[key as any])
161 | .filter((value) => typeof value === "string");
162 | export interface DomRow {
163 | idx: number;
164 | id: string | null;
165 | tagName: string;
166 | textContent: string;
167 | innerHTML: string;
168 | outerHTML: string;
169 | className: string | null;
170 | parent: string | null;
171 | selector: string;
172 | }
173 | function rowToObject(row: any[]): DomRow {
174 | const out: any = {};
175 | for (let i = 0; i < row.length; i++) {
176 | out[Columns[i]] = row[i];
177 | }
178 | return out;
179 | }
180 | export type MainThreadRequest =
181 | | { type: "select"; selector: string; columns: (keyof DomRow)[] }
182 | | { type: "delete"; selector: string }
183 | | { type: "update"; value: Partial<DomRow> }
184 | | { type: "insert"; value: Partial<DomRow> };
185 | // sends a request to the main thread via postMessage,
186 | // then synchronously waits for the result via a SharedArrayBuffer
187 | function doAsyncRequestToMainThread(request: MainThreadRequest) {
188 | // todo: dynamically adjust this for response size
189 | const sab = new SharedArrayBuffer(1024 * 1024);
190 | // first element is for atomic synchronisation, second element is the length of the response
191 | const metaArray = new Int32Array(sab, 0, 2);
192 | metaArray[0] = 1;
193 | // send message to main thread
194 | (self as DedicatedWorkerGlobalScope).postMessage({
195 | action: "eval",
196 | notify: sab,
197 | request,
198 | });
199 | Atomics.wait(metaArray, 0, 1); // wait until first element is not =1
200 | const dataLength = metaArray[1];
201 | // needs to be copied because textdecoder and encoder is not supported on sharedarraybuffers (for now)
202 | const dataArray = new Uint8Array(sab, 2 * 4, dataLength).slice();
203 | const resStr = new TextDecoder().decode(dataArray);
204 | const res: { err: string } | { ok: any } = JSON.parse(resStr);
205 | if ("err" in res) throw new Error(res.err);
206 | return res.ok;
207 | }
208 | export class SeriesVtab implements sqlite3_module {
209 | name = "dom";
210 | iVersion: number = 2;
211 | cursors = new Map<number, Cursor>();
212 | constructor(private module: SqljsEmscriptenModuleType, private db: Database) {
213 | console.log("constructed vfs");
214 | }
215 | getCursor(cursor: Ptr<sqlite3_vtab_cursor>): Cursor {
216 | const cursorObj = this.cursors.get(cursor);
217 | if (!cursorObj) throw Error("impl error");
218 | return cursorObj;
219 | }
220 | xConnect(
221 | conn: Ptr<"sqliteconn">,
222 | pAux: Ptr<void>,
223 | argc: int,
224 | argv: Ptr<string[]>,
225 | ppVTab: Ptr<Ptr<sqlite3_vtab>>,
226 | pzErr: Ptr<string[]>
227 | ): SqliteStatus {
228 | console.log("xconnect!!");
229 |
230 | const rc = (this.db.handleError as any)(
231 | this.module.ccall(
232 | "sqlite3_declare_vtab",
233 | "number",
234 | ["number", "string"],
235 | [
236 | conn,
237 | `create table x(
238 | ${columnNames.slice(0, -1).join(", ")} PRIMARY KEY
239 | ) WITHOUT ROWID`,
240 | ]
241 | )
242 | );
243 | const out_ptr = this.module._malloc(12);
244 | this.module.setValue(ppVTab, out_ptr, "*");
245 | return SQLITE_OK;
246 | }
247 |
248 | xDisconnect(pVTab: Ptr<sqlite3_vtab>): SqliteStatus {
249 | this.module._free(pVTab);
250 | return SQLITE_OK;
251 | }
252 | xOpen(
253 | pVTab: Ptr<sqlite3_vtab>,
254 | ppCursor: Ptr<Ptr<sqlite3_vtab_cursor>>
255 | ): SqliteStatus {
256 | const cursor = this.module._malloc(4);
257 | // this.module.setValue(series_cursor + 4, cursorId, "i32");
258 | this.cursors.set(cursor, { elements: [], index: 0, querySelector: "" });
259 | this.module.setValue(ppCursor, cursor, "*");
260 | return SQLITE_OK;
261 | }
262 | xClose(sqlite3_vtab_cursor: Ptr<sqlite3_vtab_cursor>): SqliteStatus {
263 | this.module._free(sqlite3_vtab_cursor);
264 | return SQLITE_OK;
265 | }
266 | /*setErrorMessage(cursorPtr: Ptr<sqlite3_vtab_cursor>) {
267 | const vtabPointer: Ptr<sqlite3_vtab> = this.module.getValue(cursorPtr, "i32");
268 | const before = this.module.getValue(vtabPointer + 8, "i32");
269 | console.log("err before", before);
270 | this.module.setValue(vtabPointer + 8, intArrayFromString("FLONKITAL"), "i32");
271 | }*/
272 | xBestIndex(
273 | pVTab: Ptr<sqlite3_vtab>,
274 | info: Ptr<sqlite3_index_info>
275 | ): SqliteStatus {
276 | try {
277 | const nConstraint = this.module.getValue(info + 0, "i32");
278 | const aConstraint = this.module.getValue(info + 4, "i32");
279 |
280 | // const constraint = this.module.getValue(aConstraint, "i32");
281 | // don't care
282 | const SQLITE_INDEX_CONSTRAINT_MATCH = 64;
283 | let haveSelectorMatchConstraint = false;
284 | for (let i = 0; i < nConstraint; i++) {
285 | const sizeofconstraint = 12;
286 | const curConstraint = aConstraint + i * sizeofconstraint;
287 | const iColumn = this.module.getValue(curConstraint, "i32");
288 | const op = this.module.getValue(curConstraint + 4, "i8");
289 | const usable = this.module.getValue(curConstraint + 5, "i8");
290 | if (!usable) continue;
291 | if (op === SQLITE_INDEX_CONSTRAINT_MATCH) {
292 | if (iColumn === Columns.selector) {
293 | // this is the one
294 | haveSelectorMatchConstraint = true;
295 | const aConstraintUsage = this.module.getValue(info + 4 * 4, "i32");
296 | const sizeofconstraintusage = 8;
297 | this.module.setValue(
298 | aConstraintUsage + i * sizeofconstraintusage,
299 | 1,
300 | "i32"
301 | );
302 | } else {
303 | throw Error(`The match operator can only be applied to the selector column!`);
304 | }
305 | }
306 | console.log(`constraint ${i}: ${Columns[iColumn]} (op=${op})`);
307 | }
308 |
309 | if (!haveSelectorMatchConstraint) {
310 | throw Error(
311 | "You must query the dom using `select ... from dom where selector MATCH <css-selector>`"
312 | );
313 | }
314 |
315 | // const aConstraintUsage0 = this.module.getValue(aConstraintUsageArr, "i32");
316 |
317 | const usedColumnsFlag = this.module.getValue(info + 16 * 4, "i32");
318 | this.module.setValue(info + 5 * 4, usedColumnsFlag, "i32"); // just save the used columns instead of an index id
319 | return SQLITE_OK;
320 | } catch (e) {
321 | console.error("xbestindex", e);
322 | this.setVtabError(pVTab, String(e));
323 | return SQLITE_MISUSE;
324 | }
325 | }
326 | xFilter(
327 | cursorPtr: Ptr<sqlite3_vtab_cursor>,
328 | idxNum: int,
329 | idxStr: Ptr<string>,
330 | argc: int,
331 | argv: Ptr<sqlite3_value[]>
332 | ): SqliteStatus {
333 | console.log("xfilter", argc);
334 | if (argc !== 1) {
335 | console.error("did not get a single argument to xFilter");
336 | return SQLITE_MISUSE;
337 | }
338 | const querySelector = this.module.extract_value(argv + 0) as string;
339 | const cursor = this.getCursor(cursorPtr);
340 | // await new Promise(e => setTimeout(e, 1000));
341 | cursor.querySelector = querySelector;
342 | const usedColumnsFlag = idxNum;
343 | const usedColumns = columnNames.filter(
344 | (c) => usedColumnsFlag & (1 << (Columns as any)[c])
345 | ) as (keyof DomRow)[];
346 | console.log("used columns", usedColumns);
347 | cursor.elements = doAsyncRequestToMainThread({
348 | type: "select",
349 | selector: querySelector,
350 | columns: usedColumns,
351 | }); // document.querySelectorAll(str);
352 | // don't filter anything
353 | return SQLITE_OK;
354 | }
355 | xNext(cursorPtr: Ptr<sqlite3_vtab_cursor>): SqliteStatus {
356 | const cursor = this.getCursor(cursorPtr);
357 | cursor.index++;
358 | return SQLITE_OK;
359 | }
360 | xEof(cursorPtr: Ptr<sqlite3_vtab_cursor>): SqliteStatus {
361 | const cursor = this.getCursor(cursorPtr);
362 | return +(cursor.index >= cursor.elements.length);
363 | }
364 | xColumn(
365 | cursorPtr: Ptr<sqlite3_vtab_cursor>,
366 | ctx: Ptr<sqlite3_context[]>,
367 | column: int
368 | ): SqliteStatus {
369 | const cursor = this.getCursor(cursorPtr);
370 | const ele = cursor.elements[cursor.index];
371 | if (Columns[column] in ele) {
372 | this.module.set_return_value(ctx, (ele as any)[Columns[column]]);
373 | } else {
374 | switch (column) {
375 | case Columns.idx: {
376 | this.module.set_return_value(ctx, cursor.index);
377 | break;
378 | }
379 | case Columns.querySelector: {
380 | this.module.set_return_value(ctx, cursor.querySelector);
381 | break;
382 | }
383 | default: {
384 | throw Error(`unknown column ${Columns[column]}`);
385 | }
386 | }
387 | }
388 | return SQLITE_OK;
389 | }
390 | setVtabError(vtab: Ptr<sqlite3_vtab>, err: string) {
391 | const len = this.module.lengthBytesUTF8(err) + 1;
392 | const ptr = this.module.sqlite3_malloc(len);
393 | console.log("writing error", err, len);
394 | this.module.stringToUTF8(err, ptr, len);
395 | this.module.setValue(vtab + 8, ptr, "i32");
396 | }
397 | xUpdate(
398 | vtab: Ptr<sqlite3_vtab>,
399 | argc: int,
400 | argv: Ptr<sqlite3_value[]>,
401 | pRowid: Ptr<int>
402 | ): SqliteStatus {
403 | try {
404 | // https://www.sqlite.org/vtab.html#xupdate
405 | const [oldPrimaryKey, newPrimaryKey, ...args] = Array.from(
406 | { length: argc },
407 | (_, i) => this.module.extract_value(argv + 4 * i)
408 | );
409 | if (!oldPrimaryKey) {
410 | console.assert(newPrimaryKey === null);
411 | // INSERT
412 | doAsyncRequestToMainThread({
413 | type: "insert",
414 | value: rowToObject(args),
415 | });
416 | } else if (oldPrimaryKey && !newPrimaryKey) {
417 | console.log("DELETE", oldPrimaryKey);
418 | doAsyncRequestToMainThread({
419 | type: "delete",
420 | selector: oldPrimaryKey as string,
421 | });
422 | // DELETE
423 | } else {
424 | // UPDATE
425 | if (oldPrimaryKey !== newPrimaryKey) {
426 | throw "The selector row can't be set";
427 | }
428 | doAsyncRequestToMainThread({
429 | type: "update",
430 | value: rowToObject(args),
431 | });
432 | }
433 |
434 | return SQLITE_OK;
435 | } catch (e) {
436 | this.setVtabError(vtab, String(e));
437 | return SQLITE_MISUSE;
438 | }
439 | }
440 |
441 | xRowid(
442 | sqlite3_vtab_cursor: Ptr<sqlite3_vtab_cursor>,
443 | pRowid: Ptr<int>
444 | ): SqliteStatus {
445 | throw Error("xRowid not implemented");
446 | }
447 |
448 | xFindFunction(
449 | pVtab: Ptr<sqlite3_vtab>,
450 | nArg: int,
451 | zName: Ptr<string>,
452 | pxFunc: Ptr<
453 | (
454 | sqlite3_context: Ptr<sqlite3_context>,
455 | argc: int,
456 | argv: Ptr<sqlite3_value[]>
457 | ) => void
458 | >,
459 | ppArg: Ptr<void>
460 | ): SqliteStatus {
461 | const name = this.module.UTF8ToString(zName);
462 | if (name !== "match") {
463 | return SQLITE_OK;
464 | }
465 | const SQLITE_INDEX_CONSTRAINT_FUNCTION = 150;
466 | this.module.setValue(
467 | pxFunc,
468 | this.module.addFunction(
469 | (ctx: Ptr<sqlite3_context>, argc: int, argv: Ptr<sqlite3_value[]>) => {
470 | // always return true since we apply this filter in the xFilter function
471 | this.module.set_return_value(ctx, true);
472 | },
473 | "viii"
474 | ),
475 | "i32"
476 | );
477 | return SQLITE_INDEX_CONSTRAINT_FUNCTION;
478 | }
479 | }
480 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | /* Visit https://aka.ms/tsconfig.json to read more about this file */
4 |
5 | /* Basic Options */
6 | // "incremental": true, /* Enable incremental compilation */
7 | "target": "es2020", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */
8 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
9 | "lib": ["es2020"], /* Specify library files to be included in the compilation. */
10 | // "allowJs": true, /* Allow javascript files to be compiled. */
11 | // "checkJs": true, /* Report errors in .js files. */
12 | "jsx": "react", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
13 | "declaration": true, /* Generates corresponding '.d.ts' file. */
14 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
15 | // "sourceMap": true, /* Generates corresponding '.map' file. */
16 | // "outFile": "./", /* Concatenate and emit output to single file. */
17 | "outDir": "dist", /* Redirect output structure to the directory. */
18 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
19 | // "composite": true, /* Enable project compilation */
20 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
21 | // "removeComments": true, /* Do not emit comments to output. */
22 | // "noEmit": true, /* Do not emit outputs. */
23 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */
24 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
25 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
26 |
27 | /* Strict Type-Checking Options */
28 | "strict": true, /* Enable all strict type-checking options. */
29 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
30 | // "strictNullChecks": true, /* Enable strict null checks. */
31 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */
32 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
33 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
34 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
35 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
36 |
37 | /* Additional Checks */
38 | // "noUnusedLocals": true, /* Report errors on unused locals. */
39 | // "noUnusedParameters": true, /* Report errors on unused parameters. */
40 | "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
41 | "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
42 | // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
43 | // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an 'override' modifier. */
44 | // "noPropertyAccessFromIndexSignature": true, /* Require undeclared properties from index signatures to use element accesses. */
45 |
46 | /* Module Resolution Options */
47 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
48 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
49 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
50 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
51 | // "typeRoots": [], /* List of folders to include type definitions from. */
52 | // "types": [], /* Type declaration files to be included in compilation. */
53 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
54 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
55 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
56 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
57 |
58 | /* Source Map Options */
59 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
60 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
61 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
62 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
63 |
64 | /* Experimental Options */
65 | "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
66 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
67 |
68 | /* Advanced Options */
69 | "skipLibCheck": true, /* Skip type checking of declaration files. */
70 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
71 | },
72 | "exclude": ["example"]
73 | }
74 |
--------------------------------------------------------------------------------
/webpack.config.js:
--------------------------------------------------------------------------------
1 | const path = require("path");
2 | const BundleAnalyzerPlugin = require("webpack-bundle-analyzer")
3 | .BundleAnalyzerPlugin;
4 |
5 | const ts = {
6 | loader: "ts-loader",
7 | options: { transpileOnly: false },
8 | };
9 | module.exports = {
10 | entry: { index: "./src", "sqlite.worker": "./src/sqlite.worker" },
11 | // mode:,
12 | devtool: "source-map",
13 | module: {
14 | noParse: /sql-wasm\.js/,
15 | rules: [
16 | {
17 | test: /\.tsx?$/,
18 | use: ts,
19 | exclude: /node_modules/,
20 | },
21 | { test: /\.wasm$/, type: "asset/resource" },
22 | ],
23 | },
24 | resolve: {
25 | extensions: [".tsx", ".ts", ".js"],
26 | fallback: {
27 | fs: false,
28 | crypto: false,
29 | path: false,
30 | },
31 | },
32 | output: {
33 | filename: "[name].js",
34 | publicPath: "",
35 | path: path.resolve(__dirname, "dist"),
36 | assetModuleFilename: "[name][ext]",
37 | library: {
38 | type: "umd",
39 | },
40 | globalObject: 'this',
41 | },
42 | // target: ['web', 'webworker', 'node'],
43 | stats: {
44 | children: true,
45 | },
46 | devServer: {
47 | publicPath: "/dist",
48 | hot: false,
49 | liveReload: false,
50 | https: true,
51 | headers: {
52 | "Cross-Origin-Opener-Policy": "same-origin",
53 | "Cross-Origin-Embedder-Policy": "require-corp",
54 | },
55 | },
56 | plugins: process.env.analyze ? [new BundleAnalyzerPlugin()] : [],
57 | };
58 |
--------------------------------------------------------------------------------