├── .gitignore ├── LICENSE ├── package-lock.json ├── package.json ├── readme.md ├── sh ├── esbuild-all.sh ├── esbuild-cli-node.sh ├── esbuild-lib-esm.sh ├── esbuild-lib-node.sh ├── push-new-version.mjs └── test-install.sh ├── src ├── cli.ts ├── lib-app │ └── help.txt ├── lib.ts ├── lib │ ├── fns-excel-json.ts │ ├── fns-sqlite-excel.ts │ ├── fns-sqlite-json-official.ts │ ├── fns-sqlite-json.ts │ ├── sql-js.ts │ └── util.ts └── vendor │ ├── readme-sql-js.md │ ├── sql-js-build │ ├── in │ │ └── Makefile │ └── out │ │ ├── sql-wasm-edited.js │ │ └── sql-wasm-original.js │ ├── sql-js-fetch-and-build.sh │ └── sqlite │ ├── readme.md │ └── sqlite3-bundler-friendly-edited.mjs ├── tests ├── 01-lib-excel-json │ └── 01.test.ts ├── 02-cli-excel-json │ └── 02.test.ts ├── 03-lib-sqlite-json │ └── 03.test.ts ├── 04-cli-sqlite-json │ ├── 04.test.ts │ └── wal-mode.sqlite ├── 05-lib-sqlite-excel │ └── 05.test.ts ├── package.json ├── sh │ ├── del-run-tests-bun.sh │ ├── esbuild-test.sh │ └── run-tests-node.sh └── util │ ├── config.ts │ └── util.ts └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | del 2 | notes-private 3 | 4 | src/vendor/sql-js 5 | 6 | tests/**/*test*.js 7 | tests/**/*test*.js.map 8 | 9 | .idea 10 | dist -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 - present, https://github.com/emadda/transform-x. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "transform-x", 3 | "description": "", 4 | "version": "0.0.2", 5 | "main": "dist/lib-node.js", 6 | "module": "dist/lib-esm.js", 7 | "bin": { 8 | "json_to_excel": "dist/cli-node.cjs", 9 | "excel_to_json": "dist/cli-node.cjs", 10 | "json_to_sqlite": "dist/cli-node.cjs", 11 | "sqlite_to_json": "dist/cli-node.cjs", 12 | "sqlite_to_excel": "dist/cli-node.cjs", 13 | "excel_to_sqlite": "dist/cli-node.cjs" 14 | }, 15 | "scripts": { 16 | "prepare": "./sh/esbuild-all.sh" 17 | }, 18 | "devDependencies": { 19 | "@jest/globals": "^29.6.2", 20 | "@types/jest": "^29.5.3", 21 | "jest": "^29.6.2", 22 | "esbuild": "^0.18.17" 23 | }, 24 | "dependencies": { 25 | "lodash": "^4.17.21", 26 | "minimist": "^1.2.8", 27 | "xlsx": "^0.18.5", 28 | "zod": "^3.21.4" 29 | }, 30 | "license": "MIT", 31 | "author": "Enzo", 32 | "keywords": [ 33 | "SQLite", 34 | "Excel", 35 | "XLSX" 36 | ], 37 | "homepage": "https://github.com/emadda/transform-x.git", 38 | "repository": { 39 | "type": "git", 40 | "url": "https://github.com/emadda/transform-x.git" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # transform-x 2 | 3 | JS functions to convert between JSON, SQLite and Excel. 4 | 5 | Use via: 6 | 7 | - [JS library](#use-as-a-js-library) 8 | - [CLI](#clis) 9 | - [Web UI](https://transform-x.dev) 10 | - [HTTP API](https://api.transform-x.dev) 11 | 12 | 13 | Portable: Aims to work in any web standards compatible JS runtime (browser, Node.js, Deno, Bun, Cloudflare Workers). 14 | 15 | 16 | ## Use as a JS library 17 | 18 | `npm install transform-x` 19 | 20 | ```js 21 | // Use with Node.js 22 | const x = require("transform-x"); 23 | x.json_to_sqlite; 24 | 25 | 26 | // Use with vanilla JS ESM (web bundle) 27 | import {json_to_excel, sqlite_to_json} from "transform-x"; 28 | 29 | 30 | // Use with Typescript (web bundle) 31 | import {json_to_excel, sqlite_to_json} from "transform-x/src/lib"; 32 | ``` 33 | 34 | See [tests](./tests) directory for function usage examples. 35 | 36 | ## Web UI 37 | 38 | You can convert between formats in your browser at [transform-x.dev](https://transform-x.dev). 39 | 40 | ## CLI's 41 | 42 | - JSON ↔ Excel 43 | - `json_to_excel` 44 | - `excel_to_json` 45 | 46 | - JSON ↔ SQLite 47 | - `json_to_sqlite` 48 | - `sqlite_to_json` 49 | 50 | - SQLite ↔ Excel 51 | - `sqlite_to_excel` 52 | - `excel_to_sqlite` 53 | 54 | ## Install 55 | 56 | ```bash 57 | # This installs `$x_to_$y` CLI's that can be auto completed in your terminal with tab. 58 | npm install -g transform-x 59 | ``` 60 | 61 | ## JSON ↔ Excel 62 | 63 | ```bash 64 | # Provide both files as args. 65 | json_to_excel --i ./input.json --o ./output.xlsx 66 | 67 | # Use stdin (single sheet) 68 | echo '[{"x": 2}, {"x": 3, "y": {"h": 1}}]' | json_to_excel --o ./output.xlsx 69 | 70 | # Use stdout 71 | json_to_excel --i ./input.json >./output.xlsx 72 | 73 | # Write output to temp file, open in Excel app (macOS only). 74 | echo '[{"x": 2}, {"x": 3, "y": {"h": 1}}]' | json_to_excel --open 75 | 76 | 77 | # Pipe SQLite query result to Excel. 78 | sqlite3 -json db.sqlite "select * from tbl_a" | json_to_excel --open 79 | ``` 80 | 81 | ## JSON ↔ SQLite 82 | 83 | ```bash 84 | # Provide both files as args. 85 | json_to_sqlite --i ./input.json --o ./output.sqlite 86 | 87 | # Use stdin (single table) 88 | echo '[{"x": 2}, {"x": 3, "y": {"h": 1}}]' | json_to_sqlite --o ./output.sqlite 89 | 90 | # Use stdout 91 | json_to_sqlite --i ./input.json >./output.sqlite 92 | 93 | # Write output to temp file, open in native desktop GUI (macOS only). 94 | echo '[{"x": 2}, {"x": 3, "y": {"h": 1}}]' | json_to_sqlite --open 95 | ``` 96 | 97 | ## CLI limits 98 | 99 | - `sqlite_to_json` 100 | - SQLite/JSON limited to around 500MB in size. 101 | - SQLite files are read/written using JS RAM (as WASM is used). 102 | - JSON output is [limited to between 500MB to 1GB in size](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/length#description) as this is the size limit of a JS string. 103 | 104 | - Binary values will be set to null in the output JSON. 105 | 106 | -------------------------------------------------------------------------------- /sh/esbuild-all.sh: -------------------------------------------------------------------------------- 1 | # Build JS bundles for all targets after NPM install. 2 | 3 | # When running via package.json `scripts` after install, the first esbuild run seems to fail. 4 | #./node_modules/.bin/esbuild 5 | 6 | # esbuild-cli-node.sh 7 | ./node_modules/.bin/esbuild ./src/cli.ts --platform=node --bundle --outfile=./dist/cli-node.cjs --loader:.html=text --loader:.wasm=binary --sourcemap --minify 8 | 9 | 10 | # esbuild-lib-esm.sh 11 | ./node_modules/.bin/esbuild ./src/lib.ts --bundle --platform=neutral --outfile=./dist/lib-esm.js --loader:.html=text --loader:.wasm=binary --sourcemap --minify 12 | 13 | 14 | # esbuild-lib-node.sh 15 | ./node_modules/.bin/esbuild ./src/lib.ts --bundle --platform=node --outfile=./dist/lib-node.js --loader:.html=text --loader:.wasm=binary --sourcemap --minify -------------------------------------------------------------------------------- /sh/esbuild-cli-node.sh: -------------------------------------------------------------------------------- 1 | 2 | #./node_modules/.bin/esbuild ./src/core.ts --bundle --outfile=./dist/core.js --minify --watch --sourcemap 3 | 4 | # Note: `.cjs` is needed to avoid the error `ReferenceError: require is not defined in ES module scope, you can use import instead` when using commonjs `require` for Node stdlib fns. 5 | ./node_modules/.bin/esbuild ./src/cli.ts --platform=node --bundle --outfile=./dist/cli-node.cjs --loader:.html=text --loader:.wasm=binary --watch --sourcemap 6 | -------------------------------------------------------------------------------- /sh/esbuild-lib-esm.sh: -------------------------------------------------------------------------------- 1 | # Output a ESM module for inclusion via `import`. 2 | # - For use in browser bundles. 3 | # - Has better dead code elimination via bundlers. 4 | 5 | # --platform=neutral outputs a ESM module for use via `import` 6 | # @see https://stackoverflow.com/a/75328208/4949386 7 | 8 | 9 | # `--main-fields=main` as xlsx uses "main" for Node style `require` usage. 10 | # @see https://esbuild.github.io/api/#platform 11 | 12 | # Compiles with esbuild but throws error in browser due to `dynamic import of Node.js stream module` 13 | #./node_modules/.bin/esbuild ./src/lib.ts --bundle --platform=neutral --main-fields=main --external:fs --external:stream --outfile=./dist/lib-esm.js --loader:.html=text --loader:.wasm=binary --sourcemap --minify 14 | 15 | # Works when using xlsx.mjs instead of Node.js version. 16 | ./node_modules/.bin/esbuild ./src/lib.ts --bundle --platform=neutral --outfile=./dist/lib-esm.js --loader:.html=text --loader:.wasm=binary --sourcemap --minify --watch 17 | 18 | -------------------------------------------------------------------------------- /sh/esbuild-lib-node.sh: -------------------------------------------------------------------------------- 1 | # Output a `require` compatible build for users using Node.js with vanilla JS. 2 | ./node_modules/.bin/esbuild ./src/lib.ts --bundle --platform=node --outfile=./dist/lib-node.js --loader:.html=text --loader:.wasm=binary --sourcemap --minify 3 | -------------------------------------------------------------------------------- /sh/push-new-version.mjs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | // Script to push a new version. 4 | // - Ensures local git tags, remote Github repo and NPM are all in sync. 5 | // - Copies tag from package.json to git tag, pushes to remote. 6 | // - Pushes version to npm. 7 | 8 | import fs from "fs"; 9 | import util from "util"; 10 | import {exec as exec0} from "child_process"; 11 | const exec = util.promisify(exec0); 12 | 13 | const github_repo = `https://github.com/emadda/transform-x`; 14 | 15 | 16 | const run = async () => { 17 | let x; 18 | 19 | x = await exec('git status --short package.json'); 20 | if (/^M/.test(x.stdout.trim())) { 21 | console.error("Error: Must commit package.json as `npm publish` will read the version from the remote Github repo."); 22 | return false; 23 | } 24 | 25 | const package_buffer = fs.readFileSync("package.json"); 26 | const p = JSON.parse(package_buffer.toString()); 27 | const cur_version = p.version; 28 | const new_tag = `v${cur_version}`; 29 | 30 | 31 | // Check the tag is new. 32 | x = await exec('git tag'); 33 | const tags = x.stdout.split("\n").filter(x => /^v\d/.test(x)); 34 | 35 | if (tags.includes(new_tag)) { 36 | console.error(`Error: Version ${cur_version} already exists as a git tag.`); 37 | return false; 38 | } 39 | 40 | 41 | // Create git tag, push to remote. 42 | console.log(`Creating local git tag ${new_tag} and pushing to remote origin.`); 43 | x = await exec(`git tag ${new_tag}`); 44 | x = await exec(`git push -u origin master`); 45 | x = await exec(`git push origin ${new_tag}`); 46 | console.log(x.stdout); 47 | 48 | 49 | // Publish to NPM. 50 | // x = await exec(`npm publish ${github_repo}`); 51 | // console.log(x.stdout); 52 | // console.log("Published to NPM"); 53 | 54 | // Use the remote Github repo to ensure the code is committed and pushed. 55 | // Run this manually to enter the OTP: 56 | console.log(`Now run "npm publish ${github_repo}"`); 57 | // npm publish https://github.com/emadda/transform-x 58 | 59 | return true; 60 | } 61 | 62 | 63 | run(); -------------------------------------------------------------------------------- /sh/test-install.sh: -------------------------------------------------------------------------------- 1 | # Test `npm install --global` 2 | 3 | # Remove prev run. 4 | npm uninstall -g transform-x 5 | 6 | rm -f ./../dist || true 7 | rm -f /tmp/transform-x-0.0.1.tgz || true 8 | 9 | npm pack --pack-destination /tmp 10 | 11 | # Note: 12 | # - `better-sqlite` compiles SQLite from source - takes around 30 seconds on M1. 13 | # - `npm install ./local-dir` does not install `node_modules`, or run the `postinstall` scripts. 14 | npm install --loglevel verbose --global /tmp/transform-x-0.0.1.tgz 15 | 16 | 17 | exa --sort cr --reverse -lha /opt/homebrew/bin | rg "node_modules" | rg "transform-x" 18 | 19 | 20 | #npm root -g 21 | exa -lha --tree --level 2 /opt/homebrew/lib/node_modules/transform-x; -------------------------------------------------------------------------------- /src/cli.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node --enable-source-maps --max-old-space-size=8192 2 | import minimist from "minimist"; 3 | import {z} from "zod"; 4 | import _ from "lodash"; 5 | 6 | import {spawn} from "child_process"; 7 | 8 | import p from "./../package.json"; 9 | import help from "./lib-app/help.txt"; 10 | import fs from "fs"; 11 | import {covert_flat_array_to_nested, excel_to_json, json_to_excel} from "./lib/fns-excel-json.ts"; 12 | import { 13 | convert_flat_to_obj, 14 | json_to_sqlite, 15 | replace_binary_vals_with_null, 16 | sqlite_to_json 17 | } from "./lib/fns-sqlite-json.ts"; 18 | import {log_json} from "./lib/util.ts"; 19 | import {excel_to_sqlite, sqlite_to_excel} from "./lib/fns-sqlite-excel.ts"; 20 | 21 | 22 | const cmd_re = /(?json_to_excel|excel_to_json|sqlite_to_json|json_to_sqlite|sqlite_to_excel|excel_to_sqlite)$/ 23 | const args_s = z.object({ 24 | cmd: z.enum([ 25 | "json_to_excel", 26 | "excel_to_json", 27 | "sqlite_to_json", 28 | "json_to_sqlite", 29 | "sqlite_to_excel", 30 | "excel_to_sqlite", 31 | ]), 32 | i: z.string().nullable().default(null), 33 | o: z.string().nullable().default(null), 34 | config: z.string().nullable().default(null), 35 | open: z.boolean().nullable().default(null) 36 | }); 37 | 38 | 39 | // Example minimist output for when installed via `npm install -g` using package.json `bin` alias. 40 | // _: [ 41 | // '/opt/homebrew/Cellar/node@18/18.16.1_1/bin/node', 42 | // '/opt/homebrew/bin/json_to_excel' 43 | // ] 44 | 45 | const get_args_from_cli = () => { 46 | const cli_args = minimist(process.argv); 47 | let [node, script_name, cmd = null, ...rest] = cli_args._; 48 | 49 | // cmd: Take from script name when installed via `npm -g` if it is not already set. 50 | if (cmd === null) { 51 | const m = script_name.match(cmd_re); 52 | if (m !== null) { 53 | cmd = m.groups.cmd; 54 | } 55 | } 56 | 57 | if (cmd === "help") { 58 | console.log(help); 59 | process.exit(); 60 | } 61 | 62 | 63 | if (cmd === "version") { 64 | console.log(p.version); 65 | process.exit(); 66 | } 67 | 68 | cli_args.cmd = cmd; 69 | const ok = args_s.safeParse(cli_args); 70 | if (!ok.success) { 71 | console.error("CLI args invalid."); 72 | console.error(ok.error.message); 73 | console.log(help); 74 | process.exit(1); 75 | } 76 | 77 | const args_clean = ok.data; 78 | return args_clean; 79 | }; 80 | 81 | const is_uint8ar = (x) => ArrayBuffer.isView(x) && x.constructor === Uint8Array; 82 | 83 | const run = async () => { 84 | const args = get_args_from_cli(); 85 | 86 | let i = null; 87 | if (args.i === null) { 88 | try { 89 | i = fs.readFileSync(process.stdin.fd, null); 90 | } catch (e) { 91 | console.error(`Reading stdin failed (no --i flag passed via CLI for input file path).`); 92 | console.error(e); 93 | console.log(help); 94 | process.exit(1); 95 | } 96 | } else { 97 | try { 98 | i = fs.readFileSync(args.i, null); 99 | } catch (e) { 100 | console.error(`Could not read input file.`); 101 | console.error(e); 102 | console.log(help); 103 | process.exit(1); 104 | } 105 | } 106 | 107 | // Convert Node Buffer to Web Uint8Array. 108 | // - Use standard web API's, avoid Node API's where possible. 109 | const i_as_uint8ar = () => new Uint8Array(i.buffer); 110 | 111 | 112 | // Converting Buffer => Uint8Array => String causes issues (leading \x00's). 113 | const i_as_string = () => { 114 | const str = i.toString(); 115 | return str; 116 | }; 117 | 118 | 119 | let out_bytes = null; 120 | let ext = `.unknown`; 121 | 122 | if (args.cmd === "json_to_excel") { 123 | const workbook_json = JSON.parse(i_as_string()); 124 | const wb = covert_flat_array_to_nested(workbook_json); 125 | 126 | const x = await json_to_excel(wb); 127 | 128 | out_bytes = x.data.xlsx_bytes; 129 | ext = ".xlsx"; 130 | } 131 | 132 | if (args.cmd === "excel_to_json") { 133 | const x = await excel_to_json({ 134 | xlsx_bytes: i_as_uint8ar() 135 | }); 136 | 137 | 138 | const e = new TextEncoder(); 139 | out_bytes = e.encode(JSON.stringify(x.data, null, 4)); 140 | ext = ".json"; 141 | } 142 | 143 | 144 | if (args.cmd === "json_to_sqlite") { 145 | // @todo/low Allow 3 types of json: {tables}, [{name: "t1", rows: []}], [...rows] (for both Excel and SQLite). 146 | // @todo/low Allow an array of rows for a single table. 147 | const db = convert_flat_to_obj(JSON.parse(i_as_string())); 148 | const x = await json_to_sqlite(db); 149 | 150 | out_bytes = x.data.sqlite_bytes; 151 | ext = ".sqlite"; 152 | } 153 | 154 | if (args.cmd === "sqlite_to_json") { 155 | const x = await sqlite_to_json({ 156 | sqlite_bytes: i_as_uint8ar() 157 | }); 158 | 159 | replace_binary_vals_with_null(x.data); 160 | 161 | // @todo/low Parse JSON stringify'd row values if they start with '{' 162 | 163 | const e = new TextEncoder(); 164 | out_bytes = e.encode(JSON.stringify(x.data, null, 4)); 165 | ext = ".json"; 166 | } 167 | 168 | 169 | if (args.cmd === "sqlite_to_excel") { 170 | const x = await sqlite_to_excel({ 171 | sqlite_bytes: i_as_uint8ar() 172 | }); 173 | 174 | out_bytes = x.data.xlsx_bytes; 175 | ext = ".xlsx"; 176 | } 177 | 178 | if (args.cmd === "excel_to_sqlite") { 179 | const x = await excel_to_sqlite({ 180 | xlsx_bytes: i_as_uint8ar() 181 | }); 182 | 183 | out_bytes = x.data.sqlite_bytes; 184 | ext = ".sqlite"; 185 | } 186 | 187 | 188 | // Uint8Array used for wide runtime compatibility (esp browser). 189 | if (!is_uint8ar(out_bytes)) { 190 | throw Error("Invalid out_bytes."); 191 | } 192 | 193 | 194 | // When --open is passed without --o file, use a temp file to allow opening it in its associated program (on Desktop GUI's). 195 | if (args.open === true && args.o === null) { 196 | const time = (new Date().toISOString()).split("T")[1].replace("Z", "").replace(/[^\d]/g, "_"); 197 | args.o = `/tmp/${time}${ext}`; 198 | } 199 | 200 | 201 | if (_.isString(args.o)) { 202 | // File 203 | fs.writeFileSync(args.o, out_bytes); 204 | 205 | if (args.open === true) { 206 | // Assumption: `spawn` does not allow CLI injection in args. 207 | // @todo/low On macOS, force Excel to reload when writing to the same file (it does not re-read the file until the app is closed and reopened). 208 | const x = spawn("open", [args.o], {shell: false}); 209 | } 210 | } else { 211 | // Stdout 212 | // Block until all written to stdout. 213 | // 1 = stdout, 2 = stderr, 3 = stdin 214 | fs.writeSync(1, out_bytes); 215 | fs.fsyncSync(1); 216 | } 217 | 218 | 219 | } 220 | 221 | run(); 222 | 223 | -------------------------------------------------------------------------------- /src/lib-app/help.txt: -------------------------------------------------------------------------------- 1 | transform-x 2 | 3 | A CLI to convert between JSON, SQLite and Excel formats. 4 | 5 | https://github.com/emadda/transform-x 6 | 7 | Commands and flags 8 | 9 | json_to_excel 10 | excel_to_json 11 | 12 | json_to_sqlite 13 | sqlite_to_json 14 | 15 | sqlite_to_excel 16 | excel_to_sqlite 17 | 18 | --i 19 | Input file. 20 | Tries to read from stdin if missing. 21 | 22 | --o 23 | Output file. 24 | Outputs to stdout if missing. 25 | 26 | --config 27 | JSON file for options. 28 | 29 | --open 30 | Open the output file with the associated program. 31 | 32 | help 33 | version -------------------------------------------------------------------------------- /src/lib.ts: -------------------------------------------------------------------------------- 1 | import {sqlite_to_json, json_to_sqlite} from "./lib/fns-sqlite-json"; 2 | import {excel_to_json, json_to_excel} from "./lib/fns-excel-json"; 3 | import {sqlite_to_excel, excel_to_sqlite} from "./lib/fns-sqlite-excel"; 4 | 5 | 6 | export { 7 | sqlite_to_json, 8 | json_to_sqlite, 9 | excel_to_json, 10 | json_to_excel, 11 | sqlite_to_excel, 12 | excel_to_sqlite 13 | } -------------------------------------------------------------------------------- /src/lib/fns-excel-json.ts: -------------------------------------------------------------------------------- 1 | import * as XLSX from 'xlsx/xlsx.mjs'; 2 | import _ from "lodash"; 3 | import {log_json} from "./util.ts"; 4 | 5 | // @todo/med Use zod to validate input types, return proper errors. 6 | // @todo/high Create a real Excel table for each sheet (auto filter and search UI, formulas can reference table alias). 7 | // @see https://docs.sheetjs.com/docs/getting-started/examples/export#reshaping-the-array 8 | const json_to_excel = async (opts) => { 9 | const { 10 | // Can be array or object where keys are table names. 11 | tables 12 | } = opts; 13 | 14 | const wb = XLSX.utils.book_new(); 15 | 16 | for (const t of _.values(tables)) { 17 | if (t.name.length > 31) { 18 | // Avoid "sheet names cannot exceed 31 chars" error. 19 | // @todo/low Create a log for any non-obvious implicit data changes, log to tmp file. 20 | continue; 21 | } 22 | 23 | const ws = XLSX.utils.json_to_sheet(t.rows); 24 | XLSX.utils.book_append_sheet(wb, ws, t.name); 25 | } 26 | 27 | const file = XLSX.writeXLSX(wb, {type: "array", compression: true}); 28 | 29 | 30 | return { 31 | ok: true, 32 | data: { 33 | xlsx_bytes: new Uint8Array(file) 34 | } 35 | } 36 | }; 37 | 38 | // Try to infer many different types of JSON structure into the single type allowed for `json_to_excel` 39 | const covert_flat_array_to_nested = (obj) => { 40 | const is_flat_array_of_objects = ( 41 | _.isArray(obj) && 42 | obj.filter(x => !_.isPlainObject(x)).length === 0 43 | ); 44 | 45 | if (is_flat_array_of_objects) { 46 | return { 47 | tables: [ 48 | { 49 | name: "sheet_1", 50 | rows: obj 51 | } 52 | ] 53 | } 54 | } 55 | 56 | if (_.isPlainObject(obj)) { 57 | return obj; 58 | } 59 | 60 | throw Error(`Unknown JSON input structure: ${JSON.stringify(obj)}`); 61 | } 62 | 63 | 64 | // @todo/med Convert Excel dates to ISO strings. And JS/ISO strings to Excel dates. 65 | const excel_to_json = async (opts) => { 66 | const { 67 | xlsx_bytes, 68 | 69 | // @todo/low Allow no headers (use A, B etc as keys). 70 | first_row_are_headers = true 71 | } = opts; 72 | 73 | 74 | const workbook = XLSX.read(xlsx_bytes, {type: "array"}); 75 | const tables = {}; 76 | for (const [k, v] of _.toPairs(workbook.Sheets)) { 77 | tables[k] = { 78 | name: k, 79 | 80 | // Note: Array of arrays is returned when header=1 81 | rows: XLSX.utils.sheet_to_json( 82 | v, 83 | { 84 | header: 0, 85 | // Fill out empty vals with this value (default is to leave out the key from the object). 86 | defval: null 87 | } 88 | ) 89 | } 90 | } 91 | 92 | return { 93 | ok: true, 94 | data: { 95 | tables 96 | } 97 | } 98 | } 99 | 100 | 101 | export { 102 | json_to_excel, 103 | excel_to_json, 104 | 105 | // Util 106 | covert_flat_array_to_nested 107 | } -------------------------------------------------------------------------------- /src/lib/fns-sqlite-excel.ts: -------------------------------------------------------------------------------- 1 | import * as XLSX from 'xlsx'; 2 | import _ from "lodash"; 3 | import {log_json} from "./util.ts"; 4 | import {json_to_sqlite, sqlite_to_json} from "./fns-sqlite-json.ts"; 5 | import {excel_to_json, json_to_excel} from "./fns-excel-json.ts"; 6 | 7 | 8 | const sqlite_to_excel = async (opts) => { 9 | const { 10 | sqlite_bytes 11 | } = opts; 12 | 13 | 14 | const res = await sqlite_to_json({sqlite_bytes}); 15 | const res_2 = await json_to_excel(res.data); 16 | 17 | return { 18 | ok: true, 19 | data: { 20 | xlsx_bytes: res_2.data.xlsx_bytes 21 | } 22 | } 23 | } 24 | 25 | const excel_to_sqlite = async (opts) => { 26 | const { 27 | xlsx_bytes 28 | } = opts; 29 | 30 | const res = await excel_to_json({xlsx_bytes}); 31 | const res_2 = await json_to_sqlite(res.data); 32 | 33 | return { 34 | ok: true, 35 | data: { 36 | sqlite_bytes: res_2.data.sqlite_bytes 37 | } 38 | } 39 | } 40 | 41 | 42 | export { 43 | sqlite_to_excel, 44 | excel_to_sqlite 45 | } -------------------------------------------------------------------------------- /src/lib/fns-sqlite-json-official.ts: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | 3 | // Note: The official SQLite WASM build does not support wal-mode files. 4 | // - This is the first implementation before trying sql-js instead. 5 | 6 | import { 7 | default as sqlite3InitModuleWasmBundled 8 | } from './../vendor/sqlite/sqlite3-bundler-friendly-edited.mjs'; 9 | 10 | 11 | const sqlite_to_json = async (opts) => { 12 | const { 13 | sqlite_bytes 14 | } = opts; 15 | 16 | const db = await get_db_oo_from_arraybuffer(sqlite_bytes); 17 | 18 | const rows = db.exec({ 19 | sql: `SELECT name FROM sqlite_schema WHERE type='table' AND name != 'sqlite_sequence' ORDER BY name`, 20 | returnValue: "resultRows", 21 | rowMode: "object" 22 | }); 23 | 24 | const tables = {}; 25 | for (const t of rows) { 26 | const rows = db.exec({ 27 | sql: `SELECT * FROM ${t.name}`, 28 | returnValue: "resultRows", 29 | rowMode: "object" 30 | }); 31 | 32 | // Use a map as it is easier to read with dot notation. 33 | tables[t.name] = { 34 | name: t.name, 35 | rows 36 | }; 37 | } 38 | 39 | // log_json(tables.t1); 40 | 41 | return { 42 | ok: true, 43 | data: {tables} 44 | } 45 | }; 46 | 47 | // Convert an array buffer into a db handle. 48 | const get_db_oo_from_arraybuffer = async (uint8) => { 49 | // @see https://sqlite.org/wasm/doc/trunk/cookbook.md 50 | const sqlite3 = await get_sqlite_3(); 51 | 52 | // Note: Docs mention array buffer, but only a uint8array works. 53 | const p = sqlite3.wasm.allocFromTypedArray(uint8); 54 | 55 | const db = new sqlite3.oo1.DB(); 56 | 57 | // Can also write to OPFS and use filename: 58 | // sqlite3.capi.sqlite3_js_vfs_create_file("opfs", "my-db.db", arrayBuffer); 59 | // const db = new sqlite3.oo1.OpfsDb("my-db.db"); 60 | 61 | 62 | const rc = sqlite3.capi.sqlite3_deserialize( 63 | db.pointer, 'main', p, uint8.byteLength, uint8.byteLength, 64 | sqlite3.capi.SQLITE_DESERIALIZE_FREEONCLOSE 65 | // Optionally: 66 | // | sqlite3.capi.SQLITE_DESERIALIZE_RESIZEABLE 67 | ); 68 | 69 | console.log({rc, a: 1}); 70 | sqlite3.oo1.DB.checkRc(db, rc); 71 | // db.checkRc(rc); 72 | 73 | return db; 74 | }; 75 | 76 | 77 | const json_to_sqlite = async (opts) => { 78 | const { 79 | // Input tables do not use a map to reduce ceremony for tools creating input for this function (no need to place name as map key and in table object key). 80 | db: db_data 81 | } = opts; 82 | 83 | const sqlite3 = await get_sqlite_3(); 84 | 85 | const db_meta = json_to_statements_and_params(db_data); 86 | // log_json(db_meta); 87 | // log_json(db_data); 88 | 89 | 90 | const db = new sqlite3.oo1.DB(`:memory:`, 'cw'); 91 | 92 | // Create tables 93 | for (const t of db_meta.tables) { 94 | db.exec({ 95 | sql: t.sql.create_table 96 | }); 97 | } 98 | 99 | // Insert rows 100 | for (const t of db_meta.tables) { 101 | const data = db_data.tables.find(x => x.name === t.name); 102 | if (data === undefined) { 103 | throw Error("Missing table"); 104 | } 105 | 106 | const i = db.prepare(t.sql.insert_into); 107 | 108 | for (const r of data.rows) { 109 | const binds = obj_to_params(r, t.sql.insert_valid_keys); 110 | 111 | // log_json(binds); 112 | i.bind(binds).step(); 113 | i.reset(); 114 | } 115 | 116 | i.finalize(); 117 | } 118 | const sqlite_bytes = sqlite3.capi.sqlite3_js_db_export(db.pointer) 119 | 120 | return { 121 | ok: true, 122 | data: { 123 | sqlite_bytes 124 | } 125 | } 126 | }; 127 | 128 | 129 | // @see https://emscripten.org/docs/api_reference/module.html#Module.instantiateWasm 130 | let sqlite3_p = null; 131 | const get_sqlite_3 = async () => { 132 | if (sqlite3_p === null) { 133 | const log = (...args) => console.log(...args); 134 | const error = (...args) => console.error(...args); 135 | 136 | sqlite3_p = sqlite3InitModuleWasmBundled({ 137 | print: log, 138 | printErr: error, 139 | // wasmBinary: sqlite_wasm_binary 140 | // instantiateWasm: function (imports, successCallback) { 141 | // return WebAssembly.instantiate(sqlite_wasm_binary, imports).then(function (output) { 142 | // successCallback(output.instance); 143 | // }); 144 | // } 145 | 146 | }); 147 | } 148 | return sqlite3_p; 149 | }; 150 | 151 | 152 | // Converts a JSON object into params/bindings to run against a statement. 153 | // - Must match prepared statement bindings and provide only valid value types. 154 | const obj_to_params = (r, only_keys) => { 155 | let binds = {}; 156 | for (const k of only_keys) { 157 | binds[k] = null; 158 | if (k in r) { 159 | const v = r[k]; 160 | binds[k] = v; 161 | 162 | if (_.isPlainObject(v) || _.isArray(v)) { 163 | binds[k] = JSON.stringify(v); 164 | } 165 | 166 | if (_.isDate(v)) { 167 | binds[k] = v.toISOString(); 168 | } 169 | } 170 | } 171 | 172 | binds = prefix_dollar(binds); 173 | return binds; 174 | } 175 | 176 | 177 | const is_valid_sql_identifier = (s) => /^[^\d][a-z\d_-]*$/i.test(s); 178 | 179 | 180 | // const get_db_as_blob = (sqlite3, db_pointer) => { 181 | // const byteArray = sqlite3.capi.sqlite3_js_db_export(db_pointer); 182 | // const blob = new Blob([byteArray.buffer], {type: "application/x-sqlite3"}); 183 | // return blob; 184 | // } 185 | 186 | 187 | const json_to_statements_and_params = (db) => { 188 | // @todo/low Validate table and col names. 189 | 190 | 191 | const db_meta = { 192 | tables: [] 193 | }; 194 | 195 | // Determine column types for create table statement. 196 | for (const tbl of db.tables) { 197 | 198 | if (!is_valid_sql_identifier(tbl.name)) { 199 | continue; 200 | } 201 | 202 | const cols = {}; 203 | for (const row of tbl.rows) { 204 | for (const [col, val] of _.toPairs(row)) { 205 | const sql_type = get_sql_type(val); 206 | if (sql_type === null) { 207 | // Unsupported type. 208 | continue; 209 | } 210 | if (!is_valid_sql_identifier(col)) { 211 | continue; 212 | } 213 | 214 | 215 | if (!(col in cols)) { 216 | cols[col] = { 217 | sql_types: {} 218 | }; 219 | } 220 | 221 | const meta = cols[col]; 222 | 223 | 224 | if (!(sql_type in meta.sql_types)) { 225 | meta.sql_types[sql_type] = 0; 226 | } 227 | meta.sql_types[sql_type]++; 228 | 229 | } 230 | 231 | } 232 | 233 | 234 | db_meta.tables.push({ 235 | name: tbl.name, 236 | cols, 237 | sql: cols_to_create_and_insert(tbl.name, cols) 238 | }); 239 | } 240 | 241 | return db_meta; 242 | }; 243 | 244 | const cols_to_create_and_insert = (table_name, cols) => { 245 | const x = []; 246 | for (const [col, meta] of _.toPairs(cols)) { 247 | const sql_types = _.keys(meta.sql_types); 248 | 249 | if (sql_types.length > 1 || sql_types.includes("NULL")) { 250 | x.push({col, sql_type: "TEXT"}); 251 | continue; 252 | } 253 | 254 | x.push({ 255 | col, 256 | sql_type: sql_types[0] 257 | }); 258 | } 259 | 260 | const create_table = `create table ${table_name}(_id INTEGER PRIMARY KEY AUTOINCREMENT, ${x.map(s => `${s.col} ${s.sql_type}`).join(", ")})`; 261 | const insert_into = `insert into ${table_name}(${x.map(s => s.col).join(", ")}) values (${x.map(s => `$${s.col}`).join(", ")})`; 262 | 263 | return { 264 | create_table, 265 | insert_into, 266 | insert_valid_keys: x.map(s => s.col) 267 | } 268 | }; 269 | 270 | const get_sql_type = (val) => { 271 | if (_.isString(val) || _.isDate(val)) { 272 | return "TEXT"; 273 | } 274 | if (val === null || val === undefined) { 275 | return "NULL"; 276 | } 277 | 278 | // JS Float without after decimal data. 279 | // SQLite uses 1 and 0 for booleans. 280 | if (Number.isInteger(val) || _.isBoolean(val)) { 281 | return "INTEGER"; 282 | } 283 | 284 | // Float. 285 | if (_.isNumber(val)) { 286 | return "REAL"; 287 | } 288 | 289 | // ArrayBuffer view = (Uint8Array and family). 290 | if (val instanceof ArrayBuffer || ArrayBuffer.isView(val)) { 291 | return "BLOB" 292 | } 293 | 294 | if (_.isPlainObject(val) || _.isArray(val)) { 295 | // JSON string. 296 | return "TEXT"; 297 | } 298 | 299 | // Unsupported type. 300 | return null; 301 | } 302 | 303 | 304 | // Prefix dollar for binding to SQL statement. 305 | const prefix_dollar = (obj) => { 306 | const o = {}; 307 | 308 | for (const [k, v] of Object.entries(obj)) { 309 | o[`$${k}`] = v; 310 | } 311 | 312 | return o; 313 | } 314 | 315 | 316 | const del_other_sqlite3_wasm_load_attempts = () => { 317 | // await sqlite3InitModule_vanilla({ 318 | // print: log, 319 | // printErr: error, 320 | // wasmBinary: sqlite_wasm_binary, 321 | // instantiateWasm: function (imports, successCallback) { 322 | // return WebAssembly.instantiate(sqlite_wasm_binary, imports).then(function (output) { 323 | // successCallback(output.instance); 324 | // }); 325 | // } 326 | // }).then((sqlite3) => { 327 | // try { 328 | // log('Done initializing. Running demo...'); 329 | // start(sqlite3); 330 | // } catch (err) { 331 | // // error(err.name, err.message); 332 | // } 333 | // }); 334 | 335 | 336 | // @see https://github.com/sql-js/sql.js/issues/554 337 | // const SQL = await initSqlJs({ 338 | // // Required to load the wasm binary asynchronously. Of course, you can host it wherever you want 339 | // // You can omit locateFile completely when running in node 340 | // // locateFile: file => `https://sql.js.org/dist/${file}` 341 | // wasmBinary: null, 342 | // 343 | // // locateFile: (file) => { 344 | // // // console.log({file}); 345 | // // if(file === "sql-wasm.wasm") { 346 | // // // Does not work (only files and http?). 347 | // // // return URL.createObjectURL(new Blob([sqlite_wasm_binary_2.buffer], {type: "application/wasm"})); 348 | // // 349 | // // // https://stackoverflow.com/questions/12710001/how-to-convert-uint8-array-to-base64-encoded-string 350 | // // // - Note there is a browser and a node version. 351 | // // 352 | // // const b64 = Buffer.from(sqlite_wasm_binary_2).toString('base64'); 353 | // // 354 | // // // const base64Data = btoa(String.fromCharCode.apply(null, sqlite_wasm_binary_2)); 355 | // // return `data:application/wasm;base64,${b64}`; 356 | // // 357 | // // } 358 | // // throw Error(`Unknown file ${file}`); 359 | // // } 360 | // }); 361 | // 362 | // const db_2 = new SQL.Database(); 363 | } 364 | 365 | 366 | export { 367 | sqlite_to_json, 368 | json_to_sqlite, 369 | } -------------------------------------------------------------------------------- /src/lib/fns-sqlite-json.ts: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | 3 | // import initSqlJs from './../vendor/sql.js/dist/sql-wasm'; 4 | import {exec_read, exec_write, get_sql_js} from "./sql-js.ts"; 5 | import {log_json} from "./util.ts"; 6 | 7 | // import initSqlJs_NPM from "sql.js"; 8 | // import sqlite_wasm_binary_2 from "./../../node_modules/sql.js/dist/sql-wasm.wasm" 9 | 10 | 11 | const sqlite_to_json = async (opts) => { 12 | const { 13 | // Must be uint8array, not arraybuffer. 14 | sqlite_bytes 15 | } = opts; 16 | 17 | const SQL_JS = await get_sql_js(); 18 | const db = new SQL_JS.Database(sqlite_bytes); 19 | 20 | 21 | // const rows = db.exec(`SELECT name FROM sqlite_schema WHERE type='table' AND name != 'sqlite_sequence' ORDER BY name`); 22 | const table_names = exec_read(db, `SELECT name FROM sqlite_schema WHERE type='table' AND name != 'sqlite_sequence' ORDER BY name`); 23 | 24 | 25 | const tables = {}; 26 | for (const t of table_names) { 27 | const rows = exec_read(db, `SELECT * FROM ${t.name}`); 28 | 29 | // Use a map as it is easier to read with dot notation. 30 | tables[t.name] = { 31 | name: t.name, 32 | rows 33 | }; 34 | } 35 | 36 | return { 37 | ok: true, 38 | data: {tables} 39 | } 40 | }; 41 | 42 | 43 | const json_to_sqlite = async (opts) => { 44 | const { 45 | // Input tables do not use a map to reduce ceremony for tools creating input for this function (no need to place name as map key and in table object key). 46 | tables 47 | } = opts; 48 | 49 | // from 50 | const db_meta = json_to_statements_and_params({tables}); 51 | 52 | // to 53 | const SQL_JS = await get_sql_js(); 54 | const db = new SQL_JS.Database(); 55 | 56 | // Create tables 57 | for (const t of db_meta.tables) { 58 | try { 59 | db.exec(t.sql.create_table); 60 | } catch (e) { 61 | console.error(t.sql.create_table); 62 | throw e; 63 | } 64 | } 65 | 66 | const tables_iter = _.values(tables); 67 | 68 | // Insert rows 69 | for (const t of db_meta.tables) { 70 | const data = tables_iter.find(x => x.name === t.name); 71 | 72 | if (data === undefined) { 73 | throw Error("Missing table"); 74 | } 75 | 76 | // When: Empty table. No JSON to derive SQL schema from. 77 | if (t.sql.insert_into === null) { 78 | continue; 79 | } 80 | 81 | let stmt = null; 82 | try { 83 | stmt = db.prepare(t.sql.insert_into); 84 | } catch (e) { 85 | console.error(t.sql.insert_into); 86 | throw e; 87 | } 88 | 89 | 90 | for (const r of data.rows) { 91 | const binds = obj_to_params(r, t.sql.insert_valid_keys); 92 | 93 | // log_json(binds); 94 | stmt.bind(binds); 95 | stmt.step(); 96 | stmt.reset(); 97 | } 98 | 99 | stmt.free(); 100 | } 101 | const sqlite_bytes = db.export(); 102 | 103 | 104 | return { 105 | ok: true, 106 | data: { 107 | sqlite_bytes 108 | } 109 | } 110 | }; 111 | 112 | 113 | // Converts a JSON object into params/bindings to run against a statement. 114 | // - Must match prepared statement bindings and provide only valid value types. 115 | const obj_to_params = (r, only_keys) => { 116 | let binds = {}; 117 | for (const k of only_keys) { 118 | binds[k] = null; 119 | if (k in r) { 120 | const v = r[k]; 121 | binds[k] = v; 122 | 123 | if (_.isPlainObject(v) || _.isArray(v)) { 124 | binds[k] = JSON.stringify(v); 125 | } 126 | 127 | if (_.isDate(v)) { 128 | binds[k] = v.toISOString(); 129 | } 130 | } 131 | } 132 | 133 | binds = prefix_dollar(binds); 134 | return binds; 135 | } 136 | 137 | 138 | const is_valid_sql_identifier = (s) => /^[^\d][a-z\d_-]*$/i.test(s); 139 | 140 | 141 | const json_to_statements_and_params = (db) => { 142 | // @todo/low Validate table and col names. 143 | 144 | 145 | const db_meta = { 146 | tables: [] 147 | }; 148 | 149 | // Determine column types for create table statement. 150 | for (const tbl of _.values(db.tables)) { 151 | 152 | if (!is_valid_sql_identifier(tbl.name)) { 153 | continue; 154 | } 155 | 156 | const cols = {}; 157 | for (const row of tbl.rows) { 158 | for (const [col, val] of _.toPairs(row)) { 159 | const sql_type = get_sql_type(val); 160 | if (sql_type === null) { 161 | // Unsupported type. 162 | continue; 163 | } 164 | if (!is_valid_sql_identifier(col)) { 165 | continue; 166 | } 167 | 168 | 169 | if (!(col in cols)) { 170 | cols[col] = { 171 | sql_types: {} 172 | }; 173 | } 174 | 175 | const meta = cols[col]; 176 | 177 | 178 | if (!(sql_type in meta.sql_types)) { 179 | meta.sql_types[sql_type] = 0; 180 | } 181 | meta.sql_types[sql_type]++; 182 | 183 | } 184 | 185 | } 186 | 187 | 188 | db_meta.tables.push({ 189 | name: tbl.name, 190 | cols, 191 | sql: cols_to_create_and_insert(tbl.name, cols) 192 | }); 193 | } 194 | 195 | return db_meta; 196 | }; 197 | 198 | const cols_to_create_and_insert = (table_name, cols) => { 199 | const x = []; 200 | for (const [col, meta] of _.toPairs(cols)) { 201 | const sql_types = _.keys(meta.sql_types); 202 | 203 | // X === X 204 | if (sql_types.length === 1 && !sql_types.includes("NULL")) { 205 | x.push({col, sql_type: sql_types[0]}); 206 | continue; 207 | } 208 | 209 | // NULL === TEXT 210 | if (_.isEqual(sql_types, ["NULL"])) { 211 | x.push({col, sql_type: "TEXT"}); 212 | continue; 213 | } 214 | 215 | // NULL AND X === X 216 | if (sql_types.length === 2 && sql_types.includes("NULL")) { 217 | x.push({col, sql_type: sql_types.find(n => n !== "NULL")}); 218 | continue; 219 | } 220 | 221 | // When: multiple types exist. 222 | // ELSE TEXT. 223 | x.push({ 224 | col, 225 | sql_type: "TEXT" 226 | }); 227 | } 228 | 229 | const meta_data_cols = [`_x_id`]; 230 | 231 | // When looping between JSON and SQLite, do not re-write private cols, just delete/re-create them. 232 | // `_x` prefix used to reduce the chance of a real col collision. 233 | const writable = x.filter(n => !meta_data_cols.includes(n.col)); 234 | 235 | // Note: This can be empty when input JSON rows are empty, as the create table defs are derived from the actual JSON values (when empty it is impossible to know what the intended schema is, but an empty table may convey information so still create it). 236 | const col_defs = writable.map(s => `${s.col} ${s.sql_type}`).join(", "); 237 | const col_refs = writable.map(s => s.col).join(", "); 238 | const col_binds = writable.map(s => `$${s.col}`).join(", "); 239 | 240 | const create_table = `create table ${table_name}(_x_id INTEGER PRIMARY KEY AUTOINCREMENT ${writable.length > 0 ? `,` : ``} ${col_defs})`; 241 | 242 | let insert_into = null; 243 | if (writable.length > 0) { 244 | insert_into = `insert into ${table_name}(${col_refs}) values (${col_binds})`; 245 | } 246 | 247 | return { 248 | create_table, 249 | insert_into, 250 | insert_valid_keys: x.map(s => s.col) 251 | } 252 | }; 253 | 254 | const get_sql_type = (val) => { 255 | if (_.isString(val) || _.isDate(val)) { 256 | return "TEXT"; 257 | } 258 | if (val === null || val === undefined) { 259 | return "NULL"; 260 | } 261 | 262 | // JS Float without after decimal data. 263 | // SQLite uses 1 and 0 for booleans. 264 | if (Number.isInteger(val) || _.isBoolean(val)) { 265 | return "INTEGER"; 266 | } 267 | 268 | // Float. 269 | if (_.isNumber(val)) { 270 | return "REAL"; 271 | } 272 | 273 | // ArrayBuffer view = (Uint8Array and family). 274 | if (val instanceof ArrayBuffer || ArrayBuffer.isView(val)) { 275 | return "BLOB" 276 | } 277 | 278 | if (_.isPlainObject(val) || _.isArray(val)) { 279 | // JSON string. 280 | return "TEXT"; 281 | } 282 | 283 | // Unsupported type. 284 | return null; 285 | } 286 | 287 | 288 | // Prefix dollar for binding to SQL statement. 289 | const prefix_dollar = (obj) => { 290 | const o = {}; 291 | 292 | for (const [k, v] of Object.entries(obj)) { 293 | o[`$${k}`] = v; 294 | } 295 | 296 | return o; 297 | } 298 | 299 | 300 | // Using esbuild to bundle the wasm and then returning the bytes in `locateFile` does not work as emscripten only supports that in SINGLE_FILE=1 mode, the default is 0 for sql-js. 301 | // const del_other_sqlite3_wasm_load_attempts = async () => { 302 | // // await sqlite3InitModule_vanilla({ 303 | // // print: log, 304 | // // printErr: error, 305 | // // wasmBinary: sqlite_wasm_binary, 306 | // // instantiateWasm: function (imports, successCallback) { 307 | // // return WebAssembly.instantiate(sqlite_wasm_binary, imports).then(function (output) { 308 | // // successCallback(output.instance); 309 | // // }); 310 | // // } 311 | // // }).then((sqlite3) => { 312 | // // try { 313 | // // log('Done initializing. Running demo...'); 314 | // // start(sqlite3); 315 | // // } catch (err) { 316 | // // // error(err.name, err.message); 317 | // // } 318 | // // }); 319 | // 320 | // 321 | // // @see https://github.com/sql-js/sql.js/issues/554 322 | // const SQL = await initSqlJs_NPM({ 323 | // // Required to load the wasm binary asynchronously. Of course, you can host it wherever you want 324 | // // You can omit locateFile completely when running in node 325 | // // locateFile: file => `https://sql.js.org/dist/${file}` 326 | // // wasmBinary: null, 327 | // 328 | // locateFile: (file) => { 329 | // // console.log({file}); 330 | // if (file === "sql-wasm.wasm") { 331 | // // Does not work (only files and http?). 332 | // // return URL.createObjectURL(new Blob([sqlite_wasm_binary_2.buffer], {type: "application/wasm"})); 333 | // 334 | // // https://stackoverflow.com/questions/12710001/how-to-convert-uint8-array-to-base64-encoded-string 335 | // // - Note there is a browser and a node version. 336 | // 337 | // const b64 = Buffer.from(sqlite_wasm_binary_2).toString('base64'); 338 | // 339 | // // const base64Data = btoa(String.fromCharCode.apply(null, sqlite_wasm_binary_2)); 340 | // // return `data:application/wasm;base64,${b64}`; 341 | // 342 | // 343 | // return `data:application/octet-stream;base64,${b64}`; 344 | // 345 | // } 346 | // throw Error(`Unknown file ${file}`); 347 | // } 348 | // }); 349 | // 350 | // const db_2 = new SQL.Database(); 351 | // } 352 | 353 | 354 | // Try to infer many different types of JSON structure into the single type allowed for `json_to_excel` 355 | const convert_flat_to_obj = (obj) => { 356 | const is_flat_array_of_objects = ( 357 | _.isArray(obj) && 358 | obj.filter(x => !_.isPlainObject(x)).length === 0 359 | ); 360 | 361 | if (is_flat_array_of_objects) { 362 | return { 363 | tables: [ 364 | { 365 | name: "tbl_1", 366 | rows: obj 367 | } 368 | ] 369 | } 370 | } 371 | 372 | if (_.isPlainObject(obj)) { 373 | return obj; 374 | } 375 | 376 | throw Error(`Unknown JSON input structure: ${JSON.stringify(obj)}`); 377 | } 378 | 379 | // Replace binary row values with null before JSON.stringify for CLI. 380 | // @todo/med Support x_to_msgpack output. 381 | const replace_binary_vals_with_null = (db) => { 382 | for (const t of _.values(db.tables)) { 383 | for (const r of t.rows) { 384 | for (const [k, v] of _.toPairs(r)) { 385 | if (ArrayBuffer.isView(v)) { 386 | r[k] = null; 387 | } 388 | } 389 | } 390 | } 391 | } 392 | 393 | 394 | export { 395 | sqlite_to_json, 396 | json_to_sqlite, 397 | convert_flat_to_obj, 398 | replace_binary_vals_with_null 399 | } -------------------------------------------------------------------------------- /src/lib/sql-js.ts: -------------------------------------------------------------------------------- 1 | import initSqlJs from './../vendor/sql-js-build/out/sql-wasm-edited'; 2 | import {log_json} from "./util.ts"; 3 | 4 | 5 | // Init once globally per JS runtime. 6 | let SQL_JS = null; 7 | const get_sql_js = async () => { 8 | if (SQL_JS === null) { 9 | SQL_JS = await initSqlJs({}); 10 | } 11 | return SQL_JS; 12 | }; 13 | 14 | // Note: There is no option for "as object" on `db.exec`. 15 | // [{ columns: [ 'name' ], values: [ [ 't1' ] ] }] to [{name: "t1"}] 16 | const rows_to_obj = (rows) => { 17 | const o = []; 18 | 19 | for (const row of rows.values) { 20 | const r = {}; 21 | for (const [i, k] of rows.columns.entries()) { 22 | r[k] = row[i] 23 | } 24 | 25 | o.push(r); 26 | } 27 | return o; 28 | } 29 | 30 | 31 | // Execute a single read (SELECT) and return rows. 32 | const exec_read = (db, sql, params) => { 33 | // Note: exec returns any array of result sets, on for each statement in sql, separated by semicolon. 34 | const res = db.exec(sql, params); 35 | 36 | // When: No rows, but table exists. 37 | if (res.length === 0) { 38 | return []; 39 | } 40 | 41 | return rows_to_obj(res[0]); 42 | }; 43 | 44 | // const exec_write = (db, sql, params) => { 45 | // const res = db.exec(sql, params); 46 | // 47 | // return { 48 | // rows_modified: db.getRowsModified() 49 | // } 50 | // }; 51 | 52 | export { 53 | get_sql_js, 54 | exec_read, 55 | // exec_write 56 | } -------------------------------------------------------------------------------- /src/lib/util.ts: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | 3 | const is_dev = () => { 4 | // process.env["NODE_ENV"] 5 | // - Bun replaces this with "development" by default. 6 | // - "production" cannot be set via shebang or via package.json `bin` value. 7 | 8 | return false; 9 | } 10 | 11 | 12 | const sleep = async (ms) => { 13 | return new Promise((resolve) => setTimeout(resolve, ms)); 14 | }; 15 | 16 | 17 | // Force Node.js to print out all JSON values instead of excluding them at depth > 2. 18 | const log_json = (x) => { 19 | // const util = require('util') 20 | // console.log(util.inspect(x, {showHidden: false, depth: null, colors: true})) 21 | console.dir(x, {depth: 100}); 22 | } 23 | 24 | const alpha = "abcdefghijklmnopqrstuvwxyz"; 25 | const get_random_alpha_id = (len) => { 26 | const id = []; 27 | for (let i = 0; i < len; i++) { 28 | id.push(_.sample(alpha)); 29 | } 30 | return id.join(""); 31 | } 32 | 33 | export { 34 | is_dev, 35 | sleep, 36 | log_json, 37 | get_random_alpha_id 38 | } -------------------------------------------------------------------------------- /src/vendor/readme-sql-js.md: -------------------------------------------------------------------------------- 1 | # Notes 2 | 3 | - This repo vendors in `sql.js` to enable Emscriptens `emcc SINGLE_FILE=1` mode. 4 | - This embeds the .wasm file into the JS file. 5 | - This is important as the default is to retrieve it at runtime (via download or reading from the file system with Node.js) 6 | - This forces deploying the .wasm alongside the JS bundle for web, or using Node.js. 7 | - Bundling the .wasm file allows using it from any JS runtime. 8 | - E.g Cloudflare Workers, Deno, Bun etc. 9 | - It allows using a single `import` to use any of the functions without having to load a .wasm outside the bundler process. 10 | - To enable web: 11 | - `emcc ENVIRONMENT=web` 12 | - Prevents using `require("fs")` Node import which breaks esbuild with `--platform=neutral` (ESM module). 13 | - `emcc --closure 0` 14 | - Must be set with the above otherwise Closure Compiler will rewrite the `columns` key on an object which prevents it being red. 15 | - Assumption: `web` typically works in Node.js too as its just pure JS (only uses v8 built in functions or web standard API's). 16 | 17 | 18 | - The official SQLite WASM build does not support WAL mode files. 19 | - See https://sqlite.org/forum/forumpost/e0b0c56e09 20 | - Note: It is possible to clear bytes 18/19 to change a db file from wal-mode to non-wal-mode. 21 | 22 | - See https://github.com/emscripten-core/emscripten/issues/20025 23 | - `locateFile` should allow `Uint8Array` and data URL. 24 | - But it does not, so recompiling with SINGLE_FILE=1 is needed. 25 | 26 | 27 | 28 | - Cloudflare Workers. 29 | - Some edits to the emscripted JS/WASM code were needed. 30 | - See comment on `sql-wasm-edited.js` 31 | - Diff the `edited` with `original` to see changes. 32 | - Steps. 33 | - 1. Extract the WASM binary from base64 embedded string into .wasm file, place into CF worker by importing the .wasm file. 34 | - 2. Replace `{credentials: "same-origin"}` to avoid `Error: The 'credentials' field on 'RequestInitializerDict' is not implemented.` 35 | - This is a bug in the CF runtime. 36 | - 3. Avoid `WebAssembly.instantiate(wasm_binary)` as this is not allowed on CF workers as it is a form of code generation. 37 | - Use `WebAssembly.instantiate(wasm_module)`, where the `wasm_module` comes from `import wasm_module from "wasm_binary.wasm"` in the CF worker. 38 | 39 | 40 | ## Build issues 41 | 42 | - `.devcontainer` does not work on Mac M1 (linux/arm64). 43 | - This is a vscode editor plugin that mounts the source directory into a Docker container. 44 | - Both the Dockerfile FROM and its dependencies installed during build require x86 (linux/amd64). 45 | 46 | - `brew install emscripten` does not work. 47 | - The build process relies on an older emscripten (3.1.20) 48 | - `brew` does not support installing older versions. 49 | 50 | 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /src/vendor/sql-js-build/in/Makefile: -------------------------------------------------------------------------------- 1 | # Note: Last built with version 2.0.15 of Emscripten 2 | 3 | # TODO: Emit a file showing which version of emcc and SQLite was used to compile the emitted output. 4 | # TODO: Create a release on Github with these compiled assets rather than checking them in 5 | # TODO: Consider creating different files based on browser vs module usage: https://github.com/vuejs/vue/tree/dev/dist 6 | 7 | # I got this handy makefile syntax from : https://github.com/mandel59/sqlite-wasm (MIT License) Credited in LICENSE 8 | # To use another version of Sqlite, visit https://www.sqlite.org/download.html and copy the appropriate values here: 9 | SQLITE_AMALGAMATION = sqlite-amalgamation-3390300 10 | SQLITE_AMALGAMATION_ZIP_URL = https://www.sqlite.org/2022/sqlite-amalgamation-3390300.zip 11 | SQLITE_AMALGAMATION_ZIP_SHA3 = 6a83b7da4b73d7148364a0033632ae1e4f9d647417e6f3654a5d0afe8424bbb9 12 | 13 | # Note that extension-functions.c hasn't been updated since 2010-02-06, so likely doesn't need to be updated 14 | EXTENSION_FUNCTIONS = extension-functions.c 15 | EXTENSION_FUNCTIONS_URL = https://www.sqlite.org/contrib/download/extension-functions.c?get=25 16 | EXTENSION_FUNCTIONS_SHA1 = c68fa706d6d9ff98608044c00212473f9c14892f 17 | 18 | EMCC=emcc 19 | 20 | SQLITE_COMPILATION_FLAGS = \ 21 | -Oz \ 22 | -DSQLITE_OMIT_LOAD_EXTENSION \ 23 | -DSQLITE_DISABLE_LFS \ 24 | -DSQLITE_ENABLE_FTS3 \ 25 | -DSQLITE_ENABLE_FTS3_PARENTHESIS \ 26 | -DSQLITE_ENABLE_FTS5 \ 27 | -DSQLITE_THREADSAFE=0 \ 28 | -DSQLITE_ENABLE_NORMALIZE 29 | 30 | # When compiling to WASM, enabling memory-growth is not expected to make much of an impact, so we enable it for all builds 31 | # Since tihs is a library and not a standalone executable, we don't want to catch unhandled Node process exceptions 32 | # So, we do : `NODEJS_CATCH_EXIT=0`, which fixes issue: https://github.com/sql-js/sql.js/issues/173 and https://github.com/sql-js/sql.js/issues/262 33 | # Note: `ENVIRONMENT=web` prevents outputting Node.js `require` which breaks esbuild bundling for the web. 34 | EMFLAGS = \ 35 | --memory-init-file 0 \ 36 | -s RESERVED_FUNCTION_POINTERS=64 \ 37 | -s ALLOW_TABLE_GROWTH=1 \ 38 | -s EXPORTED_FUNCTIONS=@src/exported_functions.json \ 39 | -s EXPORTED_RUNTIME_METHODS=@src/exported_runtime_methods.json \ 40 | -s SINGLE_FILE=1 \ 41 | -s NODEJS_CATCH_EXIT=0 \ 42 | -s NODEJS_CATCH_REJECTION=0 \ 43 | -s ENVIRONMENT=web 44 | 45 | EMFLAGS_ASM = \ 46 | -s WASM=0 47 | 48 | EMFLAGS_ASM_MEMORY_GROWTH = \ 49 | -s WASM=0 \ 50 | -s ALLOW_MEMORY_GROWTH=1 51 | 52 | EMFLAGS_WASM = \ 53 | -s WASM=1 \ 54 | -s ALLOW_MEMORY_GROWTH=1 55 | 56 | # Note: `--closure 0` prevents JS minify which rewrites important object keys (E.g. `columns`). 57 | EMFLAGS_OPTIMIZED= \ 58 | -Oz \ 59 | -flto \ 60 | --closure 0 61 | 62 | EMFLAGS_DEBUG = \ 63 | -s ASSERTIONS=1 \ 64 | -O1 65 | 66 | BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc 67 | 68 | OUTPUT_WRAPPER_FILES = src/shell-pre.js src/shell-post.js 69 | 70 | SOURCE_API_FILES = src/api.js 71 | 72 | EMFLAGS_PRE_JS_FILES = \ 73 | --pre-js src/api.js 74 | 75 | EXPORTED_METHODS_JSON_FILES = src/exported_functions.json src/exported_runtime_methods.json 76 | 77 | all: optimized debug worker 78 | 79 | .PHONY: debug 80 | debug: dist/sql-asm-debug.js dist/sql-wasm-debug.js 81 | 82 | dist/sql-asm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) 83 | $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ 84 | mv $@ out/tmp-raw.js 85 | cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ 86 | rm out/tmp-raw.js 87 | 88 | dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) 89 | $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ 90 | mv $@ out/tmp-raw.js 91 | cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ 92 | rm out/tmp-raw.js 93 | 94 | .PHONY: optimized 95 | optimized: dist/sql-asm.js dist/sql-wasm.js dist/sql-asm-memory-growth.js 96 | 97 | dist/sql-asm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) 98 | $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ 99 | mv $@ out/tmp-raw.js 100 | cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ 101 | rm out/tmp-raw.js 102 | 103 | dist/sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) 104 | $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ 105 | mv $@ out/tmp-raw.js 106 | cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ 107 | rm out/tmp-raw.js 108 | 109 | dist/sql-asm-memory-growth.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) 110 | $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ 111 | mv $@ out/tmp-raw.js 112 | cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ 113 | rm out/tmp-raw.js 114 | 115 | # Web worker API 116 | .PHONY: worker 117 | worker: dist/worker.sql-asm.js dist/worker.sql-asm-debug.js dist/worker.sql-wasm.js dist/worker.sql-wasm-debug.js 118 | 119 | dist/worker.sql-asm.js: dist/sql-asm.js src/worker.js 120 | cat $^ > $@ 121 | 122 | dist/worker.sql-asm-debug.js: dist/sql-asm-debug.js src/worker.js 123 | cat $^ > $@ 124 | 125 | dist/worker.sql-wasm.js: dist/sql-wasm.js src/worker.js 126 | cat $^ > $@ 127 | 128 | dist/worker.sql-wasm-debug.js: dist/sql-wasm-debug.js src/worker.js 129 | cat $^ > $@ 130 | 131 | # Building it this way gets us a wrapper that _knows_ it's in worker mode, which is nice. 132 | # However, since we can't tell emcc that we don't need the wasm generated, and just want the wrapper, we have to pay to have the .wasm generated 133 | # even though we would have already generated it with our sql-wasm.js target above. 134 | # This would be made easier if this is implemented: https://github.com/emscripten-core/emscripten/issues/8506 135 | # dist/worker.sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) src/api.js src/worker.js $(EXPORTED_METHODS_JSON_FILES) dist/sql-wasm-debug.wasm 136 | # $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) -s ENVIRONMENT=worker -s $(EMFLAGS_WASM) $(BITCODE_FILES) --pre-js src/api.js -o out/sql-wasm.js 137 | # mv out/sql-wasm.js out/tmp-raw.js 138 | # cat src/shell-pre.js out/tmp-raw.js src/shell-post.js src/worker.js > $@ 139 | # #mv out/sql-wasm.wasm dist/sql-wasm.wasm 140 | # rm out/tmp-raw.js 141 | 142 | # dist/worker.sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) src/api.js src/worker.js $(EXPORTED_METHODS_JSON_FILES) dist/sql-wasm-debug.wasm 143 | # $(EMCC) -s ENVIRONMENT=worker $(EMFLAGS) $(EMFLAGS_DEBUG) -s ENVIRONMENT=worker -s WASM_BINARY_FILE=sql-wasm-foo.debug $(EMFLAGS_WASM) $(BITCODE_FILES) --pre-js src/api.js -o out/sql-wasm-debug.js 144 | # mv out/sql-wasm-debug.js out/tmp-raw.js 145 | # cat src/shell-pre.js out/tmp-raw.js src/shell-post.js src/worker.js > $@ 146 | # #mv out/sql-wasm-debug.wasm dist/sql-wasm-debug.wasm 147 | # rm out/tmp-raw.js 148 | 149 | out/sqlite3.bc: sqlite-src/$(SQLITE_AMALGAMATION) 150 | mkdir -p out 151 | # Generate llvm bitcode 152 | $(EMCC) $(SQLITE_COMPILATION_FLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/sqlite3.c -o $@ 153 | 154 | # Since the extension-functions.c includes other headers in the sqlite_amalgamation, we declare that this depends on more than just extension-functions.c 155 | out/extension-functions.bc: sqlite-src/$(SQLITE_AMALGAMATION) 156 | mkdir -p out 157 | # Generate llvm bitcode 158 | $(EMCC) $(SQLITE_COMPILATION_FLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@ 159 | 160 | # TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder 161 | # module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js 162 | # tar --create --gzip $^ > $@ 163 | 164 | ## cache 165 | cache/$(SQLITE_AMALGAMATION).zip: 166 | mkdir -p cache 167 | curl -LsSf '$(SQLITE_AMALGAMATION_ZIP_URL)' -o $@ 168 | 169 | cache/$(EXTENSION_FUNCTIONS): 170 | mkdir -p cache 171 | curl -LsSf '$(EXTENSION_FUNCTIONS_URL)' -o $@ 172 | 173 | ## sqlite-src 174 | .PHONY: sqlite-src 175 | sqlite-src: sqlite-src/$(SQLITE_AMALGAMATION) sqlite-src/$(SQLITE_AMALGAMATION)/$(EXTENSION_FUNCTIONS) 176 | 177 | sqlite-src/$(SQLITE_AMALGAMATION): cache/$(SQLITE_AMALGAMATION).zip sqlite-src/$(SQLITE_AMALGAMATION)/$(EXTENSION_FUNCTIONS) 178 | mkdir -p sqlite-src/$(SQLITE_AMALGAMATION) 179 | echo '$(SQLITE_AMALGAMATION_ZIP_SHA3) ./cache/$(SQLITE_AMALGAMATION).zip' > cache/check.txt 180 | sha3sum -a 256 -c cache/check.txt 181 | # We don't delete the sqlite_amalgamation folder. That's a job for clean 182 | # Also, the extension functions get copied here, and if we get the order of these steps wrong, 183 | # this step could remove the extension functions, and that's not what we want 184 | unzip -u 'cache/$(SQLITE_AMALGAMATION).zip' -d sqlite-src/ 185 | touch $@ 186 | 187 | sqlite-src/$(SQLITE_AMALGAMATION)/$(EXTENSION_FUNCTIONS): cache/$(EXTENSION_FUNCTIONS) 188 | mkdir -p sqlite-src/$(SQLITE_AMALGAMATION) 189 | echo '$(EXTENSION_FUNCTIONS_SHA1) ./cache/$(EXTENSION_FUNCTIONS)' > cache/check.txt 190 | sha1sum -c cache/check.txt 191 | cp 'cache/$(EXTENSION_FUNCTIONS)' $@ 192 | 193 | 194 | .PHONY: clean 195 | clean: 196 | rm -f out/* dist/* cache/* 197 | rm -rf sqlite-src/ 198 | -------------------------------------------------------------------------------- /src/vendor/sql-js-fetch-and-build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This builds sql-js using emscripten with the SINGLE_FILE=1 argument (include wasm into a single JS file). 4 | # - Including the .wasm file in the JS cannot be done any other way as emscripten does not support it. 5 | # @see readme-sql-js.md 6 | # @see # https://github.com/sql-js/sql.js 7 | 8 | # Install emscripten 9 | # @see https://emscripten.org/docs/getting_started/downloads.html 10 | 11 | # May install needed dependencies. 12 | #brew install emscripten 13 | #brew uninstall emscripten 14 | 15 | #git clone https://github.com/emscripten-core/emsdk.git 16 | #./emsdk install 3.1.20 17 | #./emsdk activate 3.1.20 18 | # Append PATH edit to shell config. 19 | 20 | # Build 21 | rm -rf sql-js 22 | git clone https://github.com/sql-js/sql.js sql-js 23 | cd sql-js || exit 24 | git checkout v1.8.0 25 | 26 | # Makefile changes: Set SINGLE_FILE=1, add FTS5 module. 27 | rm Makefile 28 | cp ./../sql-js-build/in/Makefile ./Makefile 29 | 30 | make 31 | 32 | cp dist/sql-wasm.js ./../sql-js-build/out/sql-wasm.js 33 | 34 | 35 | # rm -rf dist/* && make && cp dist/sql-wasm.js ./../sql-js-build/out 36 | # - To rebuild after a Makefile edit without re-running this script. -------------------------------------------------------------------------------- /src/vendor/sqlite/readme.md: -------------------------------------------------------------------------------- 1 | 2 | ## Including SQLite WASM. 3 | 4 | - An issue with the official SQLite WASM release (https://www.npmjs.com/package/@sqlite.org/sqlite-wasm) is that it expects to load the sqlite3.wasm file asynchronously at runtime (by using fetch in a web browser, or a file read with Node.js). 5 | - It purposely does not expose emscripten config args as these may change - cannot be used to customize the wasm loading to read from Uint8Array. 6 | - The emscripten build creates both the wasm and the JS in a single process so they cannot be easily edited. 7 | - Emscripten single file mode resulted in WASM that is too large - a post process WASM minify process can be run on the standalone wasm file. 8 | 9 | - `transform-x` functions aim to run in any JS runtime with Uint8Array and WebAssembly API's. 10 | - Example runtimes 11 | - Web 12 | - Node.js (testing and CLI) 13 | - Deno/Bun 14 | - Cloudflare Workers 15 | - etc. 16 | 17 | - To achieve this, resolving and including the sqlite3.wasm file is done at compile time using esbuild with a binary file loader. 18 | - This includes the binary file contents as base64 and decodes it to a Uint8Array at runtime. 19 | - This makes the SQLite library behave as a regular .ts module (no need to deploy .wasm files alongside the bundle that are read at runtime). 20 | - This removes the need for different wasm loading logic for each runtime (just use esbuild), but forces library users to use esbuild for its custom binary loader. 21 | 22 | ## Editing 23 | 24 | - `sqlite3-bundler-friendly-edited` is copied from `"./../../../node_modules//@sqlite.org/sqlite-wasm/sqlite-wasm/jswasm" 25 | - `import x from "x.wasm" is added`. When using esbuilds binary loader this becomes Uint8Array. 26 | - createObjectUrl is used to convert the Uint8Array to a URL which is used in the emscripten fetch logic. 27 | - This adds around 2MB to the final JS bundle. 28 | 29 | 30 | 31 | - See 32 | - https://sqlite.org/forum/forumpost/60144fde3d -------------------------------------------------------------------------------- /tests/01-lib-excel-json/01.test.ts: -------------------------------------------------------------------------------- 1 | // import test from 'node:test'; 2 | import _ from "lodash"; 3 | import {excel_to_json, json_to_excel} from "../../src/lib/fns-excel-json.ts"; 4 | import assert from "assert"; 5 | import {util} from "zod"; 6 | import assertEqual = util.assertEqual; 7 | 8 | 9 | const test_json_to_excel = async () => { 10 | { 11 | const t = [ 12 | {"a": 1, "b": 1}, 13 | {"a": 2, "b": 2} 14 | ]; 15 | 16 | const t_reverse = [...t].reverse(); 17 | 18 | const a = await json_to_excel({ 19 | tables: [ 20 | { 21 | name: "sheet_1", 22 | rows: t 23 | }, 24 | { 25 | name: "sheet_2", 26 | rows: t_reverse 27 | } 28 | ] 29 | }); 30 | 31 | 32 | const b = await excel_to_json({ 33 | xlsx_bytes: a.data.xlsx_bytes 34 | }); 35 | 36 | 37 | assertEqual(b.ok, true); 38 | 39 | 40 | assert.deepEqual(t, b.data.tables.sheet_1.rows); 41 | assert.deepEqual(t_reverse, b.data.tables.sheet_2.rows); 42 | } 43 | }; 44 | 45 | // @todo/low Handle two columns with the same header key. 46 | // @todo/low Convert between JS dates and Excel dates. 47 | describe('LIB: json_to_excel, excel_to_json', () => { 48 | test('test_json_to_excel', async () => test_json_to_excel()); 49 | }); 50 | 51 | -------------------------------------------------------------------------------- /tests/02-cli-excel-json/02.test.ts: -------------------------------------------------------------------------------- 1 | // import test from 'node:test'; 2 | import assert from "assert"; 3 | 4 | import fs from "fs"; 5 | import util from "util"; 6 | import {exec as exec0} from "child_process"; 7 | 8 | const exec = util.promisify(exec0); 9 | 10 | import p from "./../../package.json"; 11 | import {log_json} from "../../src/lib/util.ts"; 12 | 13 | const cli = `./../dist/cli-node.cjs` 14 | 15 | 16 | const run = async () => { 17 | let x; 18 | 19 | x = await exec(`${cli} version`); 20 | assert.equal(p.version, x.stdout.trim()); 21 | 22 | 23 | const dir = `/tmp/transform-x-tests/${new Date().toISOString().replace(/[TZ]/g, "").replace(/[^\d]/g, "_")}` 24 | x = await exec(`mkdir -p ${dir}`); 25 | 26 | // CLI takes both flat array of objects and nested to work well with other CLI's. 27 | const flat = [ 28 | {a: 1}, 29 | {a: 2} 30 | ]; 31 | 32 | const nested = { 33 | tables: [ 34 | { 35 | name: "sheet_1", 36 | rows: flat 37 | }, 38 | { 39 | name: "sheet_2", 40 | rows: flat 41 | } 42 | ] 43 | }; 44 | 45 | fs.writeFileSync(`${dir}/a.json`, JSON.stringify(flat)); 46 | fs.writeFileSync(`${dir}/a-nested.json`, JSON.stringify(nested)); 47 | 48 | 49 | x = await exec(`${cli} json_to_excel --i ${dir}/a.json --o ${dir}/b.xlsx`); 50 | x = await exec(`${cli} json_to_excel --i ${dir}/a-nested.json --o ${dir}/b-nested.xlsx`); 51 | 52 | 53 | x = await exec(`${cli} excel_to_json --i ${dir}/b.xlsx --o ${dir}/c.json`); 54 | x = await exec(`${cli} excel_to_json --i ${dir}/b-nested.xlsx --o ${dir}/c-nested.json`); 55 | 56 | 57 | 58 | const c_flat = JSON.parse(fs.readFileSync(`${dir}/c.json`, `utf-8`)); 59 | const c_nested = JSON.parse(fs.readFileSync(`${dir}/c-nested.json`, `utf-8`)); 60 | 61 | assert.deepEqual(flat, c_flat.tables.sheet_1.rows); 62 | assert.deepEqual(flat, c_nested.tables.sheet_2.rows); 63 | 64 | 65 | // Error: ENOTSUP: operation not supported on socket, fsync 66 | // x = await exec(`${cli} excel_to_json --i ${dir}/b.xlsx`); 67 | // const b = x.stdout; 68 | // assert.deepEqual(a, b); 69 | } 70 | 71 | 72 | test('CLI: json_to_excel, excel_to_json', async () => { 73 | await run(); 74 | }); 75 | -------------------------------------------------------------------------------- /tests/03-lib-sqlite-json/03.test.ts: -------------------------------------------------------------------------------- 1 | // import test from 'node:test'; 2 | 3 | import _ from "lodash"; 4 | import assert from "assert"; 5 | import {util} from "zod"; 6 | 7 | 8 | import {json_to_sqlite, sqlite_to_json} from "../../src/lib/fns-sqlite-json.ts"; 9 | import fs from "fs"; 10 | import {setup_tests} from "../util/util.ts"; 11 | import {log_json} from "../../src/lib/util.ts"; 12 | 13 | 14 | setup_tests(); 15 | 16 | const wal_mode_files_can_be_read = async () => { 17 | // WAL-mode is not supported in WASM, not even for reading. 18 | // @see https://sqlite.org/forum/forumpost/33587a48dc 19 | 20 | // Error for wal enabled SQLite files: `SQLite3Error: sqlite3 result code 26: file is not a database`. 21 | // - Check wal mode status with `sqlite3 x.sqlite ".dbinfo"` 22 | const buffer = fs.readFileSync(`./04-cli-sqlite-json/wal-mode.sqlite`, null); 23 | const wal_mode_sqlite_bytes = new Uint8Array(buffer.buffer); 24 | 25 | const c = await sqlite_to_json({ 26 | sqlite_bytes: wal_mode_sqlite_bytes 27 | }); 28 | expect(c?.data?.tables).toEqual({ 29 | t1: {name: 't1', rows: [{c1: 'v1'}]}, 30 | t2: {name: 't2', rows: []} 31 | }); 32 | } 33 | 34 | 35 | const json_to_sqlite_to_json = async () => { 36 | 37 | const enc = new TextEncoder(); 38 | 39 | const db_1 = { 40 | tables: [ 41 | { 42 | name: "t1", 43 | rows: [ 44 | { 45 | t_int: 1, 46 | t_string: "r1", 47 | t_bool: true, 48 | t_null: null, 49 | uint8array: enc.encode("This is a string converted to a Uint8Array, encoded as utf-8"), 50 | nested: { 51 | a: 1, 52 | b: "This should be converted to JSON" 53 | }, 54 | date: new Date(), 55 | "dsfasdf@$^*&^%$": 1 56 | }, 57 | { 58 | new_col_a: 1, 59 | new_col_b: 2 60 | }, 61 | ] 62 | }, 63 | { 64 | name: "t2", 65 | rows: [ 66 | { 67 | t_int: 1, 68 | t_string: "r1", 69 | t_bool: true, 70 | t_null: null 71 | } 72 | ] 73 | } 74 | ] 75 | }; 76 | 77 | 78 | const a = await json_to_sqlite(db_1); 79 | 80 | 81 | expect(a.ok).toBe(true); 82 | 83 | const file_path = "/tmp/tfx-last-test-run.sqlite"; 84 | fs.writeFileSync(file_path, a.data.sqlite_bytes); 85 | console.log(`SQLite test db written to ${file_path}`); 86 | 87 | 88 | const b = await sqlite_to_json({ 89 | sqlite_bytes: a.data.sqlite_bytes 90 | }); 91 | 92 | 93 | expect(b.ok).toBe(true); 94 | 95 | expect(b.data?.tables?.t1.rows[0]).toEqual({ 96 | _x_id: 1, 97 | t_int: 1, 98 | t_string: 'r1', 99 | t_bool: 1, 100 | t_null: null, 101 | uint8array: db_1.tables[0].rows[0].uint8array, 102 | nested: '{"a":1,"b":"This should be converted to JSON"}', 103 | date: db_1.tables[0].rows[0].date.toISOString(), 104 | 105 | // These are added to every row. 106 | new_col_a: null, 107 | new_col_b: null 108 | }); 109 | }; 110 | 111 | // @todo/low Offer msgpack alternative for JSON with Uint8Array binary values. 112 | // @todo/low Assert FTS5 database can be read. 113 | // @todo/low Detect boolean type by asserting only 0 and 1 exist in the col. 114 | describe('LIB: sqlite_to_json, json_to_sqlite', () => { 115 | test('wal_mode_files_can_be_read', async () => wal_mode_files_can_be_read()); 116 | test('json_to_sqlite_to_json', async () => json_to_sqlite_to_json()); 117 | }); 118 | 119 | -------------------------------------------------------------------------------- /tests/04-cli-sqlite-json/04.test.ts: -------------------------------------------------------------------------------- 1 | // import test from 'node:test'; 2 | import assert from "assert"; 3 | 4 | import fs from "fs"; 5 | import util from "util"; 6 | import {exec as exec0} from "child_process"; 7 | 8 | const exec = util.promisify(exec0); 9 | 10 | import p from "./../../package.json"; 11 | 12 | const cli = `./../dist/cli-node.cjs` 13 | 14 | import {setup_tests} from "../util/util.ts"; 15 | setup_tests(); 16 | 17 | 18 | 19 | let g_dir = null; 20 | const get_dir = async () => { 21 | if (g_dir === null) { 22 | g_dir = `/tmp/transform-x-tests/${new Date().toISOString().replace(/[TZ]/g, "").replace(/[^\d]/g, "_")}` 23 | const x = await exec(`mkdir -p ${g_dir}`); 24 | } 25 | return g_dir; 26 | } 27 | 28 | const test_cli_interface = async () => { 29 | let x; 30 | const dir = await get_dir(); 31 | 32 | x = await exec(`${cli} version`); 33 | expect(p.version).toBe(x.stdout.trim()); 34 | 35 | 36 | const db_1 = { 37 | tables: [ 38 | { 39 | name: "t1", 40 | rows: [ 41 | { 42 | t_int: 1, 43 | t_string: "r1", 44 | t_bool: true, 45 | t_null: null, 46 | // uint8array: enc.encode("This is a string converted to a Uint8Array, encoded as utf-8"), 47 | nested: { 48 | a: 1, 49 | b: "This should be converted to JSON" 50 | }, 51 | date: new Date(), 52 | "dsfasdf@$^*&^%$": 1 53 | }, 54 | { 55 | new_col_a: 1, 56 | new_col_b: 2 57 | }, 58 | ] 59 | }, 60 | { 61 | name: "t2", 62 | rows: [ 63 | { 64 | t_int: 1, 65 | t_string: "r1", 66 | t_bool: true, 67 | t_null: null 68 | } 69 | ] 70 | } 71 | ] 72 | }; 73 | fs.writeFileSync(`${dir}/a.json`, JSON.stringify(db_1)); 74 | 75 | 76 | x = await exec(`${cli} json_to_sqlite --i ${dir}/a.json --o ${dir}/b.sqlite`); 77 | x = await exec(`${cli} sqlite_to_json --i ${dir}/b.sqlite --o ${dir}/c.json`); 78 | 79 | console.log({dir}); 80 | 81 | const a = JSON.parse(fs.readFileSync(`${dir}/c.json`, `utf-8`)); 82 | 83 | expect(a?.tables?.t1.rows[0]).toEqual({ 84 | "_x_id": 1, 85 | "date": db_1.tables[0].rows[0].date.toISOString(), 86 | "nested": "{\"a\":1,\"b\":\"This should be converted to JSON\"}", 87 | "new_col_a": null, 88 | "new_col_b": null, 89 | "t_bool": 1, 90 | "t_int": 1, 91 | "t_null": null, 92 | "t_string": "r1", 93 | }); 94 | } 95 | 96 | const cli_interface_flat_array_of_rows = async () => { 97 | let x; 98 | const dir = await get_dir(); 99 | 100 | x = await exec(`${cli} version`); 101 | expect(p.version).toBe(x.stdout.trim()); 102 | 103 | 104 | const rows_a = [ 105 | { 106 | t_int: 1, 107 | t_string: "r1", 108 | t_bool: true, 109 | t_null: null 110 | } 111 | ]; 112 | fs.writeFileSync(`${dir}/rows_a.json`, JSON.stringify(rows_a)); 113 | 114 | 115 | x = await exec(`${cli} json_to_sqlite --i ${dir}/rows_a.json --o ${dir}/rows_a_out.sqlite`); 116 | expect(x.stderr).toEqual(""); 117 | } 118 | 119 | 120 | const test_wal_mode = async () => { 121 | let x; 122 | const dir = await get_dir(); 123 | 124 | // Error for wal enabled SQLite files: `SQLite3Error: sqlite3 result code 26: file is not a database`. 125 | // - Check wal mode status with `sqlite3 x.sqlite ".dbinfo"` 126 | x = await exec(`${cli} sqlite_to_json --i ./04-cli-sqlite-json/wal-mode.sqlite --o ${dir}/d.json`); 127 | const d = JSON.parse(fs.readFileSync(`${dir}/d.json`, `utf-8`)); 128 | expect(d?.tables).toEqual({ 129 | "t1": { 130 | "name": "t1", 131 | "rows": [ 132 | { 133 | "c1": "v1" 134 | } 135 | ] 136 | }, 137 | "t2": { 138 | "name": "t2", 139 | "rows": [] 140 | } 141 | }); 142 | } 143 | 144 | 145 | describe('CLI: json_to_sqlite, sqlite_to_json', () => { 146 | test("cli_interface", async () => test_cli_interface()); 147 | test("cli_interface_flat_array_of_rows", async () => cli_interface_flat_array_of_rows()); 148 | test("wal_mode", async () => test_wal_mode()); 149 | }); 150 | 151 | -------------------------------------------------------------------------------- /tests/04-cli-sqlite-json/wal-mode.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/emadda/transform-x/a20b896152f64102259ac6cfcaf592455f651035/tests/04-cli-sqlite-json/wal-mode.sqlite -------------------------------------------------------------------------------- /tests/05-lib-sqlite-excel/05.test.ts: -------------------------------------------------------------------------------- 1 | // import test from 'node:test'; 2 | 3 | import _ from "lodash"; 4 | import assert from "assert"; 5 | import {util} from "zod"; 6 | 7 | 8 | import {json_to_sqlite, sqlite_to_json} from "../../src/lib/fns-sqlite-json.ts"; 9 | import fs from "fs"; 10 | import {get_dir, setup_tests} from "../util/util.ts"; 11 | import {excel_to_sqlite, sqlite_to_excel} from "../../src/lib/fns-sqlite-excel.ts"; 12 | import {excel_to_json} from "../../src/lib/fns-excel-json.ts"; 13 | import {log_json} from "../../src/lib/util.ts"; 14 | 15 | 16 | setup_tests(); 17 | 18 | const obj_ref = {}; 19 | const enc = new TextEncoder(); 20 | const db_1 = { 21 | tables: [ 22 | { 23 | name: "t1", 24 | rows: [ 25 | { 26 | _x_id: "col will be replaced as it conflicts with internal meta data col.", 27 | t_int: 1, 28 | t_string: "r1", 29 | t_bool: true, 30 | t_null: null, 31 | uint8array: enc.encode("This is a string converted to a Uint8Array, encoded as utf-8"), 32 | nested: { 33 | a: 1, 34 | b: "This should be converted to JSON" 35 | }, 36 | date: new Date("2023-08-15T18:16:13.502Z"), 37 | "dsfasdf@$^*&^%$": 1 38 | }, 39 | { 40 | new_col_a: 1, 41 | new_col_b: 2 42 | }, 43 | ] 44 | }, 45 | { 46 | name: "t2", 47 | rows: [ 48 | { 49 | t_int: 1, 50 | t_string: "r1", 51 | t_bool: true, 52 | t_null: null 53 | } 54 | ] 55 | } 56 | ] 57 | }; 58 | 59 | 60 | const db_1_out = { 61 | t1: { 62 | name: 't1', 63 | rows: [ 64 | { 65 | _x_id: 1, 66 | t_int: 1, 67 | t_string: 'r1', 68 | t_bool: 1, 69 | t_null: null, 70 | uint8array: null, 71 | nested: '{"a":1,"b":"This should be converted to JSON"}', 72 | date: '2023-08-15T18:16:13.502Z', 73 | new_col_a: null, 74 | new_col_b: null 75 | }, 76 | { 77 | _x_id: 2, 78 | t_int: null, 79 | t_string: null, 80 | t_bool: null, 81 | t_null: null, 82 | uint8array: null, 83 | nested: null, 84 | date: null, 85 | new_col_a: 1, 86 | new_col_b: 2 87 | } 88 | ] 89 | }, 90 | t2: { 91 | name: 't2', 92 | rows: [{_x_id: 1, t_int: 1, t_string: 'r1', t_bool: 1, t_null: null}] 93 | } 94 | }; 95 | 96 | const both = async () => { 97 | const dir = await get_dir(obj_ref); 98 | console.log(`Using ${dir}`); 99 | 100 | const a = await json_to_sqlite(db_1); 101 | expect(a.ok).toBe(true); 102 | fs.writeFileSync(`${dir}/a.sqlite`, a.data.sqlite_bytes); 103 | 104 | 105 | const b = await sqlite_to_excel({sqlite_bytes: a.data.sqlite_bytes}); 106 | fs.writeFileSync(`${dir}/b.xlsx`, b.data.xlsx_bytes); 107 | 108 | 109 | { 110 | // Test `excel_to_sqlite` 111 | const x = await excel_to_sqlite({xlsx_bytes: b.data.xlsx_bytes}); 112 | fs.writeFileSync(`${dir}/x.sqlite`, x.data.sqlite_bytes); 113 | 114 | const y = await sqlite_to_json({sqlite_bytes: x.data.sqlite_bytes}); 115 | 116 | expect(y.data?.tables).toEqual(db_1_out); 117 | 118 | } 119 | 120 | 121 | const c = await excel_to_json({xlsx_bytes: b.data.xlsx_bytes}); 122 | fs.writeFileSync(`${dir}/c.json`, JSON.stringify(c.data, null, 4)); 123 | 124 | expect(c.data?.tables).toEqual(db_1_out); 125 | } 126 | 127 | 128 | // @todo/low Offer msgpack alternative for JSON with Uint8Array binary values. 129 | describe('LIB: sqlite_to_excel, excel_to_sqlite', () => { 130 | test('both', async () => both()); 131 | }); 132 | 133 | -------------------------------------------------------------------------------- /tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "transform-x-tests", 3 | "description": "Package JSON used to change type=commonjs. `node --test` will read this and interpret `require` imports correctly, instead of ignoring them (as it does when type=module is used in the root package.json).", 4 | "type": "commonjs" 5 | } 6 | -------------------------------------------------------------------------------- /tests/sh/del-run-tests-bun.sh: -------------------------------------------------------------------------------- 1 | # Run from ./tests dir 2 | bun test -------------------------------------------------------------------------------- /tests/sh/esbuild-test.sh: -------------------------------------------------------------------------------- 1 | #./node_modules/.bin/esbuild ./src/core.ts --bundle --outfile=./dist/core.js --minify --watch --sourcemap 2 | 3 | # `outbase` causes parent directories to be created (like mkdir -p) for output files. 4 | ./../node_modules/.bin/esbuild \ 5 | ./01-lib-excel-json/01.test.ts \ 6 | ./02-cli-excel-json/02.test.ts \ 7 | ./03-lib-sqlite-json/03.test.ts \ 8 | ./04-cli-sqlite-json/04.test.ts \ 9 | ./05-lib-sqlite-excel/05.test.ts \ 10 | --platform=node --format=cjs --bundle --outdir=./ --outbase=./ --loader:.html=text --loader:.wasm=binary --watch --sourcemap; 11 | -------------------------------------------------------------------------------- /tests/sh/run-tests-node.sh: -------------------------------------------------------------------------------- 1 | # Run this from ./tests dir 2 | 3 | # All 4 | #node --test --watch --enable-source-maps 01-lib-excel-json/01.test.js 02-cli-excel-json/02.test.js 03-lib-sqlite-json/03.test.js 5 | 6 | 7 | # Single 8 | #node --test --watch --enable-source-maps 03-lib-sqlite-json/03.test.js 9 | 10 | #nodemon --exec "clear && printf '\e[3J'; node --test --enable-source-maps 03-lib-sqlite-json/03.test.js"; 11 | 12 | 13 | 14 | # All 15 | npx jest --colors \ 16 | ./01-lib-excel-json/01.test.js \ 17 | ./02-cli-excel-json/02.test.js \ 18 | ./03-lib-sqlite-json/03.test.js \ 19 | ./04-cli-sqlite-json/04.test.js \ 20 | ./05-lib-sqlite-excel/05.test.js; 21 | 22 | 23 | # Single 24 | # Use `jest` to avoid `node --test`'s line by line output which truncates JSON values. 25 | #npx jest --colors --watch 01-lib-excel-json/01.test.js 26 | #npx jest --colors --watch 02-cli-excel-json/02.test.js 27 | #npx jest --colors --watch 03-lib-sqlite-json/03.test.js 28 | #npx jest --colors --watch 04-cli-sqlite-json/04.test.js 29 | #npx jest --colors --watch 05-lib-sqlite-excel/05.test.js 30 | 31 | -------------------------------------------------------------------------------- /tests/util/config.ts: -------------------------------------------------------------------------------- 1 | const config = { 2 | 3 | } 4 | export { 5 | config 6 | } -------------------------------------------------------------------------------- /tests/util/util.ts: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | 3 | import fs from "fs"; 4 | import util from "util"; 5 | import {exec as exec0} from "child_process"; 6 | import {get_random_alpha_id} from "../../src/lib/util.ts"; 7 | 8 | const exec = util.promisify(exec0); 9 | 10 | const setup_tests = () => { 11 | const jestConsole = console; 12 | 13 | beforeEach(() => { 14 | global.console = require('console'); 15 | }); 16 | 17 | afterEach(() => { 18 | global.console = jestConsole; 19 | }); 20 | } 21 | 22 | 23 | 24 | // Use an obj_ref per test run to get a unique dir per test run. 25 | const get_dir = async (obj_ref) => { 26 | if (!_.isString(obj_ref?.dir)) { 27 | const random = get_random_alpha_id(4); 28 | obj_ref.dir = `/tmp/transform-x-tests/${new Date().toISOString().replace(/[TZ]/g, "").replace(/[^\d]/g, "_")}_${random}` 29 | const x = await exec(`mkdir -p ${obj_ref.dir}`); 30 | } 31 | return obj_ref.dir; 32 | } 33 | 34 | 35 | export { 36 | setup_tests, 37 | get_dir 38 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": [ 4 | "ESNext", 5 | "dom" 6 | ], 7 | "module": "esnext", 8 | "target": "esnext", 9 | "moduleResolution": "bundler", 10 | "moduleDetection": "force", 11 | "allowImportingTsExtensions": true, 12 | "strict": true, 13 | "downlevelIteration": true, 14 | "skipLibCheck": true, 15 | "jsx": "preserve", 16 | "allowSyntheticDefaultImports": true, 17 | "forceConsistentCasingInFileNames": true, 18 | "allowJs": true, 19 | "noEmit": true, 20 | "types": [ 21 | "bun-types", 22 | // add Bun global 23 | 24 | "jest" 25 | ] 26 | } 27 | } 28 | --------------------------------------------------------------------------------