├── .npmignore ├── bunfig.toml ├── bun.lockb ├── .prettierrc ├── .gitignore ├── src ├── helpers │ ├── random.ts │ ├── Unreachable.ts │ ├── isEmptyWrites.ts │ ├── isPlainObject.ts │ ├── compare.ts │ ├── mutableFilter.ts │ ├── invertString.ts │ ├── randomId.ts │ ├── useRerender.ts │ ├── iterateTuples.ts │ ├── maybeWaitForPromises.ts │ ├── enumerate.ts │ ├── mutableFilter.test.ts │ ├── namedTupleToObject.test.ts │ ├── shallowEqual.ts │ ├── invertString.test.ts │ ├── Queue.ts │ ├── naturalSort.ts │ ├── oudent.test.ts │ ├── DelayDb.ts │ ├── outdent.ts │ ├── binarySearch.ts │ ├── namedTupleToObject.ts │ ├── Queue.test.ts │ ├── subspaceHelpers.test.ts │ ├── binarySearch.test.ts │ ├── naturalSort.test.ts │ ├── sortedTupleValuePairs.ts │ ├── isBoundsWithinBounds.ts │ ├── queryBuilder.test.ts │ ├── sortedList.ts │ ├── compareTuple.test.ts │ ├── queryBuilder.ts │ ├── compareTuple.ts │ ├── subspaceHelpers.ts │ ├── sortedTupleArray.ts │ ├── sortedTupleValuePairs.test.ts │ ├── isBoundsWithinBounds.test.ts │ ├── codec.test.ts │ ├── codec.ts │ └── sortedTupleArray.test.ts ├── test │ ├── base.test.ts │ └── fixtures.ts ├── database │ ├── retry.ts │ ├── types.ts │ ├── sync │ │ ├── types.test.ts │ │ ├── transactionalRead.ts │ │ ├── transactionalReadWrite.ts │ │ ├── retry.ts │ │ ├── TupleDatabase.ts │ │ ├── subscribeQuery.ts │ │ ├── ReactivityTracker.ts │ │ ├── subscribeQuery.test.ts │ │ └── types.ts │ ├── ConcurrencyLog.test.ts │ ├── async │ │ ├── transactionalReadAsync.ts │ │ ├── transactionalReadWriteAsync.ts │ │ ├── retryAsync.ts │ │ ├── AsyncTupleDatabase.ts │ │ ├── subscribeQueryAsync.ts │ │ ├── AsyncReactivityTracker.ts │ │ └── asyncTypes.ts │ ├── transactionalWrite.ts │ ├── ConcurrencyLog.ts │ └── typeHelpers.ts ├── storage │ ├── BrowserTupleStorage.ts │ ├── InMemoryTupleStorage.ts │ ├── types.ts │ ├── SQLiteAsyncTupleStorage.ts │ ├── IndexedDbWithMemoryCacheTupleStorage.ts │ ├── MemoryBTreeTupleStorage.ts │ ├── LevelTupleStorage.ts │ ├── ExpoSQLiteLegacyStorage.ts │ ├── FileTupleStorage.ts │ ├── ExpoSQLiteStorage.ts │ ├── IndexedDbTupleStorage.ts │ ├── SQLiteTupleStorage.ts │ ├── LMDBTupleStorage.ts │ ├── storage.test.ts │ └── AdapterSqliteStorage.ts ├── main.ts ├── useTupleDatabase.ts ├── useAsyncTupleDatabase.ts ├── tools │ ├── compileMacros.ts │ └── benchmark.ts └── examples │ ├── triplestore.test.ts │ ├── triplestore.ts │ ├── socialApp.test.ts │ ├── endUserDatabase.test.ts │ └── classScheduling.test.ts ├── pack.sh ├── release.sh ├── test-hooks.ts ├── tsconfig.json ├── .github └── workflows │ └── test.yml ├── TODO.md └── package.json /.npmignore: -------------------------------------------------------------------------------- 1 | test/ 2 | *.test.* -------------------------------------------------------------------------------- /bunfig.toml: -------------------------------------------------------------------------------- 1 | [test] 2 | preload = ["./test-hooks.ts"] 3 | root = "./src" 4 | -------------------------------------------------------------------------------- /bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aspen-cloud/tuple-database/HEAD/bun.lockb -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "useTabs": true, 4 | "trailingComma": "es5" 5 | } 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | .DS_Store 4 | db 5 | build 6 | out 7 | *.db 8 | *.key 9 | tmp 10 | -------------------------------------------------------------------------------- /src/helpers/random.ts: -------------------------------------------------------------------------------- 1 | export function randomInt(ceil: number): number { 2 | return Math.floor(Math.random() * ceil) 3 | } 4 | -------------------------------------------------------------------------------- /src/test/base.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | 3 | describe("base", () => { 4 | it("works", () => { 5 | expect(true).toBeTruthy() 6 | }) 7 | }) 8 | -------------------------------------------------------------------------------- /src/helpers/Unreachable.ts: -------------------------------------------------------------------------------- 1 | export class UnreachableError extends Error { 2 | constructor(obj: never, message?: string) { 3 | super((message + ": " || "Unreachable: ") + obj) 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /pack.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | # npm version patch 4 | 5 | rm -rf build 6 | npm run build 7 | cp package.json build 8 | cp .npmignore build 9 | cp README.md build 10 | 11 | cd build 12 | npm pack 13 | cd .. -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | # npm version patch 4 | 5 | rm -rf build 6 | npm run build 7 | cp package.json build 8 | cp .npmignore build 9 | cp README.md build 10 | 11 | cd build 12 | npm publish --access=public 13 | cd .. -------------------------------------------------------------------------------- /src/helpers/isEmptyWrites.ts: -------------------------------------------------------------------------------- 1 | import { WriteOps } from "../storage/types" 2 | 3 | export function isEmptyWrites(writes: WriteOps) { 4 | if (writes.remove?.length) return false 5 | if (writes.set?.length) return false 6 | return true 7 | } 8 | -------------------------------------------------------------------------------- /src/helpers/isPlainObject.ts: -------------------------------------------------------------------------------- 1 | export function isPlainObject(value: any): boolean { 2 | if (value === null || typeof value !== "object") { 3 | return false 4 | } 5 | const proto = Object.getPrototypeOf(value) 6 | return proto === Object.prototype || proto === null 7 | } 8 | -------------------------------------------------------------------------------- /test-hooks.ts: -------------------------------------------------------------------------------- 1 | import { beforeAll, afterAll } from "bun:test" 2 | import { $ } from "bun" 3 | 4 | beforeAll(async () => { 5 | // global setup 6 | await $`rm -rf ./tmp` 7 | }) 8 | 9 | afterAll(async () => { 10 | // global teardown 11 | await $`rm -rf ./tmp` 12 | }) 13 | -------------------------------------------------------------------------------- /src/helpers/compare.ts: -------------------------------------------------------------------------------- 1 | export type Compare = (a: T, b: T) => number 2 | 3 | export function compare( 4 | a: K, 5 | b: K 6 | ): number { 7 | if (a > b) { 8 | return 1 9 | } 10 | if (a < b) { 11 | return -1 12 | } 13 | return 0 14 | } 15 | -------------------------------------------------------------------------------- /src/helpers/mutableFilter.ts: -------------------------------------------------------------------------------- 1 | export function mutableFilter(array: T[], fn: (item: T) => boolean) { 2 | let i = 0 3 | while (true) { 4 | if (i >= array.length) break 5 | const item = array[i] 6 | if (fn(item)) { 7 | i++ 8 | } else { 9 | array.splice(i, 1) 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/helpers/invertString.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This is helpful when you have a fixed-length string that you want to sort in reverse order. 3 | * For example, and ISO date string. 4 | */ 5 | export function invertString(str: string) { 6 | return str 7 | .split("") 8 | .map((char) => String.fromCharCode(-1 * char.charCodeAt(0))) 9 | .join("") 10 | } 11 | -------------------------------------------------------------------------------- /src/helpers/randomId.ts: -------------------------------------------------------------------------------- 1 | import { chunk } from "remeda" 2 | import * as uuid from "uuid" 3 | import md5 from "md5" 4 | 5 | export function randomId(seed?: string): string { 6 | if (seed) { 7 | const hexStr = md5(seed) 8 | const bytes = chunk(hexStr, 2).map((chars) => parseInt(chars.join(""), 16)) 9 | return uuid.v4({ random: bytes }) 10 | } else { 11 | return uuid.v4() 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/helpers/useRerender.ts: -------------------------------------------------------------------------------- 1 | import { useEffect, useRef, useState } from "react" 2 | 3 | export function useRerender() { 4 | const [state, setState] = useState(0) 5 | 6 | const mounted = useRef(true) 7 | useEffect( 8 | () => () => { 9 | mounted.current = false 10 | }, 11 | [] 12 | ) 13 | 14 | return () => { 15 | if (!mounted.current) return 16 | setState((x) => x + 1) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/helpers/iterateTuples.ts: -------------------------------------------------------------------------------- 1 | import { WriteOps } from "../storage/types" 2 | 3 | export function* iterateWrittenTuples(write: WriteOps) { 4 | for (const { key } of write.set || []) { 5 | yield key 6 | } 7 | for (const tuple of write.remove || []) { 8 | yield tuple 9 | } 10 | } 11 | 12 | export function getWrittenTuples(write: WriteOps) { 13 | return Array.from(iterateWrittenTuples(write)) 14 | } 15 | -------------------------------------------------------------------------------- /src/helpers/maybeWaitForPromises.ts: -------------------------------------------------------------------------------- 1 | export function maybePromiseAll(values: any[]): any { 2 | if (values.some((value) => value instanceof Promise)) 3 | return Promise.all( 4 | values.map((value) => { 5 | // Gobble up errors. 6 | if (value instanceof Promise) { 7 | return value.catch((error) => console.error(error)) 8 | } else { 9 | return value 10 | } 11 | }) 12 | ) 13 | else return values 14 | } 15 | -------------------------------------------------------------------------------- /src/helpers/enumerate.ts: -------------------------------------------------------------------------------- 1 | export function enumerate(array: T[]): [number, T][] { 2 | const pairs: [number, T][] = [] 3 | for (let i = 0; i < array.length; i++) { 4 | pairs.push([i, array[i]]) 5 | } 6 | return pairs 7 | } 8 | 9 | export function enumerateReverse(array: T[]): [number, T][] { 10 | const pairs: [number, T][] = [] 11 | for (let i = array.length - 1; i >= 0; i--) { 12 | pairs.push([i, array[i]]) 13 | } 14 | return pairs 15 | } 16 | -------------------------------------------------------------------------------- /src/helpers/mutableFilter.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { mutableFilter } from "./mutableFilter" 3 | 4 | describe("mutableFilter", () => { 5 | it("works", () => { 6 | const immutable = [1, 2, 3, 4, 5] 7 | const mutable = [1, 2, 3, 4, 5] 8 | 9 | const fn = (n: number) => n % 2 === 0 10 | 11 | const immutableResult = immutable.filter(fn) 12 | mutableFilter(mutable, fn) 13 | 14 | expect(immutableResult).toEqual(mutable) 15 | }) 16 | }) 17 | -------------------------------------------------------------------------------- /src/database/retry.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | This file is generated from async/retryAsync.ts 4 | 5 | */ 6 | 7 | type Identity = T 8 | 9 | import { ReadWriteConflictError } from "./ConcurrencyLog" 10 | 11 | export function retry(retries: number, fn: () => Identity) { 12 | while (true) { 13 | try { 14 | const result = fn() 15 | return result 16 | } catch (error) { 17 | if (retries <= 0) throw error 18 | const isConflict = error instanceof ReadWriteConflictError 19 | if (!isConflict) throw error 20 | retries -= 1 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/helpers/namedTupleToObject.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { Assert } from "../database/typeHelpers" 3 | import { namedTupleToObject } from "./namedTupleToObject" 4 | 5 | describe("namedTupleToObject", () => { 6 | it("works", () => { 7 | const tuple = ["hello", { a: 1 }, { b: ["c"] }] as [ 8 | "hello", 9 | { a: 1 }, 10 | { b: string[] } 11 | ] 12 | const obj = namedTupleToObject(tuple) 13 | type X = Assert 14 | expect(obj).toEqual({ a: 1, b: ["c"] }) 15 | }) 16 | }) 17 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.3.4", 3 | "compilerOptions": { 4 | "declaration": true, 5 | "declarationMap": true, 6 | "sourceMap": true, 7 | "mapRoot": ".", 8 | "module": "commonjs", 9 | "allowJs": true, 10 | "strictNullChecks": true, 11 | "strictFunctionTypes": true, 12 | "noImplicitThis": true, 13 | "allowSyntheticDefaultImports": true, 14 | "esModuleInterop": true, 15 | "removeComments": false, 16 | "experimentalDecorators": true, 17 | "jsx": "react", 18 | "target": "es2020", 19 | "lib": ["dom", "scripthost", "esnext.asynciterable", "ES2020"], 20 | "outDir": "build", 21 | "skipLibCheck": true 22 | }, 23 | "include": ["src/**/*"], 24 | "exclude": ["node_modules"] 25 | } 26 | -------------------------------------------------------------------------------- /src/helpers/shallowEqual.ts: -------------------------------------------------------------------------------- 1 | import { intersection, isArray } from "remeda" 2 | import { isPlainObject } from "./isPlainObject" 3 | 4 | export function shallowEqual(a: any, b: any) { 5 | if (a == b) return true 6 | if (isArray(a)) { 7 | if (!isArray(b)) return false 8 | if (a.length !== b.length) return false 9 | return a.every((x, i) => b[i] === x) 10 | } 11 | if (isPlainObject(a)) { 12 | if (!isPlainObject(b)) return false 13 | const aKeys = Object.keys(a) 14 | const bKeys = Object.keys(b) 15 | if (aKeys.length !== bKeys.length) return false 16 | const sameKeys = intersection(aKeys, bKeys) 17 | if (aKeys.length !== sameKeys.length) return false 18 | return aKeys.every((key) => a[key] == b[key]) 19 | } 20 | return false 21 | } 22 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Node.js CI 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v3 15 | 16 | - name: Install Node.js, NPM and Yarn 17 | uses: actions/setup-node@v3 18 | with: 19 | node-version: 16.6.0 20 | 21 | - name: Cache NPM dependencies 22 | uses: actions/cache@v1 23 | with: 24 | path: ~/.npm 25 | key: ${{ runner.OS }}-npm-cache-${{ hashFiles('**/package-lock.json') }} 26 | restore-keys: | 27 | ${{ runner.OS }}-npm-cache- 28 | 29 | - name: Install dependencies 30 | run: npm ci 31 | 32 | - run: npm test 33 | -------------------------------------------------------------------------------- /src/database/types.ts: -------------------------------------------------------------------------------- 1 | import { MAX, MIN, Tuple } from "../storage/types" 2 | import { RemoveTuplePrefix, TuplePrefix } from "./typeHelpers" 3 | 4 | export type ScanArgs< 5 | T extends Tuple, 6 | P extends TuplePrefix 7 | > = PrefixScanArgs 8 | 9 | export type PrefixScanArgs> = { 10 | prefix?: P 11 | gt?: AllowMinMax>> 12 | gte?: AllowMinMax>> 13 | lt?: AllowMinMax>> 14 | lte?: AllowMinMax>> 15 | limit?: number 16 | reverse?: boolean 17 | } 18 | 19 | type AllowMinMax = { 20 | [K in keyof T]: T[K] | typeof MIN | typeof MAX 21 | } 22 | 23 | export type TxId = string 24 | 25 | export type Unsubscribe = () => void 26 | -------------------------------------------------------------------------------- /src/storage/BrowserTupleStorage.ts: -------------------------------------------------------------------------------- 1 | import { TupleStorageApi } from "../database/sync/types" 2 | import { InMemoryTupleStorage } from "./InMemoryTupleStorage" 3 | import { WriteOps } from "./types" 4 | 5 | function load(key: string) { 6 | const result = localStorage.getItem(key) 7 | if (!result) return 8 | try { 9 | return JSON.parse(result) 10 | } catch (error) {} 11 | } 12 | 13 | function save(key: string, value: any) { 14 | localStorage.setItem(key, JSON.stringify(value)) 15 | } 16 | 17 | export class BrowserTupleStorage 18 | extends InMemoryTupleStorage 19 | implements TupleStorageApi 20 | { 21 | constructor(public localStorageKey: string) { 22 | super(load(localStorageKey)) 23 | } 24 | 25 | commit(writes: WriteOps): void { 26 | super.commit(writes) 27 | save(this.localStorageKey, this.data) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/helpers/invertString.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { invertString } from "./invertString" 3 | 4 | describe("invertString", () => { 5 | const data = [ 6 | "aaa", 7 | "aab", 8 | "aac", 9 | "aba", 10 | "abc", 11 | "aca", 12 | "acc", 13 | "bbb", 14 | "bca", 15 | "bcb", 16 | "caa", 17 | "cab", 18 | "ccc", 19 | ] 20 | 21 | it("can encode and decode properly", () => { 22 | for (const str of data) { 23 | expect(invertString(invertString(str))).toStrictEqual(str) 24 | } 25 | }) 26 | 27 | it("inversion is reverse sorted", () => { 28 | const sorted = [...data].sort() 29 | expect(sorted).toStrictEqual(data) 30 | 31 | const inverseSorted = sorted.map(invertString).sort().map(invertString) 32 | expect(inverseSorted).toStrictEqual(sorted.reverse()) 33 | }) 34 | }) 35 | -------------------------------------------------------------------------------- /src/helpers/Queue.ts: -------------------------------------------------------------------------------- 1 | export type Thunk = () => Promise | T 2 | 3 | export class Queue { 4 | private currentPromise: Promise | undefined 5 | 6 | public enqueue(fn: Thunk): Promise | T { 7 | if (this.currentPromise) { 8 | const nextPromise = this.currentPromise.then(fn).then((result) => { 9 | if (this.currentPromise === nextPromise) this.currentPromise = undefined 10 | return result 11 | }) 12 | this.currentPromise = nextPromise 13 | return nextPromise 14 | } 15 | 16 | const result = fn() 17 | if (result instanceof Promise) { 18 | const nextPromise = result.then((result) => { 19 | if (this.currentPromise === nextPromise) this.currentPromise = undefined 20 | return result 21 | }) 22 | this.currentPromise = nextPromise 23 | return nextPromise 24 | } 25 | 26 | return result 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/test/fixtures.ts: -------------------------------------------------------------------------------- 1 | import { Tuple } from "../storage/types" 2 | 3 | export const sortedValues: Tuple = [ 4 | null, 5 | {}, 6 | { a: 1 }, 7 | { a: 2 }, 8 | { a: 2, b: 1 }, 9 | { a: 2, c: 2 }, 10 | { b: 1 }, 11 | [], 12 | [1], 13 | [1, [2]], 14 | [1, 2], 15 | [1, 3], 16 | [2], 17 | -Number.MAX_VALUE, 18 | Number.MIN_SAFE_INTEGER, 19 | -999999, 20 | -1, 21 | -Number.MIN_VALUE, 22 | 0, 23 | Number.MIN_VALUE, 24 | 1, 25 | 999999, 26 | Number.MAX_SAFE_INTEGER, 27 | Number.MAX_VALUE, 28 | "", 29 | "\x00", 30 | "\x00\x00", 31 | "\x00\x01", 32 | "\x00\x02", 33 | "\x00A", 34 | "\x01", 35 | "\x01\x00", 36 | "\x01\x01", 37 | "\x01\x02", 38 | "\x01A", 39 | "\x02", 40 | "\x02\x00", 41 | "\x02\x01", 42 | "\x02\x02", 43 | "\x02A", 44 | "A", 45 | "A\x00", 46 | "A\x01", 47 | "A\x02", 48 | "AA", 49 | "AAB", 50 | "AB", 51 | "B", 52 | false, 53 | true, 54 | ] 55 | -------------------------------------------------------------------------------- /src/storage/InMemoryTupleStorage.ts: -------------------------------------------------------------------------------- 1 | import { TupleStorageApi } from "../database/sync/types" 2 | import * as tv from "../helpers/sortedTupleValuePairs" 3 | import { KeyValuePair, ScanStorageArgs, WriteOps } from "./types" 4 | 5 | export class InMemoryTupleStorage implements TupleStorageApi { 6 | data: KeyValuePair[] 7 | 8 | constructor(data?: KeyValuePair[]) { 9 | this.data = data || [] 10 | } 11 | 12 | scan(args?: ScanStorageArgs) { 13 | return tv.scan(this.data, args) 14 | } 15 | 16 | commit(writes: WriteOps) { 17 | // Indexers run inside the tx so we don't need to do that here. 18 | // And because of that, the order here should not matter. 19 | const { set, remove } = writes 20 | for (const tuple of remove || []) { 21 | tv.remove(this.data, tuple) 22 | } 23 | for (const { key, value } of set || []) { 24 | tv.set(this.data, key, value) 25 | } 26 | } 27 | 28 | close() {} 29 | } 30 | -------------------------------------------------------------------------------- /src/helpers/naturalSort.ts: -------------------------------------------------------------------------------- 1 | // numbers 2 | // decimals 3 | // scientific notation 4 | // case insensitive 5 | 6 | type NumberParse = { 7 | string: string 8 | exponent: { 9 | negative: boolean 10 | integer: string 11 | } 12 | integer: string 13 | decimal: string 14 | negative: boolean 15 | } 16 | 17 | const re = new RegExp( 18 | [/[-+]/, /[0-9]+/, /[0-9]+/].map((r) => r.source).join("") 19 | ) 20 | 21 | const numberRe = /([-+])?([0-9]+)?(\.[0-9]*)?(e[+-]?[0-9]+)?/g 22 | 23 | function parseNumber(str: string) { 24 | const match = str.match(numberRe) 25 | if (!match) return 26 | return parseFloat(match[0].replace(/,/g, "")) 27 | } 28 | 29 | // "1".match(re) 30 | // "1.9".match(re) 31 | // ".1".match(re) 32 | // "-12.1".match(re) 33 | // "+12.1e14".match(re) 34 | 35 | function naturalCompare(a: string, b: string) { 36 | let i = 0 37 | while (i < a.length && i < b.length) { 38 | // if () {} 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/helpers/oudent.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, test, expect } from "bun:test" 2 | import { outdent } from "./outdent" 3 | 4 | describe("outdent", () => { 5 | test("works", () => { 6 | const actual = outdent(` 7 | ReadWriteConflictError 8 | Write to tuple 9 | conflicted with a read at the bounds 10 | `) 11 | 12 | const expected = `ReadWriteConflictError 13 | Write to tuple 14 | conflicted with a read at the bounds` 15 | 16 | expect(actual).toStrictEqual(expected) 17 | }) 18 | 19 | test("only trims the minimum indent across all the lines", () => { 20 | // First line is indented only one tab 21 | const actual = outdent(` 22 | ReadWriteConflictError 23 | Write to tuple 24 | conflicted with a read at the bounds 25 | `) 26 | 27 | const expected = `ReadWriteConflictError 28 | Write to tuple 29 | conflicted with a read at the bounds` 30 | 31 | expect(actual).toStrictEqual(expected) 32 | }) 33 | }) 34 | -------------------------------------------------------------------------------- /src/helpers/DelayDb.ts: -------------------------------------------------------------------------------- 1 | import { AsyncTupleDatabaseApi } from "../database/async/asyncTypes" 2 | import { TupleDatabaseApi } from "../database/sync/types" 3 | 4 | function sleep(ms = 0) { 5 | return new Promise((resolve) => setTimeout(resolve, ms)) 6 | } 7 | 8 | // Introduce delay into a database, mostly for debugging purposes. 9 | export function DelayDb( 10 | db: AsyncTupleDatabaseApi | TupleDatabaseApi, 11 | delay = 0 12 | ): AsyncTupleDatabaseApi { 13 | return { 14 | scan: async (...args) => { 15 | await sleep(delay) 16 | return db.scan(...args) 17 | }, 18 | commit: async (...args) => { 19 | await sleep(delay) 20 | return db.commit(...args) 21 | }, 22 | cancel: async (...args) => { 23 | await sleep(delay) 24 | return db.cancel(...args) 25 | }, 26 | subscribe: async (...args) => { 27 | await sleep(delay) 28 | return db.subscribe(...args) 29 | }, 30 | close: async (...args) => { 31 | await sleep(delay) 32 | return db.close(...args) 33 | }, 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/database/sync/types.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { InMemoryTupleStorage } from "../../main" 3 | import { TupleDatabase } from "./TupleDatabase" 4 | import { TupleDatabaseClient } from "./TupleDatabaseClient" 5 | 6 | type TestSchema = 7 | | { 8 | key: [0] 9 | value: true 10 | } 11 | | { 12 | key: [1, string] 13 | value: true 14 | } 15 | 16 | describe("Type tests", () => { 17 | const db = new TupleDatabaseClient( 18 | new TupleDatabase(new InMemoryTupleStorage()) 19 | ) 20 | 21 | it("Root transaction `set` correctly types values", () => { 22 | const tx = db.transact() 23 | 24 | // @ts-expect-error 25 | tx.set([0], false) 26 | 27 | tx.set([0], true) 28 | 29 | tx.cancel() 30 | }) 31 | 32 | it("Subspace transaction `set` correctly types values", () => { 33 | const tx = db.subspace([1]).transact() 34 | 35 | // @ts-expect-error 36 | tx.set(["hello"], false) 37 | 38 | tx.set(["hello"], true) 39 | 40 | tx.cancel() 41 | }) 42 | }) 43 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | export * from "./database/async/AsyncTupleDatabase" 2 | export * from "./database/async/AsyncTupleDatabaseClient" 3 | export * from "./database/async/asyncTypes" 4 | export * from "./database/async/subscribeQueryAsync" 5 | export * from "./database/async/transactionalReadAsync" 6 | export * from "./database/async/transactionalReadWriteAsync" 7 | export * from "./database/sync/subscribeQuery" 8 | export * from "./database/sync/transactionalRead" 9 | export * from "./database/sync/transactionalReadWrite" 10 | export * from "./database/sync/TupleDatabase" 11 | export * from "./database/sync/TupleDatabaseClient" 12 | export * from "./database/sync/types" 13 | export * from "./database/transactionalWrite" 14 | export type { SchemaSubspace } from "./database/typeHelpers" 15 | export * from "./database/types" 16 | export * from "./helpers/namedTupleToObject" 17 | export * from "./storage/InMemoryTupleStorage" 18 | export * from "./storage/types" 19 | export { compareTuple } from "./helpers/compareTuple" 20 | export * from "./helpers/codec" 21 | -------------------------------------------------------------------------------- /src/database/ConcurrencyLog.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { normalizeTupleBounds } from "../helpers/sortedTupleArray" 3 | import { Tuple } from "../storage/types" 4 | import { ConcurrencyLog } from "./ConcurrencyLog" 5 | 6 | function bounds(prefix: Tuple) { 7 | return normalizeTupleBounds({ prefix }) 8 | } 9 | 10 | describe("ConcurrencyLog", () => { 11 | it("Only records writes with conflicting reads.", () => { 12 | const log = new ConcurrencyLog() 13 | 14 | log.write("tx1", [1]) 15 | expect(log.log).toEqual([]) 16 | 17 | log.read("tx2", bounds([2])) 18 | 19 | log.write("tx3", [2]) 20 | log.write("tx3", [3]) 21 | 22 | expect(log.log).toEqual([ 23 | { type: "read", txId: "tx2", bounds: bounds([2]) }, 24 | { type: "write", txId: "tx3", tuple: [2] }, 25 | ]) 26 | 27 | expect(() => log.commit("tx2")).toThrow() 28 | expect(log.log).toEqual([]) 29 | }) 30 | 31 | it.todo("Keeps writes that conflict with reads of other transactions.") 32 | 33 | it.todo("Can cancel a transaction to clean up the log.") 34 | }) 35 | -------------------------------------------------------------------------------- /src/database/async/transactionalReadAsync.ts: -------------------------------------------------------------------------------- 1 | import { KeyValuePair } from "../../storage/types" 2 | import { retry } from "../retry" 3 | import { 4 | AsyncTupleDatabaseClientApi, 5 | AsyncTupleTransactionApi, 6 | ReadOnlyAsyncTupleDatabaseClientApi, 7 | } from "./asyncTypes" 8 | 9 | /** 10 | * Similar to transactionalReadWrite and transactionalWrite but only allows reads. 11 | */ 12 | export function transactionalReadAsync( 13 | retries = 5 14 | ) { 15 | return function ( 16 | fn: (tx: ReadOnlyAsyncTupleDatabaseClientApi, ...args: I) => O 17 | ) { 18 | return function ( 19 | dbOrTx: 20 | | AsyncTupleDatabaseClientApi 21 | | AsyncTupleTransactionApi 22 | | ReadOnlyAsyncTupleDatabaseClientApi, 23 | ...args: I 24 | ): O { 25 | if (!("transact" in dbOrTx)) return fn(dbOrTx, ...args) 26 | return retry(retries, () => { 27 | const tx = dbOrTx.transact() 28 | const result = fn(tx, ...args) 29 | tx.commit() 30 | return result 31 | }) 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/database/sync/transactionalRead.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | This file is generated from async/transactionalReadAsync.ts 4 | 5 | */ 6 | 7 | type Identity = T 8 | 9 | import { KeyValuePair } from "../../storage/types" 10 | import { retry } from "../retry" 11 | import { 12 | ReadOnlyTupleDatabaseClientApi, 13 | TupleDatabaseClientApi, 14 | TupleTransactionApi, 15 | } from "./types" 16 | 17 | /** 18 | * Similar to transactionalReadWrite and transactionalWrite but only allows reads. 19 | */ 20 | export function transactionalRead( 21 | retries = 5 22 | ) { 23 | return function ( 24 | fn: (tx: ReadOnlyTupleDatabaseClientApi, ...args: I) => O 25 | ) { 26 | return function ( 27 | dbOrTx: 28 | | TupleDatabaseClientApi 29 | | TupleTransactionApi 30 | | ReadOnlyTupleDatabaseClientApi, 31 | ...args: I 32 | ): O { 33 | if (!("transact" in dbOrTx)) return fn(dbOrTx, ...args) 34 | return retry(retries, () => { 35 | const tx = dbOrTx.transact() 36 | const result = fn(tx, ...args) 37 | tx.commit() 38 | return result 39 | }) 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/storage/types.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Just basic JSON data-types. This is a pragmatic decision: 4 | - If we have custom data types in here, we have to consider how to deserialize 5 | into different languages. For JavaScript, that means creating a class. But 6 | these classes don't serialize well over a JSON bridge between processes. 7 | - The kind of data types we might want is endless. To start, I can think of 8 | {uuid: string}, {date: string} but then there's things like {url: string} or 9 | {phone: string} which dive deeper into application-level concepts. 10 | 11 | So that is why this database layer only deals with JSON. 12 | 13 | */ 14 | 15 | export type Value = string | number | boolean | null | Array | object 16 | 17 | export type Tuple = Value[] 18 | 19 | export type KeyValuePair = { key: Tuple; value: any } 20 | 21 | export const MIN = null 22 | export const MAX = true 23 | 24 | export type WriteOps = { 25 | set?: S[] 26 | remove?: S["key"][] 27 | } 28 | 29 | export type ScanStorageArgs = { 30 | gt?: Tuple 31 | gte?: Tuple 32 | lt?: Tuple 33 | lte?: Tuple 34 | limit?: number 35 | reverse?: boolean 36 | } 37 | -------------------------------------------------------------------------------- /src/helpers/outdent.ts: -------------------------------------------------------------------------------- 1 | // How many spaces to count in a tab (project-level config) 2 | const tabToSpaces = 2 3 | 4 | function convertTabsToSpaces(line: string) { 5 | return line.replace(/\t/g, " ".repeat(tabToSpaces)) 6 | } 7 | 8 | function getIndentCount(line: string) { 9 | let indent = 0 10 | 11 | for (const char of line) { 12 | if (char === " ") { 13 | indent += 1 14 | } else { 15 | return indent 16 | } 17 | } 18 | 19 | return indent 20 | } 21 | 22 | /** 23 | * Achieves the same thing as https://www.npmjs.com/package/outdent, but a little cleaner 24 | */ 25 | export function outdent(contents: string) { 26 | let lines = contents.split("\n").map(convertTabsToSpaces) 27 | 28 | // Ignore all-whitespace lines at the beginning and end 29 | // (which are common in template literals) 30 | if (lines[0].trim() === "") { 31 | lines = lines.slice(1) 32 | } 33 | if (lines[lines.length - 1].trim() === "") { 34 | lines = lines.slice(0, lines.length - 1) 35 | } 36 | 37 | const indentCounts = lines.map(getIndentCount) 38 | const minIndentCount = Math.min(...indentCounts) 39 | 40 | const trimmedLines = lines.map((line) => line.slice(minIndentCount)) 41 | 42 | return trimmedLines.join("\n") 43 | } 44 | -------------------------------------------------------------------------------- /src/useTupleDatabase.ts: -------------------------------------------------------------------------------- 1 | import { useEffect, useMemo, useRef } from "react" 2 | import { subscribeQuery } from "./database/sync/subscribeQuery" 3 | import { TupleDatabaseClientApi } from "./database/sync/types" 4 | import { shallowEqual } from "./helpers/shallowEqual" 5 | import { useRerender } from "./helpers/useRerender" 6 | import { KeyValuePair } from "./storage/types" 7 | 8 | /** Useful for managing UI state for React with a TupleDatabase. */ 9 | export function useTupleDatabase( 10 | db: TupleDatabaseClientApi, 11 | fn: (db: TupleDatabaseClientApi, ...arg: A) => T, 12 | args: A 13 | ) { 14 | const rerender = useRerender() 15 | const resultRef = useRef({} as any) 16 | 17 | const destroy = useMemo(() => { 18 | const { result, destroy } = subscribeQuery( 19 | db, 20 | (db) => fn(db, ...args), 21 | (result) => { 22 | if (!shallowEqual(resultRef.current, result)) { 23 | resultRef.current = result 24 | rerender() 25 | } 26 | } 27 | ) 28 | resultRef.current = result 29 | return destroy 30 | }, [db, fn, ...args]) 31 | 32 | useEffect(() => { 33 | return destroy 34 | }, [destroy]) 35 | 36 | return resultRef.current 37 | } 38 | -------------------------------------------------------------------------------- /src/helpers/binarySearch.ts: -------------------------------------------------------------------------------- 1 | import { Compare } from "./compare" 2 | 3 | export type BinarySearchResult = 4 | | { found: number; closest?: undefined } 5 | | { found?: undefined; closest: number } 6 | 7 | // This binary search is generalized so that we can use it for both normal lists 8 | // as well as associative lists. 9 | export function generalizedBinarySearch( 10 | getValue: (item: I) => V, 11 | cmp: Compare 12 | ) { 13 | return function (list: Array, item: V): BinarySearchResult { 14 | var min = 0 15 | var max = list.length - 1 16 | while (min <= max) { 17 | var k = (max + min) >> 1 18 | var dir = cmp(item, getValue(list[k])) 19 | if (dir > 0) { 20 | min = k + 1 21 | } else if (dir < 0) { 22 | max = k - 1 23 | } else { 24 | return { found: k } 25 | } 26 | } 27 | return { closest: min } 28 | } 29 | } 30 | 31 | export function binarySearch(list: T[], item: T, cmp: Compare) { 32 | return generalizedBinarySearch((x) => x, cmp)(list, item) 33 | } 34 | 35 | export function binarySearchAssociativeList( 36 | list: [T, any][], 37 | item: T, 38 | cmp: Compare 39 | ) { 40 | return generalizedBinarySearch<[T, any], T>((x) => x[0], cmp)(list, item) 41 | } 42 | -------------------------------------------------------------------------------- /src/storage/SQLiteAsyncTupleStorage.ts: -------------------------------------------------------------------------------- 1 | import { Database } from "better-sqlite3" 2 | import { KeyValuePair, ScanStorageArgs, WriteOps } from "./types" 3 | import { AsyncTupleStorageApi } from "../main" 4 | import { SQLiteTupleStorage } from "./SQLiteTupleStorage" 5 | 6 | export class SQLiteAsyncTupleStorage implements AsyncTupleStorageApi { 7 | /** 8 | * import sqlite from "better-sqlite3" 9 | * new SQLiteTupleStorage(sqlite("path/to.db")) 10 | */ 11 | private _storage: SQLiteTupleStorage 12 | constructor(private db: Database) { 13 | this._storage = new SQLiteTupleStorage(db) 14 | } 15 | 16 | async scan(args: ScanStorageArgs = {}) { 17 | return new Promise((res, rej) => { 18 | try { 19 | res(this._storage.scan(args)) 20 | } catch (e) { 21 | rej(e) 22 | } 23 | }) 24 | } 25 | 26 | async commit(writes: WriteOps) { 27 | return new Promise((res, rej) => { 28 | try { 29 | this._storage.commit(writes) 30 | res() 31 | } catch (e) { 32 | rej(e) 33 | } 34 | }) 35 | } 36 | 37 | async close() { 38 | return new Promise((res, rej) => { 39 | try { 40 | this.db.close() 41 | res() 42 | } catch (e) { 43 | rej(e) 44 | } 45 | }) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/useAsyncTupleDatabase.ts: -------------------------------------------------------------------------------- 1 | import { useEffect, useState } from "react" 2 | import { AsyncTupleDatabaseClientApi } from "./database/async/asyncTypes" 3 | import { shallowEqual } from "./helpers/shallowEqual" 4 | import { subscribeQueryAsync } from "./main" 5 | import { KeyValuePair } from "./storage/types" 6 | 7 | /** Useful for managing UI state for React with a TupleDatabase. */ 8 | export function useAsyncTupleDatabase< 9 | S extends KeyValuePair, 10 | T, 11 | A extends any[] 12 | >( 13 | db: AsyncTupleDatabaseClientApi, 14 | fn: (db: AsyncTupleDatabaseClientApi, ...arg: A) => Promise, 15 | args: A 16 | ) { 17 | const [result, setResult] = useState(undefined) 18 | 19 | useEffect(() => { 20 | let stopped = false 21 | let stop: (() => void) | undefined 22 | subscribeQueryAsync( 23 | db, 24 | (db) => fn(db, ...args), 25 | (newResult) => { 26 | if (stopped) return 27 | if (!shallowEqual(newResult, result)) { 28 | setResult(newResult) 29 | } 30 | } 31 | ).then(({ result, destroy }) => { 32 | setResult(result) 33 | if (stopped) destroy() 34 | else stop = destroy 35 | }) 36 | return () => { 37 | stopped = true 38 | if (stop) stop() 39 | } 40 | }, [db, fn, ...args]) 41 | 42 | return result 43 | } 44 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | ## TODO 2 | 3 | - listen vs subscribe. transactional. makes things much simpler over here. 4 | 5 | 6 | - types... need to constraint key-value args. value cna be wrong. 7 | - types... should be able to pass a string literal to a string and it should match for prefix extension stuff. 8 | 9 | 10 | - client.expose(subspace, indexer) 11 | 12 | - keep track of transactions and expire them after a timeout 13 | - keep track of committed transactions too and expire after timeout? 14 | 15 | - using ipc-peer over a socket for client across a process / network. 16 | 17 | - rtree for reactivity 18 | - migration abstraction for MIN/MAX 19 | 20 | - queue up commits to wait til after previous emits to prevent infinite loop issues. though also warn about it. 21 | Should we? This reduces concurrency performance advantage... Storage needs to handle concurrent writes? 22 | 23 | 24 | 25 | - play with more triplestore ergonomics, order value, proxy objects. 26 | keep it simple for now though, no need to EXPLAIN or baked in indexes yet. 27 | 28 | - types for ScanArgs with min/max for scanning.. 29 | 30 | - use reactive-magic strategy for composing queries more naturally. 31 | - get reactivity? 32 | 33 | ### Sometime Maybe 34 | - Readable CSV FileStorage that isnt a cache. 35 | - proper abstraction for encoding/decoding objects, with `prototype.{compare, serialize, deserialize}` 36 | - compound sort directions? 37 | -------------------------------------------------------------------------------- /src/helpers/namedTupleToObject.ts: -------------------------------------------------------------------------------- 1 | import { isPlainObject } from "./isPlainObject" 2 | import { Tuple, Value } from "../storage/types" 3 | 4 | // It is convenient to use named tuples when defining a schema so that you 5 | // don't get confused what [string, number] is and can instead write something 6 | // like [{playerId: string}, {score: number}]. 7 | type NamedTupleItem = { [key: string | number]: Value } 8 | 9 | // When we have a named tuple, we can merge all items into an object that is 10 | // more convenient to work with: 11 | export type NamedTupleToObject = CleanUnionToIntersection< 12 | Extract 13 | > 14 | 15 | function isNamedTupleItem(value: Value): value is NamedTupleItem { 16 | return isPlainObject(value) 17 | } 18 | 19 | export function namedTupleToObject(key: T) { 20 | const obj = key 21 | .filter(isNamedTupleItem) 22 | .reduce((obj, item) => Object.assign(obj, item), {}) 23 | return obj as NamedTupleToObject 24 | } 25 | 26 | // Some type wizardry. 27 | // https://stackoverflow.com/questions/63542526/merge-discriminated-union-of-object-types-in-typescript 28 | type UnionToIntersection = (U extends any ? (k: U) => void : never) extends ( 29 | k: infer I 30 | ) => void 31 | ? I 32 | : never 33 | 34 | type CleanUnionToIntersection = UnionToIntersection extends infer O 35 | ? { [K in keyof O]: O[K] } 36 | : never 37 | -------------------------------------------------------------------------------- /src/helpers/Queue.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { Queue } from "./Queue" 3 | 4 | describe("Queue", () => { 5 | it("evaluates synchronously", () => { 6 | const q = new Queue() 7 | 8 | const items: any[] = [] 9 | 10 | q.enqueue(() => items.push(1)) 11 | expect(items).toEqual([1]) 12 | 13 | q.enqueue(() => items.push(2)) 14 | expect(items).toEqual([1, 2]) 15 | }) 16 | 17 | it("evaluates asynchronously", async () => { 18 | const q = new Queue() 19 | 20 | const items: any[] = [] 21 | 22 | const d1 = new DeferredPromise() 23 | const q1 = q.enqueue(async () => { 24 | await d1.promise 25 | items.push(1) 26 | }) 27 | 28 | const d2 = new DeferredPromise() 29 | const q2 = q.enqueue(async () => { 30 | await d2.promise 31 | items.push(2) 32 | }) 33 | 34 | expect(items).toEqual([]) 35 | 36 | d1.resolve() 37 | await q1 38 | expect(items).toEqual([1]) 39 | 40 | d2.resolve() 41 | await q2 42 | expect(items).toEqual([1, 2]) 43 | }) 44 | }) 45 | 46 | /** 47 | * A Promise utility that lets you specify the resolve/reject after the promise is made 48 | * (or outside of the Promise constructor) 49 | */ 50 | class DeferredPromise { 51 | resolve!: (value: T) => void 52 | reject!: (error: any) => void 53 | promise: Promise 54 | constructor() { 55 | this.promise = new Promise((resolve, reject) => { 56 | this.resolve = resolve 57 | this.reject = reject 58 | }) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/database/async/transactionalReadWriteAsync.ts: -------------------------------------------------------------------------------- 1 | import { KeyValuePair } from "../../main" 2 | import { 3 | AsyncTupleDatabaseClientApi, 4 | AsyncTupleTransactionApi, 5 | } from "./asyncTypes" 6 | import { RetryOptions, retryAsync } from "./retryAsync" 7 | 8 | // Similar to FoundationDb's abstraction: https://apple.github.io/foundationdb/class-scheduling.html 9 | // Accepts a transaction or a database and allows you to compose transactions together. 10 | 11 | // This outer function is just used for the schema type because currying is the only way 12 | // we can partially infer generic type parameters. 13 | // https://stackoverflow.com/questions/60377365/typescript-infer-type-of-generic-after-optional-first-generic 14 | export function transactionalReadWriteAsync< 15 | S extends KeyValuePair = KeyValuePair 16 | >(retries = 5, options: RetryOptions = {}) { 17 | return function ( 18 | fn: (tx: AsyncTupleTransactionApi, ...args: I) => Promise 19 | ) { 20 | return async function ( 21 | dbOrTx: AsyncTupleDatabaseClientApi | AsyncTupleTransactionApi, 22 | ...args: I 23 | ): Promise { 24 | if (!("transact" in dbOrTx)) return fn(dbOrTx, ...args) 25 | return await retryAsync( 26 | retries, 27 | async () => { 28 | const tx = dbOrTx.transact() 29 | const result = await fn(tx, ...args) 30 | await tx.commit() 31 | return result 32 | }, 33 | options 34 | ) 35 | } 36 | } 37 | } 38 | 39 | /** @deprecated */ 40 | export const transactionalAsyncQuery = transactionalReadWriteAsync 41 | -------------------------------------------------------------------------------- /src/database/sync/transactionalReadWrite.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | This file is generated from async/transactionalReadWriteAsync.ts 4 | 5 | */ 6 | 7 | type Identity = T 8 | 9 | import { KeyValuePair } from "../../main" 10 | import { retry, RetryOptions } from "./retry" 11 | import { TupleDatabaseClientApi, TupleTransactionApi } from "./types" 12 | 13 | // Similar to FoundationDb's abstraction: https://apple.github.io/foundationdb/class-scheduling.html 14 | // Accepts a transaction or a database and allows you to compose transactions together. 15 | 16 | // This outer function is just used for the schema type because currying is the only way 17 | // we can partially infer generic type parameters. 18 | // https://stackoverflow.com/questions/60377365/typescript-infer-type-of-generic-after-optional-first-generic 19 | export function transactionalReadWrite( 20 | retries = 5, 21 | options: RetryOptions = {} 22 | ) { 23 | return function ( 24 | fn: (tx: TupleTransactionApi, ...args: I) => Identity 25 | ) { 26 | return function ( 27 | dbOrTx: TupleDatabaseClientApi | TupleTransactionApi, 28 | ...args: I 29 | ): Identity { 30 | if (!("transact" in dbOrTx)) return fn(dbOrTx, ...args) 31 | return retry( 32 | retries, 33 | () => { 34 | const tx = dbOrTx.transact() 35 | const result = fn(tx, ...args) 36 | tx.commit() 37 | return result 38 | }, 39 | options 40 | ) 41 | } 42 | } 43 | } 44 | 45 | /** @deprecated */ 46 | export const transactionalQuery = transactionalReadWrite 47 | -------------------------------------------------------------------------------- /src/database/async/retryAsync.ts: -------------------------------------------------------------------------------- 1 | import { ReadWriteConflictError } from "../../database/ConcurrencyLog" 2 | 3 | export type RetryOptions = { 4 | exponentialBackoff?: boolean 5 | backoffBase?: number 6 | maxDelay?: number 7 | jitter?: boolean 8 | } 9 | 10 | const DEFAULT_MAX_DELAY = 1000 11 | const DEFAULT_BACKOFF_BASE = 10 12 | 13 | export async function retryAsync( 14 | retries: number, 15 | fn: () => Promise, 16 | options: RetryOptions = {} 17 | ) { 18 | let delay = 0 19 | let attempt = 0 20 | const { exponentialBackoff, jitter } = options 21 | const backoffBase = options.backoffBase ?? DEFAULT_BACKOFF_BASE 22 | const maxDelay = options.maxDelay ?? DEFAULT_MAX_DELAY 23 | 24 | // Note: not sure this translates perfectly to synchronous code 25 | while (true) { 26 | try { 27 | attempt += 1 28 | const result = await fn() 29 | return result 30 | } catch (error) { 31 | if (retries <= 0) throw error 32 | const isConflict = error instanceof ReadWriteConflictError 33 | if (!isConflict) throw error 34 | 35 | // If there is exponential backoff, update the delay 36 | if (exponentialBackoff) { 37 | delay = Math.min(backoffBase * 2 ** attempt, maxDelay) 38 | } 39 | 40 | // If there is jitter, randomize the delay to the current range 41 | if (jitter) { 42 | delay = randomBetween(0, delay || maxDelay) 43 | } 44 | retries -= 1 45 | 46 | if (delay) { 47 | await new Promise((resolve) => setTimeout(resolve, delay)) 48 | } 49 | } 50 | } 51 | } 52 | 53 | function randomBetween(min: number, max: number) { 54 | return Math.floor(Math.random() * (max - min + 1) + min) 55 | } 56 | -------------------------------------------------------------------------------- /src/storage/IndexedDbWithMemoryCacheTupleStorage.ts: -------------------------------------------------------------------------------- 1 | import { AsyncTupleStorageApi } from "../database/async/asyncTypes" 2 | import { IndexedDbTupleStorage } from "./IndexedDbTupleStorage" 3 | import { MemoryBTreeStorage } from "./MemoryBTreeTupleStorage" 4 | import { KeyValuePair, ScanStorageArgs, WriteOps } from "./types" 5 | 6 | export class CachedIndexedDbStorage implements AsyncTupleStorageApi { 7 | private _indexedDB: IndexedDbTupleStorage 8 | private _cache: MemoryBTreeStorage 9 | private _cacheReadyForReads = false 10 | constructor( 11 | public dbName: string, 12 | private options: { cache: boolean } = { cache: true } 13 | ) { 14 | this._indexedDB = new IndexedDbTupleStorage(dbName) 15 | this._cache = new MemoryBTreeStorage() 16 | if (options.cache) { 17 | this.initializeCacheFromIndexedDB() 18 | } 19 | } 20 | 21 | private async initializeCacheFromIndexedDB() { 22 | const results = await this._indexedDB.scan() 23 | this._cache.commit({ set: results }) 24 | this._cacheReadyForReads = true 25 | } 26 | 27 | async scan(args?: ScanStorageArgs | undefined): Promise { 28 | return this.options.cache && this._cacheReadyForReads 29 | ? this._cache.scan(args) 30 | : this._indexedDB.scan(args) 31 | } 32 | 33 | async commit(writes: WriteOps): Promise { 34 | if (this.options.cache) { 35 | this._cache.commit(writes) 36 | await this._indexedDB.commit(writes) 37 | } else { 38 | await this._indexedDB.commit(writes) 39 | } 40 | } 41 | async close(): Promise { 42 | if (this._cache) this._cache.close() 43 | await this._indexedDB.close() 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/database/transactionalWrite.ts: -------------------------------------------------------------------------------- 1 | import { KeyValuePair, WriteOps } from "../storage/types" 2 | import { 3 | AsyncTupleDatabaseClientApi, 4 | AsyncTupleTransactionApi, 5 | } from "./async/asyncTypes" 6 | import { retry } from "./retry" 7 | import { TupleDatabaseClientApi, TupleTransactionApi } from "./sync/types" 8 | import { 9 | RemoveTupleValuePairPrefix, 10 | TuplePrefix, 11 | ValueForTuple, 12 | } from "./typeHelpers" 13 | 14 | export type TransactionWriteApi = { 15 | set: ( 16 | tuple: T, 17 | value: ValueForTuple 18 | ) => TransactionWriteApi 19 | remove: (tuple: S["key"]) => TransactionWriteApi 20 | write: (writes: WriteOps) => TransactionWriteApi 21 | subspace:

>( 22 | prefix: P 23 | ) => TransactionWriteApi> 24 | } 25 | 26 | /** 27 | * Similar to transactionalReadWrite and transactionalReadWriteAsync but only allows writes. 28 | */ 29 | export function transactionalWrite( 30 | retries = 5 31 | ) { 32 | return function ( 33 | fn: (tx: TransactionWriteApi, ...args: I) => O 34 | ) { 35 | return function ( 36 | dbOrTx: 37 | | AsyncTupleDatabaseClientApi 38 | | AsyncTupleTransactionApi 39 | | TupleDatabaseClientApi 40 | | TupleTransactionApi 41 | | TransactionWriteApi, 42 | ...args: I 43 | ): O { 44 | if ("set" in dbOrTx) return fn(dbOrTx, ...args) 45 | return retry(retries, () => { 46 | const tx = dbOrTx.transact() 47 | const result = fn(tx, ...args) 48 | tx.commit() 49 | return result 50 | }) 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/database/sync/retry.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | This file is generated from async/retryAsync.ts 4 | 5 | */ 6 | 7 | type Identity = T 8 | 9 | import { ReadWriteConflictError } from "../../database/ConcurrencyLog" 10 | 11 | export type RetryOptions = { 12 | exponentialBackoff?: boolean 13 | backoffBase?: number 14 | maxDelay?: number 15 | jitter?: boolean 16 | } 17 | 18 | const DEFAULT_MAX_DELAY = 1000 19 | const DEFAULT_BACKOFF_BASE = 10 20 | 21 | export function retry( 22 | retries: number, 23 | fn: () => Identity, 24 | options: RetryOptions = {} 25 | ) { 26 | let delay = 0 27 | let attempt = 0 28 | const { exponentialBackoff, jitter } = options 29 | const backoffBase = options.backoffBase ?? DEFAULT_BACKOFF_BASE 30 | const maxDelay = options.maxDelay ?? DEFAULT_MAX_DELAY 31 | 32 | // Note: not sure this translates perfectly to synchronous code 33 | while (true) { 34 | try { 35 | attempt += 1 36 | const result = fn() 37 | return result 38 | } catch (error) { 39 | if (retries <= 0) throw error 40 | const isConflict = error instanceof ReadWriteConflictError 41 | if (!isConflict) throw error 42 | 43 | // If there is exponential backoff, update the delay 44 | if (exponentialBackoff) { 45 | delay = Math.min(backoffBase * 2 ** attempt, maxDelay) 46 | } 47 | 48 | // If there is jitter, randomize the delay to the current range 49 | if (jitter) { 50 | delay = randomBetween(0, delay || maxDelay) 51 | } 52 | retries -= 1 53 | 54 | if (delay) { 55 | new Promise((resolve) => setTimeout(resolve, delay)) 56 | } 57 | } 58 | } 59 | } 60 | 61 | function randomBetween(min: number, max: number) { 62 | return Math.floor(Math.random() * (max - min + 1) + min) 63 | } 64 | -------------------------------------------------------------------------------- /src/helpers/subspaceHelpers.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { MaxTuple } from "./sortedTupleArray" 3 | import { 4 | normalizeSubspaceScanArgs, 5 | prependPrefixToWriteOps, 6 | removePrefixFromWriteOps, 7 | } from "./subspaceHelpers" 8 | 9 | describe("subspaceHelpers", () => { 10 | describe("prependPrefixToWrites", () => { 11 | it("works", () => { 12 | expect( 13 | prependPrefixToWriteOps(["x"], { 14 | set: [ 15 | { key: ["a"], value: 1 }, 16 | { key: ["b"], value: 2 }, 17 | ], 18 | remove: [["c"]], 19 | }) 20 | ).toEqual({ 21 | set: [ 22 | { key: ["x", "a"], value: 1 }, 23 | { key: ["x", "b"], value: 2 }, 24 | ], 25 | remove: [["x", "c"]], 26 | }) 27 | }) 28 | }) 29 | 30 | describe("removePrefixFromWrites", () => { 31 | it("works", () => { 32 | expect( 33 | removePrefixFromWriteOps(["x"], { 34 | set: [ 35 | { key: ["x", "a"], value: 1 }, 36 | { key: ["x", "b"], value: 2 }, 37 | ], 38 | remove: [["x", "c"]], 39 | }) 40 | ).toEqual({ 41 | set: [ 42 | { key: ["a"], value: 1 }, 43 | { key: ["b"], value: 2 }, 44 | ], 45 | remove: [["c"]], 46 | }) 47 | }) 48 | it("throws if its the wrong prefix", () => { 49 | expect(() => { 50 | removePrefixFromWriteOps(["y"], { 51 | set: [ 52 | { key: ["x", "a"], value: 1 }, 53 | { key: ["x", "b"], value: 2 }, 54 | ], 55 | remove: [["x", "c"]], 56 | }) 57 | }).toThrow() 58 | }) 59 | }) 60 | 61 | describe("normalizeSubspaceScanArgs", () => { 62 | it("works", () => { 63 | expect(normalizeSubspaceScanArgs([1], { prefix: [2], gt: [3] })).toEqual({ 64 | gt: [1, 2, 3], 65 | lte: [1, 2, ...MaxTuple], 66 | }) 67 | }) 68 | }) 69 | }) 70 | -------------------------------------------------------------------------------- /src/database/sync/TupleDatabase.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | This file is generated from async/AsyncTupleDatabase.ts 4 | 5 | */ 6 | 7 | type Identity = T 8 | 9 | import { iterateWrittenTuples } from "../../helpers/iterateTuples" 10 | import { randomId } from "../../helpers/randomId" 11 | import { KeyValuePair, ScanStorageArgs, WriteOps } from "../../storage/types" 12 | import { ConcurrencyLog } from "../ConcurrencyLog" 13 | import { TupleStorageApi } from "../sync/types" 14 | import { TxId, Unsubscribe } from "../types" 15 | import { ReactivityTracker } from "./ReactivityTracker" 16 | import { Callback, TupleDatabaseApi } from "./types" 17 | 18 | export class TupleDatabase implements TupleDatabaseApi { 19 | constructor(private storage: TupleStorageApi) {} 20 | 21 | log = new ConcurrencyLog() 22 | reactivity = new ReactivityTracker() 23 | 24 | scan(args: ScanStorageArgs = {}, txId?: TxId): Identity { 25 | const { reverse, limit, ...bounds } = args 26 | if (txId) this.log.read(txId, bounds) 27 | return this.storage.scan({ ...bounds, reverse, limit }) 28 | } 29 | 30 | subscribe(args: ScanStorageArgs, callback: Callback): Identity { 31 | return this.reactivity.subscribe(args, callback) 32 | } 33 | 34 | commit(writes: WriteOps, txId?: string) { 35 | // Note: commit is called for transactional reads as well! 36 | const emits = this.reactivity.computeReactivityEmits(writes) 37 | 38 | if (txId) this.log.commit(txId) 39 | for (const tuple of iterateWrittenTuples(writes)) { 40 | this.log.write(txId, tuple) 41 | } 42 | this.storage.commit(writes) 43 | 44 | return this.reactivity.emit(emits, txId || randomId()) 45 | } 46 | 47 | cancel(txId: string) { 48 | this.log.cancel(txId) 49 | } 50 | 51 | close() { 52 | this.storage.close() 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@triplit/tuple-database", 3 | "version": "2.2.25", 4 | "description": "An optimized fork of ccorcos/tuple-database", 5 | "repository": "github:aspen-cloud/tuple-database", 6 | "main": "./main.js", 7 | "scripts": { 8 | "build": "run-s build:macros build:tsc", 9 | "build:tsc": "tsc", 10 | "build:macros": "bun src/tools/compileMacros.ts", 11 | "watch": "tsc -w", 12 | "test": "bun test", 13 | "test:clean": "rm -rf tmp", 14 | "test:watch": "npm test -- --watch --watch-extensions ts", 15 | "typecheck": "tsc --project tsconfig.json --noEmit", 16 | "prettier": "prettier -w src", 17 | "release": "./release.sh", 18 | "pack": "./pack.sh" 19 | }, 20 | "devDependencies": { 21 | "@types/better-sqlite3": "^7.6.3", 22 | "@types/bun": "^1.0.12", 23 | "@types/fs-extra": "^11.0.1", 24 | "@types/level": "^6.0.1", 25 | "@types/lodash": "^4.14.191", 26 | "@types/mocha": "whitecolor/mocha-types", 27 | "@types/node": "^18.11.18", 28 | "@types/react": "^18.0.26", 29 | "better-sqlite3": "^9.5.0", 30 | "expo-sqlite": "^14.0.3", 31 | "fake-indexeddb": "^4.0.1", 32 | "idb": "^7.1.1", 33 | "level": "^8.0.0", 34 | "lmdb": "^3.0.11", 35 | "mocha": "^10.2.0", 36 | "npm-run-all": "^4.1.5", 37 | "organize-imports-cli": "^0.10.0", 38 | "prettier": "^2.8.2", 39 | "react": "^18.2.0", 40 | "ts-node": "^10.9.1", 41 | "typescript": "^5.4.5" 42 | }, 43 | "peerDependencies": { 44 | "react": "*" 45 | }, 46 | "peerDependenciesMeta": { 47 | "react": { 48 | "optional": true 49 | } 50 | }, 51 | "dependencies": { 52 | "elen": "^1.0.10", 53 | "fractional-indexing": "^3.1.0", 54 | "fs-extra": "^11.1.0", 55 | "md5": "^2.3.0", 56 | "remeda": "^1.37.0", 57 | "sorted-btree": "^1.8.1", 58 | "uuid": "^9.0.0" 59 | }, 60 | "license": "MIT" 61 | } 62 | -------------------------------------------------------------------------------- /src/database/async/AsyncTupleDatabase.ts: -------------------------------------------------------------------------------- 1 | import { iterateWrittenTuples } from "../../helpers/iterateTuples" 2 | import { randomId } from "../../helpers/randomId" 3 | import { KeyValuePair, ScanStorageArgs, WriteOps } from "../../storage/types" 4 | import { ConcurrencyLog } from "../ConcurrencyLog" 5 | import { TupleStorageApi } from "../sync/types" 6 | import { TxId, Unsubscribe } from "../types" 7 | import { AsyncReactivityTracker } from "./AsyncReactivityTracker" 8 | import { 9 | AsyncCallback, 10 | AsyncTupleDatabaseApi, 11 | AsyncTupleStorageApi, 12 | } from "./asyncTypes" 13 | 14 | export class AsyncTupleDatabase implements AsyncTupleDatabaseApi { 15 | constructor(private storage: TupleStorageApi | AsyncTupleStorageApi) {} 16 | 17 | log = new ConcurrencyLog() 18 | reactivity = new AsyncReactivityTracker() 19 | 20 | async scan(args: ScanStorageArgs = {}, txId?: TxId): Promise { 21 | const { reverse, limit, ...bounds } = args 22 | if (txId) this.log.read(txId, bounds) 23 | return this.storage.scan({ ...bounds, reverse, limit }) 24 | } 25 | 26 | async subscribe( 27 | args: ScanStorageArgs, 28 | callback: AsyncCallback 29 | ): Promise { 30 | return this.reactivity.subscribe(args, callback) 31 | } 32 | 33 | async commit(writes: WriteOps, txId?: string) { 34 | // Note: commit is called for transactional reads as well! 35 | const emits = this.reactivity.computeReactivityEmits(writes) 36 | 37 | if (txId) this.log.commit(txId) 38 | for (const tuple of iterateWrittenTuples(writes)) { 39 | this.log.write(txId, tuple) 40 | } 41 | await this.storage.commit(writes) 42 | 43 | return this.reactivity.emit(emits, txId || randomId()) 44 | } 45 | 46 | async cancel(txId: string) { 47 | this.log.cancel(txId) 48 | } 49 | 50 | async close() { 51 | await this.storage.close() 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/storage/MemoryBTreeTupleStorage.ts: -------------------------------------------------------------------------------- 1 | import BTree from "sorted-btree" 2 | import { 3 | KeyValuePair, 4 | MAX, 5 | MIN, 6 | ScanStorageArgs, 7 | Tuple, 8 | WriteOps, 9 | } from "./types" 10 | import { compareTuple } from "../helpers/compareTuple" 11 | import { TupleStorageApi } from "../database/sync/types" 12 | 13 | // Hack for https://github.com/qwertie/btree-typescript/issues/36 14 | // @ts-ignore 15 | const BTreeClass = (BTree.default ? BTree.default : BTree) as typeof BTree 16 | export class MemoryBTreeStorage implements TupleStorageApi { 17 | btree: BTree 18 | constructor() { 19 | this.btree = new BTreeClass(undefined, compareTuple) 20 | } 21 | scan(args?: ScanStorageArgs | undefined): KeyValuePair[] { 22 | const low = args?.gte ?? args?.gt ?? MIN 23 | const high = args?.lte ?? args?.lt ?? MAX 24 | const results: KeyValuePair[] = [] 25 | // TODO use entries and entriesReversed instead? 26 | this.btree.forRange(low, high, args?.lte != null, (key, value, n) => { 27 | // if using gt (greater than) then skip equal keys 28 | if (args?.gt && compareTuple(key, args.gt) === 0) return 29 | results.push({ key, value }) 30 | if ( 31 | args?.reverse !== true && 32 | results.length >= (args?.limit ?? Infinity) 33 | ) { 34 | return { break: true } 35 | } 36 | }) 37 | 38 | if (args?.reverse) results.reverse() 39 | if (args?.limit) return results.slice(0, args.limit) 40 | return results 41 | } 42 | commit(writes: WriteOps): void { 43 | const { set, remove } = writes 44 | for (const tuple of remove || []) { 45 | this.btree.delete(tuple) 46 | } 47 | for (const { key, value } of set || []) { 48 | this.btree.set(key, value, true) 49 | } 50 | } 51 | close(): void {} 52 | 53 | wipe(): void { 54 | this.btree = new BTreeClass(undefined, compareTuple) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/storage/LevelTupleStorage.ts: -------------------------------------------------------------------------------- 1 | import { AbstractBatch } from "abstract-leveldown" 2 | import { Level } from "level" 3 | import { AsyncTupleStorageApi } from "../database/async/asyncTypes" 4 | import { 5 | decodeTuple, 6 | decodeValue, 7 | encodeTuple, 8 | encodeValue, 9 | } from "../helpers/codec" 10 | import { KeyValuePair, ScanStorageArgs, WriteOps } from "./types" 11 | 12 | export class LevelTupleStorage implements AsyncTupleStorageApi { 13 | /** 14 | * import level from "level" 15 | * new LevelTupleStorage(level("path/to.db")) 16 | */ 17 | constructor(public db: Level) {} 18 | 19 | async scan(args: ScanStorageArgs = {}): Promise { 20 | const dbArgs: any = {} 21 | if (args.gt !== undefined) dbArgs.gt = encodeTuple(args.gt) 22 | if (args.gte !== undefined) dbArgs.gte = encodeTuple(args.gte) 23 | if (args.lt !== undefined) dbArgs.lt = encodeTuple(args.lt) 24 | if (args.lte !== undefined) dbArgs.lte = encodeTuple(args.lte) 25 | if (args.limit !== undefined) dbArgs.limit = args.limit 26 | if (args.reverse !== undefined) dbArgs.reverse = args.reverse 27 | 28 | const results: KeyValuePair[] = [] 29 | for await (const [key, value] of this.db.iterator(dbArgs)) { 30 | results.push({ 31 | key: decodeTuple(key), 32 | value: decodeValue(value), 33 | }) 34 | } 35 | return results 36 | } 37 | 38 | async commit(writes: WriteOps): Promise { 39 | const ops = [ 40 | ...(writes.remove || []).map( 41 | (tuple) => 42 | ({ 43 | type: "del", 44 | key: encodeTuple(tuple), 45 | } as AbstractBatch) 46 | ), 47 | ...(writes.set || []).map( 48 | ({ key, value }) => 49 | ({ 50 | type: "put", 51 | key: encodeTuple(key), 52 | value: encodeValue(value), 53 | } as AbstractBatch) 54 | ), 55 | ] 56 | 57 | await this.db.batch(ops) 58 | } 59 | 60 | async close(): Promise { 61 | return this.db.close() 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/helpers/binarySearch.test.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Binary Search Tests. 4 | ./node_modules/.bin/mocha -r ts-node/register ./src/helpers/binarySearch.test.ts 5 | 6 | */ 7 | 8 | import { describe, it, expect } from "bun:test" 9 | import { binarySearch, binarySearchAssociativeList } from "./binarySearch" 10 | import { compare } from "./compare" 11 | 12 | describe("binarySearch", () => { 13 | const list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] 14 | it("find before", () => { 15 | const result = binarySearch(list, -1, compare) 16 | expect(result.found).toBeUndefined() 17 | expect(result.closest).toEqual(0) 18 | }) 19 | it("find after", () => { 20 | const result = binarySearch(list, 10, compare) 21 | expect(result.found).toBeUndefined() 22 | expect(result.closest).toEqual(10) 23 | }) 24 | it("find middle", () => { 25 | const result = binarySearch(list, 1.5, compare) 26 | expect(result.found).toBeUndefined() 27 | expect(result.closest).toEqual(2) 28 | }) 29 | it("find exact", () => { 30 | const result = binarySearch(list, 5, compare) 31 | expect(result.found).toEqual(5) 32 | expect(result.closest).toBeUndefined() 33 | }) 34 | }) 35 | 36 | describe("binarySearchAssociativeList", () => { 37 | // An associative array. 38 | const list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9].map( 39 | (n) => [n, {}] as [number, any] 40 | ) 41 | it("find before", () => { 42 | const result = binarySearchAssociativeList(list, -1, compare) 43 | expect(result.found).toBeUndefined() 44 | expect(result.closest).toEqual(0) 45 | }) 46 | it("find after", () => { 47 | const result = binarySearchAssociativeList(list, 10, compare) 48 | expect(result.found).toBeUndefined() 49 | expect(result.closest).toEqual(10) 50 | }) 51 | it("find middle", () => { 52 | const result = binarySearchAssociativeList(list, 1.5, compare) 53 | expect(result.found).toBeUndefined() 54 | expect(result.closest).toEqual(2) 55 | }) 56 | it("find exact", () => { 57 | const result = binarySearchAssociativeList(list, 5, compare) 58 | expect(result.found).toEqual(5) 59 | expect(result.closest).toBeUndefined() 60 | }) 61 | }) 62 | -------------------------------------------------------------------------------- /src/storage/ExpoSQLiteLegacyStorage.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | 3 | import * as SQLite from "expo-sqlite" 4 | import { 5 | AsyncSQLiteAdapter, 6 | AsyncSQLiteExecutor, 7 | AsyncAdapterSQLiteStorage, 8 | AdapterSQLiteOptions, 9 | } from "./AdapterSQLiteStorage" 10 | import { AsyncTupleStorageApi } from "../database/async/asyncTypes" 11 | 12 | export class ExpoSQLiteLegacyTupleStorage implements AsyncTupleStorageApi { 13 | private db: SQLite.SQLiteDatabase 14 | private store: AsyncAdapterSQLiteStorage 15 | 16 | constructor(name: string, options?: AdapterSQLiteOptions) 17 | constructor(db: SQLite.SQLiteDatabase, options?: AdapterSQLiteOptions) 18 | constructor( 19 | arg0: string | SQLite.SQLiteDatabase, 20 | options: AdapterSQLiteOptions = {} 21 | ) { 22 | if (typeof arg0 === "string") { 23 | this.db = SQLite.openDatabase(arg0) 24 | } else { 25 | this.db = arg0 26 | } 27 | this.store = new AsyncAdapterSQLiteStorage( 28 | new ExpoSQLiteLegacyAdapter(this.db), 29 | options 30 | ) 31 | } 32 | 33 | scan: AsyncTupleStorageApi["scan"] = async (args = {}) => { 34 | return this.store.scan(args) 35 | } 36 | 37 | commit: AsyncTupleStorageApi["commit"] = async (ops) => { 38 | return this.store.commit(ops) 39 | } 40 | 41 | close: AsyncTupleStorageApi["close"] = async () => { 42 | return this.store.close() 43 | } 44 | } 45 | 46 | class ExpoSQLiteLegacyAdapter implements AsyncSQLiteAdapter { 47 | constructor(private db: SQLite.SQLiteDatabase) {} 48 | async execute(sql: string, args?: any[] | undefined) { 49 | return (await this.db.execAsync([{ sql, args: args ?? [] }], false))[0] 50 | } 51 | normalizeResults(results: any): { key: string; value: string }[] { 52 | if (!results.rows) return [] 53 | return results.rows as { key: string; value: string }[] 54 | } 55 | async transact(fn: (adapter: AsyncSQLiteExecutor) => Promise) { 56 | await this.db.transactionAsync(async (tx) => { 57 | await fn({ 58 | execute: async (sql, args) => { 59 | return await tx.executeSqlAsync(sql, args ?? []) 60 | }, 61 | }) 62 | }) 63 | } 64 | async close() { 65 | await this.db.closeAsync() 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/storage/FileTupleStorage.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs-extra" 2 | import * as path from "path" 3 | import { InMemoryTupleStorage } from "./InMemoryTupleStorage" 4 | import { KeyValuePair, WriteOps } from "./types" 5 | 6 | export function parseFile(str: string): KeyValuePair[] { 7 | if (str === "") { 8 | return [] 9 | } 10 | return str.split("\n").map((line) => { 11 | const pair = JSON.parse(line) 12 | // Backward compatibility with [key, value]. 13 | if (Array.isArray(pair)) { 14 | const [key, value] = pair 15 | return { key, value } 16 | } 17 | return pair 18 | }) 19 | } 20 | 21 | function serializeFile(data: KeyValuePair[]) { 22 | return data.map((pair) => JSON.stringify(pair)).join("\n") 23 | } 24 | 25 | export class FileTupleStorage extends InMemoryTupleStorage { 26 | cache: FileCache 27 | 28 | // This is pretty bonkers: https://github.com/Microsoft/TypeScript/issues/8277 29 | // @ts-ignore 30 | constructor(public dbPath: string) { 31 | const cache = new FileCache(dbPath) 32 | super(cache.get()) 33 | this.cache = cache 34 | } 35 | 36 | commit(writes: WriteOps) { 37 | super.commit(writes) 38 | this.cache.set(this.data) 39 | } 40 | } 41 | 42 | class FileCache { 43 | constructor(private dbPath: string) {} 44 | 45 | private getFilePath() { 46 | return this.dbPath + ".txt" 47 | } 48 | 49 | get() { 50 | // Check that the file exists. 51 | const filePath = this.getFilePath() 52 | try { 53 | const stat = fs.statSync(filePath) 54 | if (!stat.isFile()) { 55 | throw new Error("Database is not a file.") 56 | } 57 | } catch (error) { 58 | if (error.code === "ENOENT") { 59 | // File does not exist. 60 | return [] 61 | } 62 | throw error 63 | } 64 | 65 | const fileContents = fs.readFileSync(filePath, "utf8") 66 | const data = parseFile(fileContents) 67 | return data 68 | } 69 | 70 | // TODO: throttle this call if it makes sense. 71 | set(data: KeyValuePair[]) { 72 | const filePath = this.getFilePath() 73 | const fileContents = serializeFile(data) 74 | fs.mkdirpSync(path.dirname(this.dbPath)) 75 | fs.writeFileSync(filePath, fileContents, "utf8") 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/storage/ExpoSQLiteStorage.ts: -------------------------------------------------------------------------------- 1 | import * as SQLite from "expo-sqlite" 2 | import { 3 | AsyncSQLiteAdapter, 4 | AsyncSQLiteExecutor, 5 | AsyncAdapterSQLiteStorage, 6 | AdapterSQLiteOptions, 7 | } from "./AdapterSQLiteStorage" 8 | import { AsyncTupleStorageApi } from "../database/async/asyncTypes" 9 | 10 | export class ExpoSQLiteTupleStorage implements AsyncTupleStorageApi { 11 | private storeReady: Promise 12 | 13 | constructor(name: string, options?: AdapterSQLiteOptions) 14 | constructor(db: SQLite.SQLiteDatabase, options?: AdapterSQLiteOptions) 15 | constructor( 16 | arg0: string | SQLite.SQLiteDatabase, 17 | options: AdapterSQLiteOptions = {} 18 | ) { 19 | if (typeof arg0 === "string") { 20 | this.storeReady = SQLite.openDatabaseAsync(arg0).then((db) => { 21 | return new AsyncAdapterSQLiteStorage(new ExpoSQLiteAdapter(db), options) 22 | }) 23 | } else { 24 | this.storeReady = Promise.resolve( 25 | new AsyncAdapterSQLiteStorage(new ExpoSQLiteAdapter(arg0), options) 26 | ) 27 | } 28 | } 29 | 30 | scan: AsyncTupleStorageApi["scan"] = async (args = {}) => { 31 | const store = await this.storeReady 32 | return store.scan(args) 33 | } 34 | 35 | commit: AsyncTupleStorageApi["commit"] = async (ops) => { 36 | const store = await this.storeReady 37 | return store.commit(ops) 38 | } 39 | 40 | close: AsyncTupleStorageApi["close"] = async () => { 41 | const store = await this.storeReady 42 | return store.close() 43 | } 44 | } 45 | 46 | class ExpoSQLiteAdapter implements AsyncSQLiteAdapter { 47 | constructor(private db: SQLite.SQLiteDatabase) {} 48 | async execute(sql: string, args?: any[] | undefined) { 49 | return await this.db.getAllAsync(sql, args ?? []) 50 | } 51 | normalizeResults(results: any): { key: string; value: string }[] { 52 | if (!results) return [] 53 | return results as { key: string; value: string }[] 54 | } 55 | async transact(fn: (adapter: AsyncSQLiteExecutor) => Promise) { 56 | await this.db.withTransactionAsync(async () => { 57 | await fn({ 58 | execute: async (sql, args) => { 59 | return await this.db.getAllAsync(sql, args ?? []) 60 | }, 61 | }) 62 | }) 63 | } 64 | async close() { 65 | await this.db.closeAsync() 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/helpers/naturalSort.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { invertString } from "./invertString" 3 | 4 | type ParseTest = [string, number] 5 | 6 | const digitsTest: ParseTest[] = [ 7 | ["1", 1], 8 | ["2", 2], 9 | ["20", 20], 10 | ["24", 24], 11 | ["024", 24], 12 | ["0024", 24], 13 | ] 14 | 15 | const decimalsTest: ParseTest[] = [ 16 | ["0.1", 0.1], 17 | [".1", 0.1], 18 | [".12", 0.12], 19 | ["12.12", 12.12], 20 | ["012.1200", 12.12], 21 | ] 22 | 23 | function toNegative([a, b]: ParseTest): ParseTest { 24 | return ["-" + a, -1 * b] 25 | } 26 | 27 | function toPositive([a, b]: ParseTest): ParseTest { 28 | return ["+" + a, b] 29 | } 30 | 31 | const negativeDigits: ParseTest[] = digitsTest.map(toNegative) 32 | const positiveDigits: ParseTest[] = digitsTest.map(toPositive) 33 | 34 | const negativeDecimals: ParseTest[] = decimalsTest.map(toNegative) 35 | const positiveDecimals: ParseTest[] = decimalsTest.map(toPositive) 36 | 37 | const scientificNotation: ParseTest[] = [ 38 | ["1e3", 1000], 39 | ["1e-2", 0.01], 40 | ["1.2e-2", 0.012], 41 | ["1.20e-2", 0.012], 42 | ["01.20e-2", 0.012], 43 | ["10.20e-2", 0.102], 44 | ["10.20e2", 1020], 45 | ] 46 | 47 | const negativeScientificNotation: ParseTest[] = 48 | scientificNotation.map(toNegative) 49 | const positiveScientificNotation: ParseTest[] = 50 | scientificNotation.map(toPositive) 51 | 52 | const commaNumbers: ParseTest[] = [ 53 | ["1,000", 1000], 54 | ["1,000.4", 1000.4], 55 | ["-1,000,000.4", -1000000.4], 56 | ["-1,000,000.4e-6", -1.0000004], 57 | ["+1,000e3", -1000000], 58 | ] 59 | 60 | describe("parseNumber", () => { 61 | const data = [ 62 | "aaa", 63 | "aab", 64 | "aac", 65 | "aba", 66 | "abc", 67 | "aca", 68 | "acc", 69 | "bbb", 70 | "bca", 71 | "bcb", 72 | "caa", 73 | "cab", 74 | "ccc", 75 | ] 76 | 77 | it("can encode and decode properly", () => { 78 | for (const str of data) { 79 | expect(invertString(invertString(str))).toStrictEqual(str) 80 | } 81 | }) 82 | 83 | it("inversion is reverse sorted", () => { 84 | const sorted = [...data].sort() 85 | expect(sorted).toStrictEqual(data) 86 | 87 | const inverseSorted = sorted.map(invertString).sort().map(invertString) 88 | expect(inverseSorted).toStrictEqual(sorted.reverse()) 89 | }) 90 | }) 91 | -------------------------------------------------------------------------------- /src/helpers/sortedTupleValuePairs.ts: -------------------------------------------------------------------------------- 1 | import { ScanArgs } from "../database/types" 2 | import { KeyValuePair, Tuple } from "../storage/types" 3 | import { compareTuple } from "./compareTuple" 4 | import * as sortedList from "./sortedList" 5 | import { normalizeTupleBounds } from "./sortedTupleArray" 6 | 7 | export function compareTupleValuePair(a: KeyValuePair, b: KeyValuePair) { 8 | return compareTuple(a.key, b.key) 9 | } 10 | 11 | function compareTupleValuePairReverse(a: KeyValuePair, b: KeyValuePair) { 12 | return compareTuple(a.key, b.key) * -1 13 | } 14 | 15 | export function set( 16 | data: KeyValuePair[], 17 | key: Tuple, 18 | value: any, 19 | reverse = false 20 | ) { 21 | return sortedList.set( 22 | data, 23 | { key, value }, 24 | reverse ? compareTupleValuePairReverse : compareTupleValuePair 25 | ) 26 | } 27 | 28 | export function remove(data: KeyValuePair[], key: Tuple, reverse = false) { 29 | return sortedList.remove( 30 | data, 31 | { key, value: null }, 32 | reverse ? compareTupleValuePairReverse : compareTupleValuePair 33 | ) 34 | } 35 | 36 | export function get(data: KeyValuePair[], key: Tuple, reverse = false) { 37 | const pair = sortedList.get( 38 | data, 39 | { key, value: null }, 40 | reverse ? compareTupleValuePairReverse : compareTupleValuePair 41 | ) 42 | if (pair !== undefined) return pair.value 43 | } 44 | 45 | export function exists(data: KeyValuePair[], key: Tuple, reverse = false) { 46 | return sortedList.exists( 47 | data, 48 | { key, value: null }, 49 | reverse ? compareTupleValuePairReverse : compareTupleValuePair 50 | ) 51 | } 52 | 53 | function normalizeTupleValuePairBounds(args: ScanArgs) { 54 | const bounds = normalizeTupleBounds(args) 55 | const { gt, lt, gte, lte } = bounds 56 | return { 57 | gt: gt ? ({ key: gt, value: null } as KeyValuePair) : undefined, 58 | gte: gte ? ({ key: gte, value: null } as KeyValuePair) : undefined, 59 | lt: lt ? ({ key: lt, value: null } as KeyValuePair) : undefined, 60 | lte: lte ? ({ key: lte, value: null } as KeyValuePair) : undefined, 61 | } 62 | } 63 | 64 | export function scan(data: KeyValuePair[], args: ScanArgs = {}) { 65 | const { limit, reverse, ...rest } = args 66 | const bounds = normalizeTupleValuePairBounds(rest) 67 | return sortedList.scan( 68 | data, 69 | { limit, reverse, ...bounds }, 70 | compareTupleValuePair 71 | ) 72 | } 73 | -------------------------------------------------------------------------------- /src/storage/IndexedDbTupleStorage.ts: -------------------------------------------------------------------------------- 1 | import { IDBPDatabase, openDB } from "idb/with-async-ittr" 2 | import { decodeTuple, encodeTuple } from "../helpers/codec" 3 | import { AsyncTupleStorageApi, ScanStorageArgs, WriteOps } from "../main" 4 | import { KeyValuePair } from "./types" 5 | 6 | const version = 1 7 | 8 | const storeName = "tupledb" 9 | 10 | export class IndexedDbTupleStorage implements AsyncTupleStorageApi { 11 | private db: Promise> 12 | 13 | constructor(public dbName: string) { 14 | this.db = openDB(dbName, version, { 15 | upgrade(db) { 16 | db.createObjectStore(storeName) 17 | }, 18 | }) 19 | } 20 | 21 | async scan(args?: ScanStorageArgs) { 22 | const db = await this.db 23 | const tx = db.transaction(storeName, "readonly", { durability: "relaxed" }) 24 | const index = tx.store // primary key 25 | 26 | const lower = args?.gt || args?.gte 27 | const lowerEq = Boolean(args?.gte) 28 | 29 | const upper = args?.lt || args?.lte 30 | const upperEq = Boolean(args?.lte) 31 | 32 | let range: IDBKeyRange | null 33 | if (upper) { 34 | if (lower) { 35 | range = IDBKeyRange.bound( 36 | encodeTuple(lower), 37 | encodeTuple(upper), 38 | !lowerEq, 39 | !upperEq 40 | ) 41 | } else { 42 | range = IDBKeyRange.upperBound(encodeTuple(upper), !upperEq) 43 | } 44 | } else { 45 | if (lower) { 46 | range = IDBKeyRange.lowerBound(encodeTuple(lower), !lowerEq) 47 | } else { 48 | range = null 49 | } 50 | } 51 | 52 | const direction: IDBCursorDirection = args?.reverse ? "prev" : "next" 53 | 54 | const limit = args?.limit || Infinity 55 | let results: KeyValuePair[] = [] 56 | for await (const cursor of index.iterate(range, direction)) { 57 | results.push({ 58 | key: decodeTuple(cursor.key), 59 | value: cursor.value, 60 | }) 61 | if (results.length >= limit) break 62 | } 63 | await tx.done 64 | 65 | return results 66 | } 67 | 68 | async commit(writes: WriteOps) { 69 | const db = await this.db 70 | const tx = db.transaction(storeName, "readwrite", { durability: "relaxed" }) 71 | for (const { key, value } of writes.set || []) { 72 | tx.store.put(value, encodeTuple(key)) 73 | } 74 | for (const key of writes.remove || []) { 75 | tx.store.delete(encodeTuple(key)) 76 | } 77 | await tx.done 78 | } 79 | 80 | async close() { 81 | const db = await this.db 82 | db.close() 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/database/async/subscribeQueryAsync.ts: -------------------------------------------------------------------------------- 1 | import { isEmptyWrites } from "../../helpers/isEmptyWrites" 2 | import { Queue } from "../../helpers/Queue" 3 | import { KeyValuePair } from "../../storage/types" 4 | import { TxId } from "../types" 5 | import { AsyncTupleDatabaseClient } from "./AsyncTupleDatabaseClient" 6 | import { AsyncTupleDatabaseClientApi } from "./asyncTypes" 7 | 8 | const throwError = () => { 9 | throw new Error() 10 | } 11 | 12 | export async function subscribeQueryAsync( 13 | db: AsyncTupleDatabaseClientApi, 14 | fn: (db: AsyncTupleDatabaseClientApi) => Promise, 15 | callback: (result: T) => void 16 | ): Promise<{ result: T; destroy: () => void }> { 17 | let destroyed = false 18 | const listeners = new Set() 19 | 20 | const compute = () => fn(listenDb) 21 | 22 | const resetListeners = () => { 23 | listeners.forEach((destroy) => destroy()) 24 | listeners.clear() 25 | } 26 | 27 | let lastComputedTxId: string | undefined 28 | 29 | const recompute = async (txId: TxId) => { 30 | if (destroyed) return 31 | // Skip over duplicate emits. 32 | if (txId === lastComputedTxId) return 33 | 34 | // Recompute. 35 | lastComputedTxId = txId 36 | resetListeners() 37 | const result = await compute() 38 | callback(result) 39 | } 40 | 41 | const recomputeQueue = new Queue() 42 | 43 | // Subscribe for every scan that gets called. 44 | const listenDb = new AsyncTupleDatabaseClient({ 45 | scan: async (args: any, txId) => { 46 | // if (txId) 47 | // // Maybe one day we can transactionally subscribe to a bunch of things. But 48 | // // for now, lets just avoid that... 49 | // throw new Error("Not allowed to subscribe transactionally.") 50 | 51 | const destroy = await db.subscribe(args, async (_writes, txId) => 52 | recomputeQueue.enqueue(() => recompute(txId)) 53 | ) 54 | listeners.add(destroy) 55 | 56 | const results = await db.scan(args) 57 | return results 58 | }, 59 | cancel: async (txId) => { 60 | await db.cancel(txId) 61 | }, 62 | commit: async (writes, txId) => { 63 | if (!isEmptyWrites(writes)) 64 | throw new Error("No writing in a subscribeQueryAsync.") 65 | // Commit to resolve conflicts with transactional reads. 66 | await db.commit({}, txId) 67 | }, 68 | subscribe: throwError, 69 | close: throwError, 70 | }) 71 | 72 | const result = await compute() 73 | const destroy = () => { 74 | resetListeners() 75 | destroyed = true 76 | } 77 | return { result, destroy } 78 | } 79 | -------------------------------------------------------------------------------- /src/database/sync/subscribeQuery.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | This file is generated from async/subscribeQueryAsync.ts 4 | 5 | */ 6 | 7 | type Identity = T 8 | 9 | import { isEmptyWrites } from "../../helpers/isEmptyWrites" 10 | import { Queue } from "../../helpers/Queue" 11 | import { KeyValuePair } from "../../storage/types" 12 | import { TxId } from "../types" 13 | import { TupleDatabaseClient } from "./TupleDatabaseClient" 14 | import { TupleDatabaseClientApi } from "./types" 15 | 16 | const throwError = () => { 17 | throw new Error() 18 | } 19 | 20 | export function subscribeQuery( 21 | db: TupleDatabaseClientApi, 22 | fn: (db: TupleDatabaseClientApi) => Identity, 23 | callback: (result: T) => void 24 | ): Identity<{ result: T; destroy: () => void }> { 25 | let destroyed = false 26 | const listeners = new Set() 27 | 28 | const compute = () => fn(listenDb) 29 | 30 | const resetListeners = () => { 31 | listeners.forEach((destroy) => destroy()) 32 | listeners.clear() 33 | } 34 | 35 | let lastComputedTxId: string | undefined 36 | 37 | const recompute = (txId: TxId) => { 38 | if (destroyed) return 39 | // Skip over duplicate emits. 40 | if (txId === lastComputedTxId) return 41 | 42 | // Recompute. 43 | lastComputedTxId = txId 44 | resetListeners() 45 | const result = compute() 46 | callback(result) 47 | } 48 | 49 | const recomputeQueue = new Queue() 50 | 51 | // Subscribe for every scan that gets called. 52 | const listenDb = new TupleDatabaseClient({ 53 | scan: (args: any, txId) => { 54 | // if (txId) 55 | // // Maybe one day we can transactionally subscribe to a bunch of things. But 56 | // // for now, lets just avoid that... 57 | // throw new Error("Not allowed to subscribe transactionally.") 58 | 59 | const destroy = db.subscribe(args, (_writes, txId) => 60 | recomputeQueue.enqueue(() => recompute(txId)) 61 | ) 62 | listeners.add(destroy) 63 | 64 | const results = db.scan(args) 65 | return results 66 | }, 67 | cancel: (txId) => { 68 | db.cancel(txId) 69 | }, 70 | commit: (writes, txId) => { 71 | if (!isEmptyWrites(writes)) 72 | throw new Error("No writing in a subscribeQuery.") 73 | // Commit to resolve conflicts with transactional reads. 74 | db.commit({}, txId) 75 | }, 76 | subscribe: throwError, 77 | close: throwError, 78 | }) 79 | 80 | const result = compute() 81 | const destroy = () => { 82 | resetListeners() 83 | destroyed = true 84 | } 85 | return { result, destroy } 86 | } 87 | -------------------------------------------------------------------------------- /src/helpers/isBoundsWithinBounds.ts: -------------------------------------------------------------------------------- 1 | import { Tuple } from "../storage/types" 2 | import { compareTuple } from "./compareTuple" 3 | import { Bounds } from "./sortedTupleArray" 4 | 5 | function isLessThanOrEqualTo(a: Tuple, b: Tuple) { 6 | return compareTuple(a, b) !== 1 7 | } 8 | 9 | function isLessThan(a: Tuple, b: Tuple) { 10 | return compareTuple(a, b) === -1 11 | } 12 | 13 | function isGreaterThanOrEqualTo(a: Tuple, b: Tuple) { 14 | return compareTuple(a, b) !== -1 15 | } 16 | 17 | function isGreaterThan(a: Tuple, b: Tuple) { 18 | return compareTuple(a, b) === 1 19 | } 20 | 21 | export function isBoundsWithinBounds(args: { 22 | bounds: Bounds 23 | container: Bounds 24 | }) { 25 | const { bounds, container } = args 26 | if (container.gt) { 27 | if (bounds.gt) { 28 | if (!isGreaterThanOrEqualTo(bounds.gt, container.gt)) return false 29 | } 30 | if (bounds.gte) { 31 | if (!isGreaterThan(bounds.gte, container.gt)) return false 32 | } 33 | } 34 | 35 | if (container.gte) { 36 | if (bounds.gt) { 37 | if (!isGreaterThanOrEqualTo(bounds.gt, container.gte)) return false 38 | } 39 | if (bounds.gte) { 40 | if (!isGreaterThanOrEqualTo(bounds.gte, container.gte)) return false 41 | } 42 | } 43 | 44 | if (container.lt) { 45 | if (bounds.lt) { 46 | if (!isLessThanOrEqualTo(bounds.lt, container.lt)) return false 47 | } 48 | if (bounds.lte) { 49 | if (!isLessThan(bounds.lte, container.lt)) return false 50 | } 51 | } 52 | 53 | if (container.lte) { 54 | if (bounds.lt) { 55 | if (!isLessThanOrEqualTo(bounds.lt, container.lte)) return false 56 | } 57 | if (bounds.lte) { 58 | if (!isLessThanOrEqualTo(bounds.lte, container.lte)) return false 59 | } 60 | } 61 | 62 | // if (bounds.lt) { 63 | // if (container.lt && !isLessThanOrEqualTo(bounds.lt, container.lt)) 64 | // return false 65 | // if (container.lte && !isLessThanOrEqualTo(bounds.lt, container.lte)) 66 | // return false 67 | // } 68 | // if (bounds.lte) { 69 | // if (container.lt && isLessThan(bounds.lte, container.lt)) return false 70 | // if (container.lte && isLessThanOrEqualTo(bounds.lte, container.lte)) 71 | // return false 72 | // } 73 | 74 | // if (bounds.gt) { 75 | // if (container.gt && !isGreaterThanOrEqualTo(bounds.gt, container.gt)) 76 | // return false 77 | // if (container.gte && !isGreaterThanOrEqualTo(bounds.gt, container.gte)) 78 | // return false 79 | // } 80 | // if (bounds.gte) { 81 | // if (container.gt && isGreaterThan(bounds.gte, container.gt)) return false 82 | // if (container.gte && isGreaterThanOrEqualTo(bounds.gte, container.gte)) 83 | // return false 84 | // } 85 | 86 | return true 87 | } 88 | -------------------------------------------------------------------------------- /src/helpers/queryBuilder.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { AsyncTupleDatabaseClient } from "../database/async/AsyncTupleDatabaseClient" 3 | import { TupleDatabase } from "../database/sync/TupleDatabase" 4 | import { TupleDatabaseClient } from "../database/sync/TupleDatabaseClient" 5 | import { InMemoryTupleStorage } from "../storage/InMemoryTupleStorage" 6 | import { Value } from "../storage/types" 7 | import { DelayDb } from "./DelayDb" 8 | import { execute, QueryBuilder } from "./queryBuilder" 9 | 10 | export type Order = number 11 | export type Fact = [string, string, Order, Value] 12 | export type Triple = [string, string, Value] 13 | 14 | export type TriplestoreSchema = 15 | | { key: ["eaov", ...Fact]; value: null } 16 | | { key: ["aveo", string, Value, string, Order]; value: null } 17 | | { key: ["veao", Value, string, string, Order]; value: null } 18 | 19 | const q = new QueryBuilder() 20 | 21 | const getNextOrder = (e: string, a: string) => 22 | q 23 | .subspace(["eaov", e, a]) 24 | .scan({ reverse: true, limit: 1 }) 25 | .map((results) => { 26 | const lastOrder = results.map(({ key: [o, _v] }) => o)[0] 27 | const nextOrder = typeof lastOrder === "number" ? lastOrder + 1 : 0 28 | return nextOrder 29 | }) 30 | 31 | const writeFact = (fact: Fact) => { 32 | const [e, a, o, v] = fact 33 | return q.write({ 34 | set: [ 35 | { key: ["eaov", e, a, o, v], value: null }, 36 | { key: ["aveo", a, v, e, o], value: null }, 37 | { key: ["veao", v, e, a, o], value: null }, 38 | ], 39 | }) 40 | } 41 | 42 | const appendTriple = ([e, a, v]: Triple) => 43 | getNextOrder(e, a).chain((nextOrder) => { 44 | return writeFact([e, a, nextOrder, v]) 45 | }) 46 | 47 | describe("queryBuilder", () => { 48 | it("works", () => { 49 | const db = new TupleDatabaseClient( 50 | new TupleDatabase(new InMemoryTupleStorage()) 51 | ) 52 | 53 | expect(execute(db, getNextOrder("chet", "color"))).toBe(0) 54 | execute(db, appendTriple(["chet", "color", "red"])) 55 | 56 | expect(execute(db, getNextOrder("chet", "color"))).toBe(1) 57 | execute(db, appendTriple(["chet", "color", "blue"])) 58 | }) 59 | 60 | it("works async", async () => { 61 | const db = new AsyncTupleDatabaseClient( 62 | DelayDb(new TupleDatabase(new InMemoryTupleStorage()), 10) 63 | ) 64 | 65 | expect(await execute(db, getNextOrder("chet", "color"))).toBe(0) 66 | await execute(db, appendTriple(["chet", "color", "red"])) 67 | 68 | expect(await execute(db, getNextOrder("chet", "color"))).toBe(1) 69 | await execute(db, appendTriple(["chet", "color", "blue"])) 70 | }) 71 | }) 72 | -------------------------------------------------------------------------------- /src/tools/compileMacros.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | ./node_modules/.bin/ts-node src/tools/compileMacros.ts 4 | 5 | */ 6 | 7 | import { execSync } from "child_process" 8 | import * as fs from "fs-extra" 9 | import * as path from "path" 10 | 11 | const rootPath = path.resolve(__dirname, "../..") 12 | 13 | function convertAsyncToSync(contents: string) { 14 | // Collapse union types. 15 | contents = contents.replace( 16 | /AsyncTupleStorageApi \| TupleStorageApi/g, 17 | "TupleStorageApi" 18 | ) 19 | contents = contents.replace( 20 | /TupleStorageApi \| AsyncTupleStorageApi/g, 21 | "TupleStorageApi" 22 | ) 23 | 24 | contents = contents.replace("const isSync = false", "const isSync = true") 25 | 26 | // Maintain camelcase 27 | contents = contents.replace(/async(.)/g, (x) => x.toLowerCase()) 28 | 29 | // Promise.all 30 | contents = contents.replace(/Promise\.all/g, "") 31 | 32 | // Remove async 33 | contents = contents.replace(/[Aa]sync/g, "") 34 | contents = contents.replace(/await/g, "") 35 | // Return "Identity" to avoid having to parse matching brackets. 36 | contents = contents.replace(/Promise<([^>]+)>/g, "Identity<$1>") 37 | 38 | // Sync test assertions. 39 | contents = contents.replace(/assert\.rejects/g, "assert.throws") 40 | 41 | return contents 42 | } 43 | 44 | function convertAsyncToSyncFile(inputPath: string, outputPath: string) { 45 | console.log( 46 | path.relative(rootPath, inputPath), 47 | "->", 48 | path.relative(rootPath, outputPath) 49 | ) 50 | 51 | let contents = fs.readFileSync(inputPath, "utf8") 52 | contents = convertAsyncToSync(contents) 53 | 54 | contents = ` 55 | /* 56 | 57 | This file is generated from async/${path.parse(inputPath).base} 58 | 59 | */ 60 | 61 | type Identity = T 62 | 63 | ${contents} 64 | ` 65 | 66 | fs.writeFileSync(outputPath, contents) 67 | 68 | execSync( 69 | path.join(rootPath, "node_modules/.bin/organize-imports-cli") + 70 | " " + 71 | outputPath 72 | ) 73 | execSync( 74 | path.join(rootPath, "node_modules/.bin/prettier") + " --write " + outputPath 75 | ) 76 | } 77 | 78 | const asyncDir = path.join(rootPath, "src/database/async") 79 | const syncDir = path.join(rootPath, "src/database/sync") 80 | 81 | // Remove all non-test files 82 | for (const fileName of fs.readdirSync(syncDir)) { 83 | if (!fileName.endsWith(".test.ts")) { 84 | fs.removeSync(path.join(syncDir, fileName)) 85 | } 86 | } 87 | 88 | for (const fileName of fs.readdirSync(asyncDir)) { 89 | if (fileName.endsWith(".test.ts")) continue 90 | if (!fileName.endsWith(".ts")) continue 91 | 92 | convertAsyncToSyncFile( 93 | path.join(asyncDir, fileName), 94 | path.join(syncDir, convertAsyncToSync(fileName)) 95 | ) 96 | } 97 | -------------------------------------------------------------------------------- /src/database/async/AsyncReactivityTracker.ts: -------------------------------------------------------------------------------- 1 | import { maybePromiseAll } from "../../helpers/maybeWaitForPromises" 2 | import { Bounds } from "../../helpers/sortedTupleArray" 3 | import { 4 | KeyValuePair, 5 | ScanStorageArgs, 6 | Tuple, 7 | WriteOps, 8 | } from "../../storage/types" 9 | import { TxId } from "../types" 10 | import { AsyncCallback } from "./asyncTypes" 11 | import * as SortedTupleValue from "../../helpers/sortedTupleValuePairs" 12 | import * as SortedTuple from "../../helpers/sortedTupleArray" 13 | 14 | type Listeners = Map 15 | 16 | export class AsyncReactivityTracker { 17 | private listeners: Listeners = new Map() 18 | 19 | subscribe(args: ScanStorageArgs, callback: AsyncCallback) { 20 | return subscribe(this.listeners, args, callback) 21 | } 22 | 23 | computeReactivityEmits(writes: WriteOps) { 24 | return getReactivityEmits(this.listeners, writes) 25 | } 26 | 27 | async emit(emits: ReactivityEmits, txId: TxId) { 28 | let promises: any[] = [] 29 | for (const [callback, writes] of emits.entries()) { 30 | try { 31 | // Catch sync callbacks. 32 | promises.push(callback(writes, txId)) 33 | } catch (error) { 34 | console.error(error) 35 | } 36 | } 37 | // This trick allows us to return a Promise from a sync TupleDatabase#commit 38 | // when there are async callbacks. And this allows us to create an async client 39 | // on top of a sync client. 40 | return maybePromiseAll(promises) 41 | } 42 | } 43 | 44 | type ReactivityEmits = Map> 45 | 46 | function getReactivityEmits(listenersDb: Listeners, writes: WriteOps) { 47 | const emits: ReactivityEmits = new Map() 48 | 49 | for (const [callback, bounds] of listenersDb) { 50 | const matchingWrites: KeyValuePair[] = [] 51 | const matchingRemoves: Tuple[] = [] 52 | // Found it to be slightly faster to not assume this is sorted and check bounds individually instead of using scan(writes.set, bounds) 53 | for (const kv of writes.set || []) { 54 | if (SortedTuple.isTupleWithinBounds(kv.key, bounds)) { 55 | matchingWrites.push(kv) 56 | } 57 | } 58 | for (const tuple of writes.remove || []) { 59 | if (SortedTuple.isTupleWithinBounds(tuple, bounds)) { 60 | matchingRemoves.push(tuple) 61 | } 62 | } 63 | if (matchingWrites.length > 0 || matchingRemoves.length > 0) { 64 | emits.set(callback, { set: matchingWrites, remove: matchingRemoves }) 65 | } 66 | } 67 | 68 | return emits 69 | } 70 | 71 | function subscribe( 72 | listenersDb: Listeners, 73 | args: ScanStorageArgs, 74 | callback: AsyncCallback 75 | ) { 76 | listenersDb.set(callback, args) 77 | 78 | return () => { 79 | listenersDb.delete(callback) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/database/sync/ReactivityTracker.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | This file is generated from async/AsyncReactivityTracker.ts 4 | 5 | */ 6 | 7 | type Identity = T 8 | 9 | import { maybePromiseAll } from "../../helpers/maybeWaitForPromises" 10 | import * as SortedTuple from "../../helpers/sortedTupleArray" 11 | import { Bounds } from "../../helpers/sortedTupleArray" 12 | import { 13 | KeyValuePair, 14 | ScanStorageArgs, 15 | Tuple, 16 | WriteOps, 17 | } from "../../storage/types" 18 | import { TxId } from "../types" 19 | import { Callback } from "./types" 20 | 21 | type Listeners = Map 22 | 23 | export class ReactivityTracker { 24 | private listeners: Listeners = new Map() 25 | 26 | subscribe(args: ScanStorageArgs, callback: Callback) { 27 | return subscribe(this.listeners, args, callback) 28 | } 29 | 30 | computeReactivityEmits(writes: WriteOps) { 31 | return getReactivityEmits(this.listeners, writes) 32 | } 33 | 34 | emit(emits: ReactivityEmits, txId: TxId) { 35 | let promises: any[] = [] 36 | for (const [callback, writes] of emits.entries()) { 37 | try { 38 | // Catch sync callbacks. 39 | promises.push(callback(writes, txId)) 40 | } catch (error) { 41 | console.error(error) 42 | } 43 | } 44 | // This trick allows us to return a Promise from a sync TupleDatabase#commit 45 | // when there are callbacks. And this allows us to create an client 46 | // on top of a sync client. 47 | return maybePromiseAll(promises) 48 | } 49 | } 50 | 51 | type ReactivityEmits = Map> 52 | 53 | function getReactivityEmits(listenersDb: Listeners, writes: WriteOps) { 54 | const emits: ReactivityEmits = new Map() 55 | 56 | for (const [callback, bounds] of listenersDb) { 57 | const matchingWrites: KeyValuePair[] = [] 58 | const matchingRemoves: Tuple[] = [] 59 | // Found it to be slightly faster to not assume this is sorted and check bounds individually instead of using scan(writes.set, bounds) 60 | for (const kv of writes.set || []) { 61 | if (SortedTuple.isTupleWithinBounds(kv.key, bounds)) { 62 | matchingWrites.push(kv) 63 | } 64 | } 65 | for (const tuple of writes.remove || []) { 66 | if (SortedTuple.isTupleWithinBounds(tuple, bounds)) { 67 | matchingRemoves.push(tuple) 68 | } 69 | } 70 | if (matchingWrites.length > 0 || matchingRemoves.length > 0) { 71 | emits.set(callback, { set: matchingWrites, remove: matchingRemoves }) 72 | } 73 | } 74 | 75 | return emits 76 | } 77 | 78 | function subscribe( 79 | listenersDb: Listeners, 80 | args: ScanStorageArgs, 81 | callback: Callback 82 | ) { 83 | listenersDb.set(callback, args) 84 | 85 | return () => { 86 | listenersDb.delete(callback) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/helpers/sortedList.ts: -------------------------------------------------------------------------------- 1 | import { binarySearch } from "./binarySearch" 2 | import { Compare } from "./compare" 3 | 4 | export function set(list: T[], item: T, cmp: Compare) { 5 | const result = binarySearch(list, item, cmp) 6 | if (result.found !== undefined) { 7 | // Replace the whole item. 8 | list.splice(result.found, 1, item) 9 | } else { 10 | // Insert at missing index. 11 | list.splice(result.closest, 0, item) 12 | } 13 | } 14 | 15 | export function get(list: T[], item: T, cmp: Compare) { 16 | const result = binarySearch(list, item, cmp) 17 | if (result.found === undefined) return 18 | return list[result.found] 19 | } 20 | 21 | export function exists(list: T[], item: T, cmp: Compare) { 22 | const result = binarySearch(list, item, cmp) 23 | return result.found !== undefined 24 | } 25 | 26 | export function remove(list: T[], item: T, cmp: Compare) { 27 | let { found } = binarySearch(list, item, cmp) 28 | if (found !== undefined) { 29 | // Remove from index. 30 | return list.splice(found, 1)[0] 31 | } 32 | } 33 | 34 | type ScanArgs = { 35 | gt?: T 36 | gte?: T 37 | lt?: T 38 | lte?: T 39 | limit?: number 40 | reverse?: boolean 41 | } 42 | 43 | export function scan(list: T[], args: ScanArgs, cmp: Compare) { 44 | const start = args.gte || args.gt 45 | const end = args.lte || args.lt 46 | 47 | if (start !== undefined && end !== undefined && cmp(start, end) > 0) { 48 | throw new Error("Invalid bounds.") 49 | } 50 | 51 | let lowerSearchBound: number 52 | let upperSearchBound: number 53 | 54 | if (start === undefined) { 55 | lowerSearchBound = 0 56 | } else { 57 | const result = binarySearch(list, start, cmp) 58 | if (result.found === undefined) { 59 | lowerSearchBound = result.closest 60 | } else { 61 | if (args.gt) lowerSearchBound = result.found + 1 62 | else lowerSearchBound = result.found 63 | } 64 | } 65 | 66 | if (end === undefined) { 67 | upperSearchBound = list.length 68 | } else { 69 | const result = binarySearch(list, end, cmp) 70 | if (result.found === undefined) { 71 | upperSearchBound = result.closest 72 | } else { 73 | if (args.lt) upperSearchBound = result.found 74 | else upperSearchBound = result.found + 1 75 | } 76 | } 77 | 78 | const lowerDataBound = 79 | args.reverse && args.limit 80 | ? Math.max(lowerSearchBound, upperSearchBound - args.limit) 81 | : lowerSearchBound 82 | const upperDataBound = 83 | !args.reverse && args.limit 84 | ? Math.min(lowerSearchBound + args.limit, upperSearchBound) 85 | : upperSearchBound 86 | 87 | return args.reverse 88 | ? list.slice(lowerDataBound, upperDataBound).reverse() 89 | : list.slice(lowerDataBound, upperDataBound) 90 | } 91 | -------------------------------------------------------------------------------- /src/helpers/compareTuple.test.ts: -------------------------------------------------------------------------------- 1 | import { shuffle } from "remeda" 2 | import { describe, it, expect } from "bun:test" 3 | import { Tuple } from "../storage/types" 4 | import { sortedValues } from "../test/fixtures" 5 | import { 6 | compareTuple, 7 | compareValue, 8 | TupleToString, 9 | ValueToString, 10 | } from "./compareTuple" 11 | import { randomInt } from "./random" 12 | 13 | describe("compareValue", () => { 14 | it("sorting is correct", () => { 15 | for (let i = 0; i < sortedValues.length; i++) { 16 | for (let j = 0; j < sortedValues.length; j++) { 17 | expect(compareValue(sortedValues[i], sortedValues[j])).toBe( 18 | compareValue(i, j) 19 | ) 20 | } 21 | } 22 | }) 23 | 24 | it("sorts class objects properly", () => { 25 | class A {} 26 | 27 | const values = shuffle([ 28 | { a: 1 }, 29 | { a: 2 }, 30 | { b: -1 }, 31 | new A(), 32 | new A(), 33 | ]).sort(compareValue) 34 | 35 | expect(values[0]).toEqual({ a: 1 }) 36 | expect(values[1]).toEqual({ a: 2 }) 37 | expect(values[2]).toEqual({ b: -1 }) 38 | expect(values[3] instanceof A).toBeTruthy() 39 | expect(values[4] instanceof A).toBeTruthy() 40 | }) 41 | 42 | it("Compares object equality", () => { 43 | class A {} 44 | const a = new A() 45 | expect(compareValue(a, a)).toBe(0) 46 | }) 47 | }) 48 | 49 | describe("compareTuple", () => { 50 | it("Sorting works for pairs in-order.", () => { 51 | const test = (a: Tuple, b: Tuple, value: number) => { 52 | expect(compareTuple(a, b)).toBe(value) 53 | } 54 | 55 | // Ensure it works for all pairwise tuples. 56 | for (let i = 0; i < sortedValues.length - 1; i++) { 57 | const a = sortedValues[i] 58 | const b = sortedValues[i + 1] 59 | test([a, a], [a, b], -1) 60 | test([a, b], [b, a], -1) 61 | test([b, a], [b, b], -1) 62 | test([a, a], [a, a], 0) 63 | test([b, b], [b, b], 0) 64 | } 65 | }) 66 | 67 | it("Sorting does a true deep-compare", () => { 68 | const test = (a: Tuple, b: Tuple, value: number) => { 69 | expect(compareTuple(a, b)).toBe(value) 70 | } 71 | 72 | test(["a", { a: { b: "c" } }], ["a", { a: { b: "c" } }], 0) 73 | }) 74 | 75 | it("3-length tuple sorting is correct (sampled)", () => { 76 | const sample = () => { 77 | const x = sortedValues.length 78 | const i = randomInt(x - 1) 79 | const j = randomInt(x - 1) 80 | const k = randomInt(x - 1) 81 | const tuple: Tuple = [sortedValues[i], sortedValues[j], sortedValues[k]] 82 | const rank = i * x * x + j * x + k 83 | return { tuple, rank } 84 | } 85 | 86 | // (40*40*40)^2 = 4 billion variations for these sorted 3-length tuples. 87 | for (let iter = 0; iter < 100_000; iter++) { 88 | const a = sample() 89 | const b = sample() 90 | expect(compareTuple(a.tuple, b.tuple)).toBe(compareValue(a.rank, b.rank)) 91 | } 92 | }) 93 | }) 94 | -------------------------------------------------------------------------------- /src/database/sync/subscribeQuery.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { InMemoryTupleStorage } from "../../main" 3 | import { subscribeQuery } from "./subscribeQuery" 4 | import { TupleDatabase } from "./TupleDatabase" 5 | import { TupleDatabaseClient } from "./TupleDatabaseClient" 6 | 7 | describe("subscribeQuery", () => { 8 | it("works", async () => { 9 | const db = new TupleDatabaseClient( 10 | new TupleDatabase(new InMemoryTupleStorage()) 11 | ) 12 | 13 | function setA(a: number) { 14 | const tx = db.transact() 15 | tx.set(["a"], a) 16 | tx.commit() 17 | } 18 | 19 | setA(0) 20 | 21 | let aResult: number | undefined = undefined 22 | 23 | const { result, destroy } = subscribeQuery( 24 | db, 25 | (db) => db.get(["a"]), 26 | (result: number) => { 27 | aResult = result 28 | } 29 | ) 30 | 31 | expect(aResult).toEqual(undefined) 32 | expect(result).toEqual(0) 33 | 34 | setA(1) 35 | 36 | expect(aResult as unknown).toEqual(1) 37 | 38 | destroy() 39 | }) 40 | 41 | it("doesn't run second callback if it is destroyed in first", async () => { 42 | type Schema = 43 | | { 44 | key: ["filesById", number] 45 | value: string 46 | } 47 | | { 48 | key: ["focusedFileId"] 49 | value: number 50 | } 51 | 52 | const db = new TupleDatabaseClient( 53 | new TupleDatabase(new InMemoryTupleStorage()) 54 | ) 55 | 56 | const initTx = db.transact() 57 | initTx.set(["filesById", 1], "file 1 value") 58 | initTx.set(["focusedFileId"], 1) 59 | initTx.commit() 60 | 61 | let focusedFile: number | undefined = undefined 62 | let focusedFileValue: string | undefined = undefined 63 | let subscription: 64 | | { result: string | undefined; destroy: () => void } 65 | | undefined = undefined 66 | 67 | function subscribeToFocusedFile(focusedFile: number) { 68 | subscription = subscribeQuery( 69 | db, 70 | (db) => db.get(["filesById", focusedFile]), 71 | (value) => { 72 | focusedFileValue = value 73 | } 74 | ) 75 | 76 | focusedFileValue = subscription.result 77 | } 78 | 79 | const focusedFileQuery = subscribeQuery( 80 | db, 81 | (db) => db.get(["focusedFileId"])!, 82 | (result) => { 83 | focusedFile = result 84 | subscription?.destroy() 85 | subscribeToFocusedFile(focusedFile) 86 | } 87 | ) 88 | 89 | focusedFile = focusedFileQuery.result 90 | subscribeToFocusedFile(focusedFile) 91 | 92 | expect(focusedFile).toEqual(1) 93 | expect(focusedFileValue as unknown).toEqual("file 1 value") 94 | 95 | const tx = db.transact() 96 | tx.remove(["filesById", 1]) 97 | tx.set(["filesById", 2], "file 2 value") 98 | tx.set(["focusedFileId"], 2) 99 | tx.commit() 100 | 101 | expect(focusedFile).toEqual(2) 102 | expect(focusedFileValue as unknown).toEqual("file 2 value") 103 | }) 104 | }) 105 | -------------------------------------------------------------------------------- /src/database/ConcurrencyLog.ts: -------------------------------------------------------------------------------- 1 | import { mutableFilter } from "../helpers/mutableFilter" 2 | import { outdent } from "../helpers/outdent" 3 | import { Bounds, isTupleWithinBounds } from "../helpers/sortedTupleArray" 4 | import { Tuple } from "../storage/types" 5 | import { TxId } from "./types" 6 | 7 | type ReadItem = { type: "read"; bounds: Bounds; txId: TxId } 8 | type WriteItem = { type: "write"; tuple: Tuple; txId: TxId | undefined } 9 | 10 | type LogItem = ReadItem | WriteItem 11 | 12 | export class ReadWriteConflictError extends Error { 13 | constructor(txId: string | undefined, writeTuple: Tuple, readBounds: Bounds) { 14 | const message = outdent(` 15 | ReadWriteConflictError: ${txId} 16 | Write to tuple ${writeTuple} 17 | conflicted with a read at the bounds ${JSON.stringify(readBounds)} 18 | `) 19 | 20 | super(message) 21 | } 22 | } 23 | 24 | export class ConcurrencyLog { 25 | // O(n) refers to this.log.length 26 | log: LogItem[] = [] 27 | 28 | // O(1) 29 | /** Record a read. */ 30 | read(txId: TxId, bounds: Bounds) { 31 | this.log.push({ type: "read", txId, bounds }) 32 | } 33 | 34 | // O(n) 35 | /** Add writes to the log only if there is a conflict with a read. */ 36 | write(txId: TxId | undefined, tuple: Tuple) { 37 | for (const item of this.log) { 38 | if (item.type === "read" && isTupleWithinBounds(tuple, item.bounds)) { 39 | this.log.push({ type: "write", tuple, txId }) 40 | break 41 | } 42 | } 43 | } 44 | 45 | // O(n^2/4) 46 | /** Determine if any reads conflict with writes. */ 47 | commit(txId: TxId) { 48 | try { 49 | const reads: Bounds[] = [] 50 | for (const item of this.log) { 51 | if (item.type === "read") { 52 | if (item.txId === txId) { 53 | reads.push(item.bounds) 54 | } 55 | } else if (item.type === "write") { 56 | for (const read of reads) { 57 | if (isTupleWithinBounds(item.tuple, read)) { 58 | throw new ReadWriteConflictError(item.txId, item.tuple, read) 59 | } 60 | } 61 | } 62 | } 63 | } finally { 64 | this.cleanupReads(txId) 65 | this.cleanupWrites() 66 | } 67 | } 68 | 69 | cancel(txId: TxId) { 70 | this.cleanupReads(txId) 71 | this.cleanupWrites() 72 | } 73 | 74 | // O(n) 75 | /** Cleanup any reads for this transaction. */ 76 | cleanupReads(txId: string) { 77 | mutableFilter(this.log, (item) => { 78 | const txRead = item.txId === txId && item.type === "read" 79 | return !txRead 80 | }) 81 | } 82 | 83 | // O(n) 84 | /** Cleanup any writes that don't have conflicting reads. */ 85 | cleanupWrites() { 86 | const reads: Bounds[] = [] 87 | mutableFilter(this.log, (item) => { 88 | if (item.type === "read") { 89 | reads.push(item.bounds) 90 | return true 91 | } else { 92 | for (const read of reads) { 93 | if (isTupleWithinBounds(item.tuple, read)) { 94 | return true 95 | } 96 | } 97 | return false 98 | } 99 | }) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/examples/triplestore.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { TupleDatabase } from "../database/sync/TupleDatabase" 3 | import { TupleDatabaseClient } from "../database/sync/TupleDatabaseClient" 4 | import { InMemoryTupleStorage } from "../storage/InMemoryTupleStorage" 5 | import { 6 | $, 7 | evaluateQuery, 8 | Fact, 9 | TriplestoreSchema, 10 | writeFact, 11 | } from "./triplestore" 12 | 13 | // Read triplestore.ts first to understand this this test. 14 | 15 | describe("Triplestore", () => { 16 | it("works", () => { 17 | const db = new TupleDatabaseClient( 18 | new TupleDatabase(new InMemoryTupleStorage()) 19 | ) 20 | 21 | const facts: Fact[] = [ 22 | ["1", "name", "chet"], 23 | ["2", "name", "tk"], 24 | ["3", "name", "joe"], 25 | ["2", "worksFor", "1"], 26 | ["3", "worksFor", "1"], 27 | ] 28 | 29 | for (const fact of facts) { 30 | writeFact(db, fact) 31 | } 32 | 33 | expect( 34 | evaluateQuery(db, [ 35 | [$("chetId"), "name", "chet"], 36 | [$("id"), "worksFor", $("chetId")], 37 | [$("id"), "name", $("name")], 38 | ]) 39 | ).toEqual([ 40 | { name: "tk", id: "2", chetId: "1" }, 41 | { name: "joe", id: "3", chetId: "1" }, 42 | ]) 43 | }) 44 | 45 | it("family example", () => { 46 | const db = new TupleDatabaseClient( 47 | new TupleDatabase(new InMemoryTupleStorage()) 48 | ) 49 | 50 | const facts: Fact[] = [ 51 | ["Chet", "parent", "Deborah"], 52 | ["Deborah", "sibling", "Melanie"], 53 | ["Tim", "parent", "Melanie"], 54 | ["Becca", "parent", "Melanie"], 55 | ["Roni", "parent", "Melanie"], 56 | ["Deborah", "sibling", "Ruth"], 57 | ["Izzy", "parent", "Ruth"], 58 | ["Ali", "parent", "Ruth"], 59 | ["Deborah", "sibling", "Sue"], 60 | ["Ray", "parent", "Sue"], 61 | ["Michelle", "parent", "Sue"], 62 | ["Tyler", "parent", "Sue"], 63 | ["Chet", "parent", "Leon"], 64 | ["Leon", "sibling", "Stephanie"], 65 | ["Matt", "parent", "Stephanie"], 66 | ["Tom", "parent", "Stephanie"], 67 | ] 68 | 69 | for (const fact of facts) { 70 | writeFact(db, fact) 71 | } 72 | 73 | const result = evaluateQuery(db, [ 74 | ["Chet", "parent", $("parent")], 75 | [$("parent"), "sibling", $("auntOrUncle")], 76 | [$("cousin"), "parent", $("auntOrUncle")], 77 | ]) 78 | 79 | expect(result).toEqual([ 80 | { cousin: "Becca", auntOrUncle: "Melanie", parent: "Deborah" }, 81 | { cousin: "Roni", auntOrUncle: "Melanie", parent: "Deborah" }, 82 | { cousin: "Tim", auntOrUncle: "Melanie", parent: "Deborah" }, 83 | { cousin: "Ali", auntOrUncle: "Ruth", parent: "Deborah" }, 84 | { cousin: "Izzy", auntOrUncle: "Ruth", parent: "Deborah" }, 85 | { cousin: "Michelle", auntOrUncle: "Sue", parent: "Deborah" }, 86 | { cousin: "Ray", auntOrUncle: "Sue", parent: "Deborah" }, 87 | { cousin: "Tyler", auntOrUncle: "Sue", parent: "Deborah" }, 88 | { cousin: "Matt", auntOrUncle: "Stephanie", parent: "Leon" }, 89 | { cousin: "Tom", auntOrUncle: "Stephanie", parent: "Leon" }, 90 | ]) 91 | }) 92 | }) 93 | -------------------------------------------------------------------------------- /src/storage/SQLiteTupleStorage.ts: -------------------------------------------------------------------------------- 1 | import type { Database, Transaction } from "better-sqlite3" 2 | import { TupleStorageApi } from "../database/sync/types" 3 | import { decodeTuple, encodeTuple } from "../helpers/codec" 4 | import { KeyValuePair, ScanStorageArgs, Tuple, WriteOps } from "./types" 5 | 6 | export class SQLiteTupleStorage implements TupleStorageApi { 7 | /** 8 | * import sqlite from "better-sqlite3" 9 | * new SQLiteTupleStorage(sqlite("path/to.db")) 10 | */ 11 | constructor(private db: Database) { 12 | const createTableQuery = db.prepare( 13 | `create table if not exists data ( key text primary key, value text)` 14 | ) 15 | 16 | // Make sure the table exists. 17 | createTableQuery.run() 18 | 19 | const insertQuery = db.prepare( 20 | `insert or replace into data values ($key, $value)` 21 | ) 22 | const deleteQuery = db.prepare(`delete from data where key = $key`) 23 | 24 | this.writeFactsQuery = this.db.transaction( 25 | ({ 26 | inserts, 27 | deletes, 28 | }: { 29 | inserts: KeyValuePair[] | undefined 30 | deletes: Tuple[] | undefined 31 | }) => { 32 | for (const { key, value } of inserts || []) { 33 | insertQuery.run({ 34 | key: encodeTuple(key), 35 | value: JSON.stringify(value), 36 | }) 37 | } 38 | for (const tuple of deletes || []) { 39 | deleteQuery.run({ key: encodeTuple(tuple) }) 40 | } 41 | } 42 | ) 43 | } 44 | 45 | private writeFactsQuery: Transaction 46 | 47 | scan = (args: ScanStorageArgs = {}) => { 48 | // Bounds. 49 | let start = args.gte ? encodeTuple(args.gte) : undefined 50 | let startAfter: string | undefined = args.gt 51 | ? encodeTuple(args.gt) 52 | : undefined 53 | let end: string | undefined = args.lte ? encodeTuple(args.lte) : undefined 54 | let endBefore: string | undefined = args.lt 55 | ? encodeTuple(args.lt) 56 | : undefined 57 | 58 | const sqlArgs = { 59 | start, 60 | startAfter, 61 | end, 62 | endBefore, 63 | limit: args.limit, 64 | } 65 | 66 | const where = [ 67 | start ? "key >= $start" : undefined, 68 | startAfter ? "key > $startAfter" : undefined, 69 | end ? "key <= $end" : undefined, 70 | endBefore ? "key < $endBefore" : undefined, 71 | ] 72 | .filter(Boolean) 73 | .join(" and ") 74 | 75 | let sqlQuery = `select * from data` 76 | if (where) { 77 | sqlQuery += " where " 78 | sqlQuery += where 79 | } 80 | sqlQuery += " order by key" 81 | if (args.reverse) { 82 | sqlQuery += " desc" 83 | } 84 | if (args.limit) { 85 | sqlQuery += ` limit $limit` 86 | } 87 | 88 | const results = this.db.prepare(sqlQuery).all(sqlArgs) 89 | 90 | return results.map( 91 | // @ts-ignore 92 | ({ key, value }) => 93 | ({ 94 | key: decodeTuple(key) as Tuple, 95 | value: JSON.parse(value), 96 | } as KeyValuePair) 97 | ) 98 | } 99 | 100 | commit = (writes: WriteOps) => { 101 | const { set: inserts, remove: deletes } = writes 102 | this.writeFactsQuery({ inserts, deletes }) 103 | } 104 | 105 | close() { 106 | this.db.close() 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/helpers/queryBuilder.ts: -------------------------------------------------------------------------------- 1 | import { 2 | FilterTupleValuePairByPrefix, 3 | RemoveTupleValuePairPrefix, 4 | TuplePrefix, 5 | } from "../database/typeHelpers" 6 | import { 7 | AsyncTupleDatabaseClientApi, 8 | AsyncTupleTransactionApi, 9 | ScanArgs, 10 | TupleDatabaseClientApi, 11 | TupleTransactionApi, 12 | } from "../main" 13 | import { KeyValuePair, WriteOps } from "../storage/types" 14 | 15 | export class QueryResult { 16 | constructor(public ops: any[] = []) {} 17 | map = (fn: (value: T) => O): QueryResult => { 18 | return new QueryResult([...this.ops, { fn: "map", args: [fn] }]) 19 | } 20 | chain = (fn: (value: T) => QueryResult): QueryResult => { 21 | return new QueryResult([...this.ops, { fn: "chain", args: [fn] }]) 22 | } 23 | } 24 | 25 | export class QueryBuilder { 26 | constructor(public ops: any[] = []) {} 27 | subspace =

>( 28 | prefix: P 29 | ): QueryBuilder> => { 30 | return new QueryBuilder([...this.ops, { fn: "subspace", args: [prefix] }]) 31 | } 32 | scan = >( 33 | args?: ScanArgs 34 | ): QueryResult[]> => { 35 | return new QueryResult([...this.ops, { fn: "scan", args: [args] }]) 36 | } 37 | write = (writes: WriteOps): QueryResult => { 38 | return new QueryResult([...this.ops, { fn: "write", args: [writes] }]) 39 | } 40 | } 41 | 42 | export function execute( 43 | dbOrTx: TupleDatabaseClientApi | TupleTransactionApi, 44 | query: QueryResult 45 | ): O 46 | export function execute( 47 | dbOrTx: AsyncTupleDatabaseClientApi | AsyncTupleTransactionApi, 48 | query: QueryResult 49 | ): Promise 50 | export function execute( 51 | dbOrTx: 52 | | TupleDatabaseClientApi 53 | | TupleTransactionApi 54 | | AsyncTupleDatabaseClientApi 55 | | AsyncTupleTransactionApi, 56 | query: QueryResult 57 | ): O | Promise { 58 | let tx: any = dbOrTx 59 | 60 | const isTx = "set" in dbOrTx 61 | if (!isTx) { 62 | tx = dbOrTx.transact() 63 | } 64 | 65 | let x: any = tx 66 | 67 | for (const op of query.ops) { 68 | if (op.fn === "subspace") { 69 | x = x.subspace(...op.args) 70 | } 71 | if (op.fn === "scan") { 72 | x = x.scan(...op.args) 73 | } 74 | if (op.fn === "write") { 75 | x = x.write(...op.args) 76 | } 77 | 78 | if (op.fn === "map") { 79 | if (x instanceof Promise) { 80 | x = x.then((x) => op.args[0](x)) 81 | } else { 82 | x = op.args[0](x) 83 | } 84 | } 85 | if (op.fn === "chain") { 86 | if (x instanceof Promise) { 87 | x = x.then((x) => execute(tx, op.args[0](x))) 88 | } else { 89 | x = execute(tx, op.args[0](x)) 90 | } 91 | } 92 | } 93 | 94 | if (!isTx) { 95 | if (x instanceof Promise) { 96 | x = x.then((x) => { 97 | tx.commit() 98 | return x 99 | }) 100 | } else { 101 | tx.commit() 102 | } 103 | } 104 | 105 | return x 106 | } 107 | -------------------------------------------------------------------------------- /src/helpers/compareTuple.ts: -------------------------------------------------------------------------------- 1 | import { isPlainObject } from "./isPlainObject" 2 | import { Tuple, Value } from "../storage/types" 3 | import { encodingRank, encodingTypeOf } from "./codec" 4 | import { compare } from "./compare" 5 | import { UnreachableError } from "./Unreachable" 6 | 7 | export function compareValue(a: Value, b: Value): number { 8 | const at = encodingTypeOf(a) 9 | const bt = encodingTypeOf(b) 10 | if (at === bt) { 11 | if (at === "array") { 12 | return compareTuple(a as any, b as any) 13 | } else if (at === "object") { 14 | if (a === b) return 0 15 | // TODO: prototype.compare for classes. 16 | // NOTE: it's a bit contentious to allow for unsortable data inside a sorted array. 17 | // But it is convenient at times to be able to do this sometimes and just assume that 18 | // thee classes are unsorted. 19 | if (isPlainObject(a)) { 20 | if (isPlainObject(b)) { 21 | // Plain objects are ordered. 22 | // This is convenient for meta types like `{date: "2021-12-01"}` => [["date", "2021-12-01"]] 23 | return compareObject(a as any, b as any) 24 | } else { 25 | // json > class 26 | return -1 27 | } 28 | } else if (isPlainObject(b)) { 29 | // json > class 30 | return 1 31 | } else { 32 | // class != class 33 | return 1 34 | } 35 | } else if (at === "boolean") { 36 | return compare(a as boolean, b as boolean) 37 | } else if (at === "null") { 38 | return 0 39 | } else if (at === "number") { 40 | return compare(a as number, b as number) 41 | } else if (at === "string") { 42 | return compare(a as string, b as string) 43 | } else { 44 | throw new UnreachableError(at) 45 | } 46 | } 47 | 48 | return compare(encodingRank.get(at)!, encodingRank.get(bt)!) 49 | } 50 | 51 | function compareObject( 52 | a: { [key: string]: Value }, 53 | b: { [key: string]: Value } 54 | ) { 55 | const ae = Object.entries(a) 56 | .filter(([k, v]) => v !== undefined) 57 | .sort(([k1], [k2]) => compare(k1, k2)) 58 | const be = Object.entries(b) 59 | .filter(([k, v]) => v !== undefined) 60 | .sort(([k1], [k2]) => compare(k1, k2)) 61 | 62 | const len = Math.min(ae.length, be.length) 63 | 64 | for (let i = 0; i < len; i++) { 65 | const [ak, av] = ae[i] 66 | const [bk, bv] = be[i] 67 | const dir = compareValue(ak, bk) 68 | if (dir === 0) { 69 | const dir2 = compareValue(av, bv) 70 | if (dir2 === 0) { 71 | continue 72 | } 73 | return dir2 74 | } 75 | return dir 76 | } 77 | 78 | if (ae.length > be.length) { 79 | return 1 80 | } else if (ae.length < be.length) { 81 | return -1 82 | } else { 83 | return 0 84 | } 85 | } 86 | 87 | export function compareTuple(a: Tuple, b: Tuple) { 88 | const len = Math.min(a.length, b.length) 89 | 90 | for (let i = 0; i < len; i++) { 91 | const dir = compareValue(a[i], b[i]) 92 | if (dir === 0) { 93 | continue 94 | } 95 | return dir 96 | } 97 | 98 | if (a.length > b.length) { 99 | return 1 100 | } else if (a.length < b.length) { 101 | return -1 102 | } else { 103 | return 0 104 | } 105 | } 106 | 107 | export function ValueToString(value: Value) { 108 | if (value === null) { 109 | return "null" 110 | } else { 111 | return JSON.stringify(value) 112 | } 113 | } 114 | 115 | export function TupleToString(tuple: Tuple) { 116 | return `[${tuple.map(ValueToString).join(",")}]` 117 | } 118 | -------------------------------------------------------------------------------- /src/helpers/subspaceHelpers.ts: -------------------------------------------------------------------------------- 1 | import { equals, omitBy } from "remeda" 2 | import { ScanArgs } from "../database/types" 3 | import { 4 | KeyValuePair, 5 | ScanStorageArgs, 6 | Tuple, 7 | WriteOps, 8 | } from "../storage/types" 9 | import { normalizeTupleBounds } from "./sortedTupleArray" 10 | 11 | export function prependPrefixToTuple(prefix: Tuple, tuple: Tuple): Tuple { 12 | if (!prefix.length) return tuple 13 | return [...prefix, ...tuple] 14 | } 15 | 16 | function prependPrefixToTuples(prefix: Tuple, tuples: Tuple[]): Tuple[] { 17 | if (!prefix.length) return tuples 18 | return tuples.map((tuple) => prependPrefixToTuple(prefix, tuple)) 19 | } 20 | 21 | function prependPrefixToTupleValuePair( 22 | prefix: Tuple, 23 | pair: KeyValuePair 24 | ): KeyValuePair { 25 | if (!prefix.length) return pair 26 | const { key, value } = pair 27 | return { 28 | key: prependPrefixToTuple(prefix, key), 29 | value, 30 | } 31 | } 32 | 33 | function prependPrefixToTupleValuePairs( 34 | prefix: Tuple, 35 | pairs: KeyValuePair[] 36 | ): KeyValuePair[] { 37 | if (!prefix.length) return pairs 38 | return pairs.map((pair) => prependPrefixToTupleValuePair(prefix, pair)) 39 | } 40 | 41 | export function prependPrefixToWriteOps( 42 | prefix: Tuple, 43 | writes: WriteOps 44 | ): WriteOps { 45 | if (!prefix.length) return writes 46 | const set = writes.set 47 | ? prependPrefixToTupleValuePairs(prefix, writes.set) 48 | : undefined 49 | 50 | const remove = writes.remove 51 | ? prependPrefixToTuples(prefix, writes.remove) 52 | : undefined 53 | 54 | return { set, remove } 55 | } 56 | 57 | export function removePrefixFromWriteOps( 58 | prefix: Tuple, 59 | writes: WriteOps 60 | ): WriteOps { 61 | if (!prefix.length) return writes 62 | const set = writes.set 63 | ? removePrefixFromTupleValuePairs(prefix, writes.set) 64 | : undefined 65 | 66 | const remove = writes.remove 67 | ? removePrefixFromTuples(prefix, writes.remove) 68 | : undefined 69 | 70 | return { set, remove } 71 | } 72 | 73 | export function removePrefixFromTuple(prefix: Tuple, tuple: Tuple) { 74 | if (!prefix.length) return tuple 75 | if (!equals(tuple.slice(0, prefix.length), prefix)) { 76 | throw new Error("Invalid prefix: " + JSON.stringify({ prefix, tuple })) 77 | } 78 | return tuple.slice(prefix.length) 79 | } 80 | 81 | function removePrefixFromTuples(prefix: Tuple, tuples: Tuple[]) { 82 | if (!prefix.length) return tuples 83 | return tuples.map((tuple) => removePrefixFromTuple(prefix, tuple)) 84 | } 85 | 86 | function removePrefixFromTupleValuePair( 87 | prefix: Tuple, 88 | pair: KeyValuePair 89 | ): KeyValuePair { 90 | if (!prefix.length) return pair 91 | const { key, value } = pair 92 | return { key: removePrefixFromTuple(prefix, key), value } 93 | } 94 | 95 | export function removePrefixFromTupleValuePairs( 96 | prefix: Tuple, 97 | pairs: KeyValuePair[] 98 | ): KeyValuePair[] { 99 | if (!prefix.length) return pairs 100 | return pairs.map((pair) => removePrefixFromTupleValuePair(prefix, pair)) 101 | } 102 | 103 | export function normalizeSubspaceScanArgs( 104 | subspacePrefix: Tuple, 105 | args: ScanArgs 106 | ): ScanStorageArgs { 107 | const prefix = args.prefix 108 | ? [...subspacePrefix, ...args.prefix] 109 | : subspacePrefix 110 | 111 | const bounds = normalizeTupleBounds({ ...args, prefix }) 112 | const { limit, reverse } = args 113 | 114 | return omitBy({ ...bounds, limit, reverse }, (x) => x === undefined) 115 | } 116 | -------------------------------------------------------------------------------- /src/helpers/sortedTupleArray.ts: -------------------------------------------------------------------------------- 1 | import { omitBy } from "remeda" 2 | import { ScanArgs } from "../database/types" 3 | import { MAX, Tuple } from "../storage/types" 4 | import { compareTuple } from "./compareTuple" 5 | import * as sortedList from "./sortedList" 6 | 7 | export function set(data: Array, tuple: Tuple) { 8 | return sortedList.set(data, tuple, compareTuple) 9 | } 10 | 11 | export function exists(data: Array, tuple: Tuple) { 12 | return sortedList.exists(data, tuple, compareTuple) 13 | } 14 | 15 | export function remove(data: Array, tuple: Tuple) { 16 | return sortedList.remove(data, tuple, compareTuple) 17 | } 18 | 19 | export const MaxTuple = [MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX, MAX] 20 | 21 | /** 22 | * Gets the tuple bounds taking into account any prefix specified. 23 | */ 24 | export function normalizeTupleBounds(args: ScanArgs): Bounds { 25 | let gte: Tuple | undefined 26 | let gt: Tuple | undefined 27 | let lte: Tuple | undefined 28 | let lt: Tuple | undefined 29 | 30 | if (args.gte) { 31 | if (args.prefix) { 32 | gte = [...args.prefix, ...args.gte] 33 | } else { 34 | gte = [...args.gte] 35 | } 36 | } else if (args.gt) { 37 | if (args.prefix) { 38 | gt = [...args.prefix, ...args.gt] 39 | } else { 40 | gt = [...args.gt] 41 | } 42 | } else if (args.prefix) { 43 | gte = [...args.prefix] 44 | } 45 | 46 | if (args.lte) { 47 | if (args.prefix) { 48 | lte = [...args.prefix, ...args.lte] 49 | } else { 50 | lte = [...args.lte] 51 | } 52 | } else if (args.lt) { 53 | if (args.prefix) { 54 | lt = [...args.prefix, ...args.lt] 55 | } else { 56 | lt = [...args.lt] 57 | } 58 | } else if (args.prefix) { 59 | // [MAX] is less than [true, "hello"] 60 | // So we're counting on there not being a really long, all true tuple. 61 | // TODO: ideally, we'd either specify a max tuple length, or we'd go 62 | // back to using symbols. 63 | lte = [...args.prefix, ...MaxTuple] 64 | } 65 | 66 | return omitBy({ gte, gt, lte, lt }, (x) => x === undefined) 67 | } 68 | 69 | export function getPrefixContainingBounds(bounds: Bounds) { 70 | const prefix: Tuple = [] 71 | const start = bounds.gt || bounds.gte || [] 72 | const end = bounds.lt || bounds.lte || [] 73 | const len = Math.min(start.length, end.length) 74 | for (let i = 0; i < len; i++) { 75 | if (start[i] === end[i]) { 76 | prefix.push(start[i]) 77 | } else { 78 | break 79 | } 80 | } 81 | return prefix 82 | } 83 | 84 | export function isTupleWithinBounds(tuple: Tuple, bounds: Bounds) { 85 | if (bounds.gt) { 86 | if (compareTuple(tuple, bounds.gt) !== 1) { 87 | return false 88 | } 89 | } 90 | if (bounds.gte) { 91 | if (compareTuple(tuple, bounds.gte) === -1) { 92 | return false 93 | } 94 | } 95 | if (bounds.lt) { 96 | if (compareTuple(tuple, bounds.lt) !== -1) { 97 | return false 98 | } 99 | } 100 | if (bounds.lte) { 101 | if (compareTuple(tuple, bounds.lte) === 1) { 102 | return false 103 | } 104 | } 105 | return true 106 | } 107 | 108 | export type Bounds = { 109 | /** This prevents developers from accidentally using ScanArgs instead of TupleBounds */ 110 | prefix?: never 111 | gte?: Tuple 112 | gt?: Tuple 113 | lte?: Tuple 114 | lt?: Tuple 115 | } 116 | 117 | export function scan(data: Array, args: ScanArgs = {}) { 118 | const { limit, reverse, ...rest } = args 119 | const bounds = normalizeTupleBounds(rest) 120 | return sortedList.scan(data, { limit, reverse, ...bounds }, compareTuple) 121 | } 122 | -------------------------------------------------------------------------------- /src/storage/LMDBTupleStorage.ts: -------------------------------------------------------------------------------- 1 | import type * as LMDB from "lmdb" 2 | import { AsyncTupleStorageApi } from "../database/async/asyncTypes" 3 | import { 4 | decodeTuple, 5 | decodeValue, 6 | encodeTuple, 7 | encodeValue, 8 | } from "../helpers/codec" 9 | import { KeyValuePair, MIN, ScanStorageArgs, Tuple, WriteOps } from "./types" 10 | 11 | const MIN_TUPLE = encodeTuple([MIN]) 12 | 13 | export class LMDBTupleStorage implements AsyncTupleStorageApi { 14 | public db: LMDB.Database 15 | constructor(dbFactory: (options: LMDB.RootDatabaseOptions) => LMDB.Database) { 16 | const encoder = { 17 | writeKey( 18 | key: string | Buffer, 19 | targetBuffer: Buffer, 20 | startPosition: number 21 | ) { 22 | // Sometimes key is buffer (i think for longer keys) 23 | // TODO: add test 24 | if (Buffer.isBuffer(key)) { 25 | key.copy(targetBuffer, startPosition) 26 | } else { 27 | targetBuffer.write(key, startPosition, key.length, "utf8") 28 | } 29 | return startPosition + key.length 30 | }, 31 | readKey(buffer: Buffer, startPosition: number, endPosition: number) { 32 | return buffer.toString("utf8", startPosition, endPosition) 33 | }, 34 | } 35 | // This encoder should take our encoded tuples and write them directly to a buffer 36 | // keyEncoder is used to encode keys when writing to the database, although its mentioned in docs, it is not in the types 37 | // encoder (I think) encodes values, it is not mentioned in the docs however is properly typed 38 | // TODO: test lmdb directly to determine why pre-encoded strings sometimes fail to encode properly with their default encoder 39 | this.db = dbFactory({ 40 | // @ts-expect-error 41 | keyEncoder: encoder, 42 | }) 43 | } 44 | 45 | async scan(args: ScanStorageArgs = {}): Promise { 46 | const startTuple = args.gt ?? args.gte 47 | const start = startTuple !== undefined ? encodeTuple(startTuple) : MIN_TUPLE 48 | const endTuple = args.lt ?? args.lte 49 | const end = endTuple !== undefined ? encodeTuple(endTuple) : undefined 50 | if (start && end) { 51 | if (start > end) { 52 | throw new Error("invalid bounds for scan. Start is greater than end.") 53 | } 54 | } 55 | const results: KeyValuePair[] = [] 56 | const reverse = args.reverse ?? false 57 | // console.log("scan args", args, start, end, reverse) 58 | for (const { key, value } of this.db.getRange({ 59 | start: reverse ? end : start, 60 | reverse, 61 | })) { 62 | if (args.gt && (key as string) <= start!) { 63 | if (reverse) { 64 | break 65 | } 66 | continue 67 | } 68 | if (args.gte && (key as string) < start!) { 69 | if (reverse) { 70 | break 71 | } 72 | continue 73 | } 74 | if (args.lt && (key as string) >= end!) { 75 | if (reverse) { 76 | continue 77 | } 78 | break 79 | } 80 | if (args.lte && (key as string) > end!) { 81 | if (reverse) { 82 | continue 83 | } 84 | break 85 | } 86 | results.push({ 87 | key: decodeTuple(key as string), 88 | value: value, 89 | }) 90 | if (results.length >= (args?.limit ?? Infinity)) break 91 | } 92 | return results 93 | } 94 | 95 | async commit(writes: WriteOps): Promise { 96 | await this.db.batch(() => { 97 | for (const tuple of writes.remove ?? []) { 98 | this.db.remove(encodeTuple(tuple)) 99 | } 100 | for (const { key, value } of writes.set ?? []) { 101 | const storedKey = encodeTuple(key) 102 | const storedValue = value 103 | this.db.put(storedKey, storedValue) 104 | } 105 | }) 106 | } 107 | 108 | async close(): Promise { 109 | return this.db.close() 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/helpers/sortedTupleValuePairs.test.ts: -------------------------------------------------------------------------------- 1 | import * as _ from "remeda" 2 | import { describe, it, expect } from "bun:test" 3 | import { KeyValuePair } from "../storage/types" 4 | import { get, remove, scan, set } from "./sortedTupleValuePairs" 5 | 6 | describe("sortedTupleValuePairs", () => { 7 | const items: KeyValuePair[] = [ 8 | { key: [], value: 0 }, 9 | { key: ["a"], value: 1 }, 10 | { key: ["a", "a"], value: 2 }, 11 | { key: ["a", "b"], value: 3 }, 12 | { key: ["b"], value: 4 }, 13 | { key: ["b", "a"], value: 5 }, 14 | { key: ["b", "b"], value: 6 }, 15 | ] 16 | 17 | it("sorts prefixes in the correct order", () => { 18 | const data: KeyValuePair[] = [] 19 | for (const { key, value } of _.shuffle(items)) { 20 | set(data, key, value) 21 | } 22 | expect(data).toEqual(items) 23 | }) 24 | 25 | it("set will replace a value", () => { 26 | const data = [...items] 27 | set(data, ["b"], 99) 28 | expect(data).toEqual([ 29 | { key: [], value: 0 }, 30 | { key: ["a"], value: 1 }, 31 | { key: ["a", "a"], value: 2 }, 32 | { key: ["a", "b"], value: 3 }, 33 | { key: ["b"], value: 99 }, 34 | { key: ["b", "a"], value: 5 }, 35 | { key: ["b", "b"], value: 6 }, 36 | ]) 37 | }) 38 | 39 | it("remove", () => { 40 | const data = [...items] 41 | remove(data, ["b"]) 42 | expect(data).toEqual([ 43 | { key: [], value: 0 }, 44 | { key: ["a"], value: 1 }, 45 | { key: ["a", "a"], value: 2 }, 46 | { key: ["a", "b"], value: 3 }, 47 | { key: ["b", "a"], value: 5 }, 48 | { key: ["b", "b"], value: 6 }, 49 | ]) 50 | }) 51 | 52 | it("get", () => { 53 | const data = [...items] 54 | const result = get(data, ["b"]) 55 | expect(result).toEqual(4) 56 | }) 57 | 58 | // NOTE: this logic is well tested in sortedList.test.ts and sortedTupleArray.test.ts 59 | // This is just a smoke test because there is some stuff going on with the bounds adjustment. 60 | it("scan prefix", () => { 61 | const result = scan(items, { prefix: ["a"] }) 62 | expect(result).toEqual([ 63 | { key: ["a"], value: 1 }, 64 | { key: ["a", "a"], value: 2 }, 65 | { key: ["a", "b"], value: 3 }, 66 | ]) 67 | }) 68 | 69 | it("scan gt", () => { 70 | const result = scan(items, { gt: ["a", "a"] }) 71 | expect(result).toEqual([ 72 | { key: ["a", "b"], value: 3 }, 73 | { key: ["b"], value: 4 }, 74 | { key: ["b", "a"], value: 5 }, 75 | { key: ["b", "b"], value: 6 }, 76 | ]) 77 | }) 78 | 79 | const reversed = [...items].reverse() 80 | 81 | it("set reverse", () => { 82 | const data: KeyValuePair[] = [] 83 | for (const { key, value } of _.shuffle(items)) { 84 | set(data, key, value, true) 85 | } 86 | expect(data).toEqual(reversed) 87 | }) 88 | 89 | it("remove reverse", () => { 90 | First: { 91 | const data: KeyValuePair[] = [] 92 | 93 | set(data, [1], null, true) 94 | set(data, [2], null, true) 95 | set(data, [3], null, true) 96 | 97 | remove(data, [1], true) 98 | expect(data).toEqual([ 99 | { key: [3], value: null }, 100 | { key: [2], value: null }, 101 | ]) 102 | } 103 | 104 | Middle: { 105 | const data: KeyValuePair[] = [] 106 | 107 | set(data, [1], null, true) 108 | set(data, [2], null, true) 109 | set(data, [3], null, true) 110 | 111 | remove(data, [2], true) 112 | expect(data).toEqual([ 113 | { key: [3], value: null }, 114 | { key: [1], value: null }, 115 | ]) 116 | } 117 | 118 | Last: { 119 | const data: KeyValuePair[] = [] 120 | 121 | set(data, [1], null, true) 122 | set(data, [2], null, true) 123 | set(data, [3], null, true) 124 | 125 | remove(data, [1], true) 126 | expect(data).toEqual([ 127 | { key: [3], value: null }, 128 | { key: [2], value: null }, 129 | ]) 130 | } 131 | }) 132 | }) 133 | -------------------------------------------------------------------------------- /src/helpers/isBoundsWithinBounds.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | // import { assert } from "../test/assertHelpers" 3 | import { isBoundsWithinBounds } from "./isBoundsWithinBounds" 4 | import { Bounds } from "./sortedTupleArray" 5 | 6 | const testWithinBounds = (container: Bounds) => ({ 7 | true: (bounds: Bounds) => { 8 | expect(isBoundsWithinBounds({ container, bounds })).toBeTrue() 9 | }, 10 | false: (bounds: Bounds) => { 11 | expect(isBoundsWithinBounds({ container, bounds })).toBeFalse() 12 | }, 13 | }) 14 | 15 | describe("isBoundsWithinBounds", () => { 16 | it("() bounds", () => { 17 | const test = testWithinBounds({ gt: [0], lt: [10] }) 18 | 19 | test.true({ gt: [0], lt: [10] }) 20 | test.true({ gt: [1], lt: [9] }) 21 | 22 | test.true({ gt: [0], lt: [9] }) 23 | test.true({ gt: [1], lt: [10] }) 24 | 25 | test.false({ gte: [0], lt: [10] }) 26 | test.false({ gt: [0], lte: [10] }) 27 | test.false({ gte: [0], lte: [10] }) 28 | 29 | test.true({ gte: [1], lt: [9] }) 30 | test.true({ gt: [1], lte: [9] }) 31 | test.true({ gte: [1], lte: [9] }) 32 | 33 | test.false({ gte: [0], lt: [9] }) 34 | test.true({ gt: [0], lte: [9] }) 35 | test.false({ gte: [0], lte: [9] }) 36 | 37 | test.true({ gte: [1], lt: [10] }) 38 | test.false({ gt: [1], lte: [10] }) 39 | test.false({ gte: [1], lte: [10] }) 40 | }) 41 | 42 | it("[] bounds", () => { 43 | const test = testWithinBounds({ gte: [0], lte: [10] }) 44 | 45 | test.true({ gt: [0], lt: [10] }) 46 | test.true({ gte: [0], lt: [10] }) 47 | test.true({ gt: [0], lte: [10] }) 48 | test.true({ gte: [0], lte: [10] }) 49 | 50 | test.true({ gt: [1], lt: [9] }) 51 | test.true({ gte: [1], lt: [9] }) 52 | test.true({ gt: [1], lte: [9] }) 53 | test.true({ gte: [1], lte: [9] }) 54 | 55 | test.true({ gt: [0], lt: [9] }) 56 | test.true({ gte: [0], lt: [9] }) 57 | test.true({ gt: [0], lte: [9] }) 58 | test.true({ gte: [0], lte: [9] }) 59 | 60 | test.true({ gt: [1], lt: [10] }) 61 | test.true({ gte: [1], lt: [10] }) 62 | test.true({ gt: [1], lte: [10] }) 63 | test.true({ gte: [1], lte: [10] }) 64 | 65 | test.false({ gt: [0], lt: [11] }) 66 | test.false({ gt: [0], lte: [11] }) 67 | 68 | test.false({ gt: [0], lt: [10, 0] }) 69 | test.false({ gt: [0], lte: [10, 0] }) 70 | 71 | test.false({ gt: [-1], lt: [10] }) 72 | test.false({ gte: [-1], lt: [10] }) 73 | 74 | test.true({ gt: [0, 0], lt: [10] }) 75 | test.true({ gte: [0, 0], lt: [10] }) 76 | }) 77 | 78 | it("(] bounds", () => { 79 | const test = testWithinBounds({ gt: [0], lte: [10] }) 80 | 81 | test.true({ gt: [0], lt: [10] }) 82 | test.false({ gte: [0], lt: [10] }) 83 | test.true({ gt: [0], lte: [10] }) 84 | test.false({ gte: [0], lte: [10] }) 85 | 86 | test.true({ gt: [1], lt: [9] }) 87 | test.true({ gte: [1], lt: [9] }) 88 | test.true({ gt: [1], lte: [9] }) 89 | test.true({ gte: [1], lte: [9] }) 90 | 91 | test.true({ gt: [0], lt: [9] }) 92 | test.false({ gte: [0], lt: [9] }) 93 | test.true({ gt: [0], lte: [9] }) 94 | test.false({ gte: [0], lte: [9] }) 95 | 96 | test.true({ gt: [1], lt: [10] }) 97 | test.true({ gte: [1], lt: [10] }) 98 | test.true({ gt: [1], lte: [10] }) 99 | test.true({ gte: [1], lte: [10] }) 100 | }) 101 | 102 | it("[) bounds", () => { 103 | const test = testWithinBounds({ gte: [0], lt: [10] }) 104 | 105 | test.true({ gt: [0], lt: [10] }) 106 | test.true({ gte: [0], lt: [10] }) 107 | test.false({ gt: [0], lte: [10] }) 108 | test.false({ gte: [0], lte: [10] }) 109 | 110 | test.true({ gt: [1], lt: [9] }) 111 | test.true({ gte: [1], lt: [9] }) 112 | test.true({ gt: [1], lte: [9] }) 113 | test.true({ gte: [1], lte: [9] }) 114 | 115 | test.true({ gt: [0], lt: [9] }) 116 | test.true({ gte: [0], lt: [9] }) 117 | test.true({ gt: [0], lte: [9] }) 118 | test.true({ gte: [0], lte: [9] }) 119 | 120 | test.true({ gt: [1], lt: [10] }) 121 | test.true({ gte: [1], lt: [10] }) 122 | test.false({ gt: [1], lte: [10] }) 123 | test.false({ gte: [1], lte: [10] }) 124 | }) 125 | }) 126 | -------------------------------------------------------------------------------- /src/examples/triplestore.ts: -------------------------------------------------------------------------------- 1 | import { transactionalReadWrite } from "../database/sync/transactionalReadWrite" 2 | 3 | export type Value = string | number | boolean 4 | export type Fact = [string, string, Value] 5 | 6 | export type TriplestoreSchema = 7 | | { key: ["eav", ...Fact]; value: null } 8 | | { key: ["ave", string, Value, string]; value: null } 9 | | { key: ["vea", Value, string, string]; value: null } 10 | 11 | export const writeFact = transactionalReadWrite()( 12 | (tx, fact: Fact) => { 13 | const [e, a, v] = fact 14 | tx.set(["eav", e, a, v], null) 15 | tx.set(["ave", a, v, e], null) 16 | tx.set(["vea", v, e, a], null) 17 | } 18 | ) 19 | 20 | export const removeFact = transactionalReadWrite()( 21 | (tx, fact: Fact) => { 22 | const [e, a, v] = fact 23 | tx.remove(["eav", e, a, v]) 24 | tx.remove(["ave", a, v, e]) 25 | tx.remove(["vea", v, e, a]) 26 | } 27 | ) 28 | 29 | export class Variable { 30 | constructor(public name: string) {} 31 | } 32 | 33 | // Just for dev UX. 34 | export function $(name: string) { 35 | return new Variable(name) 36 | } 37 | 38 | export type Expression = [ 39 | Fact[0] | Variable, 40 | Fact[1] | Variable, 41 | Fact[2] | Variable 42 | ] 43 | 44 | export type Binding = { [varName: string]: Value } 45 | 46 | // Evaluate an expression by scanning the appropriate index. 47 | export const queryExpression = transactionalReadWrite()( 48 | (tx, expr: Expression): Binding[] => { 49 | const [$e, $a, $v] = expr 50 | if ($e instanceof Variable) { 51 | if ($a instanceof Variable) { 52 | if ($v instanceof Variable) { 53 | // ___ 54 | return tx 55 | .scan({ prefix: ["eav"] }) 56 | .map(({ key: [_eav, e, a, v] }) => ({ 57 | [$e.name]: e, 58 | [$a.name]: a, 59 | [$v.name]: v, 60 | })) 61 | } else { 62 | // __V 63 | return tx 64 | .scan({ prefix: ["vea", $v] }) 65 | .map(({ key: [_vea, _v, e, a] }) => ({ 66 | [$e.name]: e, 67 | [$a.name]: a, 68 | })) 69 | } 70 | } else { 71 | if ($v instanceof Variable) { 72 | // A__ 73 | return tx 74 | .scan({ prefix: ["ave", $a] }) 75 | .map(({ key: [_ave, _a, v, e] }) => ({ 76 | [$e.name]: e, 77 | [$v.name]: v, 78 | })) 79 | } else { 80 | // A_V 81 | return tx 82 | .scan({ prefix: ["ave", $a, $v] }) 83 | .map(({ key: [_ave, _a, _v, e] }) => ({ 84 | [$e.name]: e, 85 | })) 86 | } 87 | } 88 | } else { 89 | if ($a instanceof Variable) { 90 | if ($v instanceof Variable) { 91 | // E__ 92 | return tx 93 | .scan({ prefix: ["eav", $e] }) 94 | .map(({ key: [_eav, _e, a, v] }) => ({ 95 | [$a.name]: a, 96 | [$v.name]: v, 97 | })) 98 | } else { 99 | // E_V 100 | return tx 101 | .scan({ prefix: ["vea", $v, $e] }) 102 | .map(({ key: [_vea, _v, _e, a] }) => ({ 103 | [$a.name]: a, 104 | })) 105 | } 106 | } else { 107 | if ($v instanceof Variable) { 108 | // EA_ 109 | return tx 110 | .scan({ prefix: ["eav", $e, $a] }) 111 | .map(({ key: [_eav, _e, _a, v] }) => ({ 112 | [$v.name]: v, 113 | })) 114 | } else { 115 | // EAV 116 | return tx 117 | .scan({ prefix: ["eav", $e, $a, $v] }) 118 | .map(({ key: [_eav, _e, _a, _v] }) => ({})) 119 | } 120 | } 121 | } 122 | } 123 | ) 124 | 125 | export type Query = Expression[] 126 | 127 | export function substituteBinding(query: Query, binding: Binding): Query { 128 | return query.map((expr) => { 129 | return expr.map((item) => 130 | item instanceof Variable && item.name in binding 131 | ? binding[item.name] 132 | : item 133 | ) as Expression 134 | }) 135 | } 136 | 137 | // Recursively evaluate a query. 138 | export const evaluateQuery = transactionalReadWrite()( 139 | (tx, query: Query): Binding[] => { 140 | const [first, ...rest] = query 141 | 142 | if (rest.length === 0) return queryExpression(tx, first) 143 | 144 | const bindings = queryExpression(tx, first) 145 | 146 | const result = bindings 147 | .map((binding) => { 148 | // Substitute the rest of the variables for any bindings. 149 | const restQuery = substituteBinding(rest, binding) 150 | 151 | // Recursively evaluate 152 | const moreBindings = evaluateQuery(tx, restQuery) 153 | 154 | // Join the results 155 | return moreBindings.map((b) => ({ ...b, ...binding })) 156 | }) 157 | // Flatten the arrays 158 | .reduce((acc, next) => acc.concat(next), []) 159 | 160 | return result 161 | } 162 | ) 163 | -------------------------------------------------------------------------------- /src/storage/storage.test.ts: -------------------------------------------------------------------------------- 1 | // import sqlite from "better-sqlite3" 2 | import { Level } from "level" 3 | import * as path from "path" 4 | import { asyncDatabaseTestSuite } from "../database/async/asyncDatabaseTestSuite" 5 | import { AsyncTupleDatabaseClient } from "../database/async/AsyncTupleDatabaseClient" 6 | import { databaseTestSuite } from "../database/sync/databaseTestSuite" 7 | import { TupleDatabase } from "../database/sync/TupleDatabase" 8 | import { AsyncTupleDatabase, TupleDatabaseClient } from "../main" 9 | import { FileTupleStorage } from "./FileTupleStorage" 10 | import { IndexedDbTupleStorage } from "./IndexedDbTupleStorage" 11 | import { InMemoryTupleStorage } from "./InMemoryTupleStorage" 12 | import { LevelTupleStorage } from "./LevelTupleStorage" 13 | import { CachedIndexedDbStorage } from "./IndexedDbWithMemoryCacheTupleStorage" 14 | import { MemoryBTreeStorage } from "./MemoryBTreeTupleStorage" 15 | import { LMDBTupleStorage } from "./LMDBTupleStorage" 16 | import * as LMDB from "lmdb" 17 | import { SQLiteTupleStorage } from "./SQLiteTupleStorage" 18 | import sqlite from "better-sqlite3" 19 | 20 | const tmpDir = path.resolve(__dirname, "./../../tmp") 21 | 22 | // databaseTestSuite( 23 | // "TupleDatabaseClient(TupleDatabase(InMemoryTupleStorage))", 24 | // () => new TupleDatabaseClient(new TupleDatabase(new InMemoryTupleStorage())), 25 | // false 26 | // ) 27 | 28 | // databaseTestSuite( 29 | // "TupleDatabaseClient(TupleDatabase(FileTupleStorage))", 30 | // (id) => 31 | // new TupleDatabaseClient( 32 | // new TupleDatabase(new FileTupleStorage(path.join(tmpDir, id))) 33 | // ) 34 | // ) 35 | 36 | // databaseTestSuite( 37 | // "TupleDatabaseClient(TupleDatabase(SQLiteTupleStorage))", 38 | // (id) => 39 | // new TupleDatabaseClient( 40 | // new TupleDatabase(new SQLiteTupleStorage(sqlite(":memory:"))) 41 | // ) 42 | // ) 43 | 44 | asyncDatabaseTestSuite( 45 | "AsyncTupleDatabaseClient(TupleDatabase(LMDBTupleStorage))", 46 | (id) => { 47 | return new AsyncTupleDatabaseClient( 48 | new AsyncTupleDatabase( 49 | new LMDBTupleStorage((options) => 50 | LMDB.open(path.join(tmpDir, `test-${id}.lmdb`), { 51 | ...options, 52 | // sharedStructuresKey: Symbol.for("structures"), 53 | // keyEncoding: "ordered-binary", 54 | // dupSort: true, 55 | // strictAsyncOrder: true, 56 | }) 57 | ) 58 | ) 59 | ) 60 | }, 61 | 62 | true 63 | ) 64 | 65 | asyncDatabaseTestSuite( 66 | "AsyncTupleDatabaseClient(TupleDatabase(InMemoryTupleStorage))", 67 | () => 68 | new AsyncTupleDatabaseClient(new TupleDatabase(new InMemoryTupleStorage())), 69 | false 70 | ) 71 | 72 | asyncDatabaseTestSuite( 73 | "AsyncTupleDatabaseClient(AsyncTupleDatabase(InMemoryTupleStorage))", 74 | () => 75 | new AsyncTupleDatabaseClient( 76 | new AsyncTupleDatabase(new InMemoryTupleStorage()) 77 | ), 78 | false 79 | ) 80 | 81 | asyncDatabaseTestSuite( 82 | "AsyncTupleDatabaseClient(AsyncTupleDatabase(MemoryBTreeTupleStorage))", 83 | () => 84 | new AsyncTupleDatabaseClient( 85 | new AsyncTupleDatabase(new MemoryBTreeStorage()) 86 | ), 87 | false 88 | ) 89 | 90 | asyncDatabaseTestSuite( 91 | "AsyncTupleDatabaseClient(AsyncTupleDatabase(LevelTupleStorage))", 92 | (id) => 93 | new AsyncTupleDatabaseClient( 94 | new AsyncTupleDatabase( 95 | new LevelTupleStorage(new Level(path.join(tmpDir, id + ".db"))) 96 | ) 97 | ), 98 | true 99 | ) 100 | 101 | require("fake-indexeddb/auto") 102 | asyncDatabaseTestSuite( 103 | "AsyncTupleDatabaseClient(AsyncTupleDatabase(IndexedDbTupleStorage))", 104 | (id) => 105 | new AsyncTupleDatabaseClient( 106 | new AsyncTupleDatabase(new IndexedDbTupleStorage(id)) 107 | ), 108 | true 109 | ) 110 | asyncDatabaseTestSuite( 111 | "AsyncTupleDatabaseClient(AsyncTupleDatabase(CachedIndexedDbTupleStorage))", 112 | (id) => 113 | new AsyncTupleDatabaseClient( 114 | new AsyncTupleDatabase(new CachedIndexedDbStorage(id)) 115 | ), 116 | true 117 | ) 118 | 119 | // Test that the entire test suite works within a subspace. 120 | asyncDatabaseTestSuite( 121 | "Subspace: AsyncTupleDatabaseClient(AsyncTupleDatabase(InMemoryTupleStorage))", 122 | () => { 123 | const store = new AsyncTupleDatabaseClient( 124 | new AsyncTupleDatabase(new InMemoryTupleStorage()) 125 | ) 126 | return store.subspace(["myApp"]) as any 127 | }, 128 | false 129 | ) 130 | 131 | databaseTestSuite( 132 | "Subspace: TupleDatabaseClient(TupleDatabase(InMemoryTupleStorage))", 133 | () => { 134 | const store = new TupleDatabaseClient( 135 | new TupleDatabase(new InMemoryTupleStorage()) 136 | ) 137 | return store.subspace(["myApp"]) as any 138 | }, 139 | false 140 | ) 141 | -------------------------------------------------------------------------------- /src/examples/socialApp.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { transactionalReadWrite } from "../database/sync/transactionalReadWrite" 3 | import { TupleDatabase } from "../database/sync/TupleDatabase" 4 | import { TupleDatabaseClient } from "../database/sync/TupleDatabaseClient" 5 | import { namedTupleToObject } from "../helpers/namedTupleToObject" 6 | import { ReadOnlyTupleDatabaseClientApi } from "../main" 7 | import { InMemoryTupleStorage } from "../storage/InMemoryTupleStorage" 8 | 9 | type User = { username: string; bio: string } 10 | 11 | type Post = { 12 | id: string 13 | username: string 14 | timestamp: number 15 | text: string 16 | } 17 | 18 | type Schema = 19 | | { key: ["user", { username: string }]; value: User } 20 | | { key: ["post", { id: string }]; value: Post } 21 | | { 22 | key: ["follows", { from: string }, { to: string }] 23 | value: null 24 | } 25 | | { 26 | key: ["following", { to: string }, { from: string }] 27 | value: null 28 | } 29 | | { 30 | key: [ 31 | "profile", 32 | { username: string }, 33 | { timestamp: number }, 34 | { postId: string } 35 | ] 36 | value: null 37 | } 38 | | { 39 | key: [ 40 | "feed", 41 | { username: string }, 42 | { timestamp: number }, 43 | { postId: string } 44 | ] 45 | value: null 46 | } 47 | 48 | const addFollow = transactionalReadWrite()( 49 | (tx, from: string, to: string) => { 50 | // Setup the follow relationships. 51 | tx.set(["follows", { from }, { to }], null) 52 | tx.set(["following", { to }, { from }], null) 53 | 54 | // Get the followed user's posts. 55 | tx.scan({ prefix: ["profile", { username: to }] }) 56 | .map(({ key }) => namedTupleToObject(key)) 57 | .forEach(({ timestamp, postId }) => { 58 | // Write those posts to the user's feed. 59 | tx.set(["feed", { username: from }, { timestamp }, { postId }], null) 60 | }) 61 | } 62 | ) 63 | 64 | const createPost = transactionalReadWrite()((tx, post: Post) => { 65 | tx.set(["post", { id: post.id }], post) 66 | 67 | // Add to the user's profile 68 | const { username, timestamp } = post 69 | tx.set(["profile", { username }, { timestamp }, { postId: post.id }], null) 70 | 71 | // Find everyone who follows this username. 72 | const followers = tx 73 | .scan({ prefix: ["following", { to: username }] }) 74 | .map(({ key }) => namedTupleToObject(key)) 75 | .map(({ from }) => from) 76 | 77 | // Write to their feed. 78 | followers.forEach((username) => { 79 | tx.set(["feed", { username }, { timestamp }, { postId: post.id }], null) 80 | }) 81 | }) 82 | 83 | const createUser = transactionalReadWrite()((tx, user: User) => { 84 | tx.set(["user", { username: user.username }], user) 85 | }) 86 | 87 | function getFeed(db: ReadOnlyTupleDatabaseClientApi, username: string) { 88 | return db 89 | .scan({ prefix: ["feed", { username }] }) 90 | .map(({ key }) => namedTupleToObject(key)) 91 | .map(({ postId }) => postId) 92 | } 93 | 94 | function getProfile( 95 | db: ReadOnlyTupleDatabaseClientApi, 96 | username: string 97 | ) { 98 | return db 99 | .scan({ prefix: ["profile", { username }] }) 100 | .map(({ key }) => namedTupleToObject(key)) 101 | .map(({ postId }) => postId) 102 | } 103 | 104 | describe("Social App", () => { 105 | it("works", () => { 106 | // Lets try it out. 107 | const db = new TupleDatabaseClient( 108 | new TupleDatabase(new InMemoryTupleStorage()) 109 | ) 110 | 111 | createUser(db, { username: "chet", bio: "I like to build things." }) 112 | createUser(db, { username: "elon", bio: "Let's go to mars." }) 113 | createUser(db, { username: "meghan", bio: "" }) 114 | 115 | // Chet makes a post. 116 | createPost(db, { 117 | id: "post1", 118 | username: "chet", 119 | timestamp: 1, 120 | text: "post1", 121 | }) 122 | createPost(db, { 123 | id: "post2", 124 | username: "meghan", 125 | timestamp: 2, 126 | text: "post2", 127 | }) 128 | 129 | expect(getProfile(db, "chet")).toEqual(["post1"]) 130 | expect(getProfile(db, "meghan")).toEqual(["post2"]) 131 | 132 | expect(getFeed(db, "chet")).toEqual([]) 133 | expect(getFeed(db, "meghan")).toEqual([]) 134 | 135 | // When meghan follows chet, the post should appear in her feed. 136 | addFollow(db, "meghan", "chet") 137 | 138 | expect(getFeed(db, "chet")).toEqual([]) 139 | expect(getFeed(db, "meghan")).toEqual(["post1"]) 140 | 141 | // When chet makes another post, it should show up in meghan's feed. 142 | createPost(db, { 143 | id: "post3", 144 | username: "chet", 145 | timestamp: 3, 146 | text: "post3", 147 | }) 148 | 149 | expect(getProfile(db, "chet")).toEqual(["post1", "post3"]) 150 | expect(getFeed(db, "meghan")).toEqual(["post1", "post3"]) 151 | }) 152 | }) 153 | -------------------------------------------------------------------------------- /src/helpers/codec.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { Tuple } from "../storage/types" 3 | import { sortedValues as allSortedValues } from "../test/fixtures" 4 | import { 5 | EncodingOptions, 6 | decodeTuple, 7 | decodeValue, 8 | encodeTuple, 9 | encodeValue, 10 | } from "./codec" 11 | import { compare } from "./compare" 12 | import { randomInt } from "./random" 13 | 14 | const ENCODING_OPTIONS: [string, EncodingOptions | undefined][] = [ 15 | ["defaults", undefined], 16 | ["overrides", { delimiter: "\x01", escape: "\x02", disallow: ["\x00"] }], 17 | ] 18 | 19 | describe.each(ENCODING_OPTIONS)("codec with options: %s", (_desc, options) => { 20 | // Removes disallow options 21 | const sortedValues = allSortedValues.filter((x) => 22 | typeof x === "string" 23 | ? options?.disallow?.every((d) => !x.includes(d)) 24 | : true 25 | ) 26 | describe("encodeValue", () => { 27 | it("Encodes and decodes properly", () => { 28 | for (let i = 0; i < sortedValues.length; i++) { 29 | const value = sortedValues[i] 30 | const encoded = encodeValue(value, options) 31 | const decoded = decodeValue(encoded, options) 32 | 33 | expect(decoded).toStrictEqual(value) 34 | } 35 | }) 36 | 37 | it("Encodes in lexicographical order", () => { 38 | for (let i = 0; i < sortedValues.length; i++) { 39 | for (let j = 0; j < sortedValues.length; j++) { 40 | const a = encodeValue(sortedValues[i], options) 41 | const b = encodeValue(sortedValues[j], options) 42 | expect(compare(a, b)).toStrictEqual(compare(i, j)) 43 | } 44 | } 45 | }) 46 | }) 47 | 48 | describe("encodeTuple", () => { 49 | it("Encodes and decodes properly", () => { 50 | const test = (tuple: Tuple) => { 51 | const encoded = encodeTuple(tuple, options) 52 | const decoded = decodeTuple(encoded, options) 53 | expect(decoded).toStrictEqual(tuple) 54 | } 55 | test([]) 56 | for (let i = 0; i < sortedValues.length; i++) { 57 | const a = sortedValues[i] 58 | test([a]) 59 | for (let j = 0; j < sortedValues.length; j++) { 60 | const b = sortedValues[j] 61 | test([a, b]) 62 | } 63 | } 64 | 65 | for (let i = 0; i < sortedValues.length - 2; i++) { 66 | const opts = sortedValues.slice(i, i + 3) 67 | for (const a of opts) { 68 | for (const b of opts) { 69 | for (const c of opts) { 70 | test([a, b, c]) 71 | } 72 | } 73 | } 74 | } 75 | }) 76 | 77 | it("Encodes in lexicographical order", () => { 78 | const test2 = ( 79 | a: { tuple: Tuple; rank: number }, 80 | b: { tuple: Tuple; rank: number }, 81 | result: number 82 | ) => { 83 | try { 84 | test(a.tuple, b.tuple, result) 85 | } catch (e) { 86 | console.log({ aRank: a.rank, bRank: b.rank }) 87 | throw e 88 | } 89 | } 90 | 91 | const test = (aTuple: Tuple, bTuple: Tuple, result: number) => { 92 | const a = encodeTuple(aTuple, options) 93 | const b = encodeTuple(bTuple, options) 94 | const actual = compare(a, b) 95 | const expected = result 96 | try { 97 | expect(actual).toStrictEqual(expected) 98 | } catch (e) { 99 | console.log({ aTuple, bTuple, a, b, actual, expected }) 100 | throw e 101 | } 102 | } 103 | 104 | for (let i = 0; i < sortedValues.length; i++) { 105 | for (let j = 0; j < sortedValues.length; j++) { 106 | const a = sortedValues[i] 107 | const b = sortedValues[j] 108 | try { 109 | test([a, a], [a, b], compare(i, j)) 110 | } catch (e) { 111 | console.log({ i, j }) 112 | throw e 113 | } 114 | test([a, b], [b, a], compare(i, j)) 115 | test([b, a], [b, b], compare(i, j)) 116 | if (i !== j) { 117 | test([a], [a, a], -1) 118 | test([a], [a, b], -1) 119 | test([a], [b, a], compare(i, j)) 120 | test([a], [b, b], compare(i, j)) 121 | test([b], [a, a], compare(j, i)) 122 | test([b], [a, b], compare(j, i)) 123 | test([b], [b, a], -1) 124 | test([b], [b, b], -1) 125 | } 126 | } 127 | } 128 | 129 | const sample = () => { 130 | const x = sortedValues.length 131 | const i = randomInt(x - 1) 132 | const j = randomInt(x - 1) 133 | const k = randomInt(x - 1) 134 | const tuple: Tuple = [sortedValues[i], sortedValues[j], sortedValues[k]] 135 | const rank = i * x * x + j * x + k 136 | return { tuple, rank } 137 | } 138 | 139 | // (40*40*40)^2 = 4 billion variations for these sorted 3-length tuples. 140 | for (let iter = 0; iter < 100_000; iter++) { 141 | const a = sample() 142 | const b = sample() 143 | test2(a, b, compare(a.rank, b.rank)) 144 | } 145 | }) 146 | }) 147 | }) 148 | 149 | it("Throws error if a value cannot be encoded", () => { 150 | expect(() => encodeValue("a\x00b", { disallow: ["\x00"] })).toThrow() 151 | }) 152 | -------------------------------------------------------------------------------- /src/database/typeHelpers.ts: -------------------------------------------------------------------------------- 1 | import { KeyValuePair, Tuple } from "../storage/types" 2 | 3 | export type Assert = Actual 4 | 5 | // Can't create recursive string types, otherwise: `${Ints}${Ints}` 6 | export type Ints = `${number}` 7 | 8 | /** Convert ["a", "b"] in {0: "a", 1: "b"} so that we can use Extract to match tuple prefixes. */ 9 | export type TupleToObject = Pick> 10 | 11 | type A1 = Assert, { 0: 1; 1: 2 }> 12 | 13 | export type FilterTupleByPrefix = Extract< 14 | S, 15 | TupleToObject

16 | > 17 | type A2 = Assert< 18 | FilterTupleByPrefix<[1, 2] | [1, 3] | [2, 1], [1]>, 19 | [1, 2] | [1, 3] 20 | > 21 | // @ts-expect-error missing a tuple that should be filtered. 22 | type A22 = Assert, [1, 2]> 23 | 24 | export type FilterTupleValuePairByPrefix< 25 | S extends KeyValuePair, 26 | P extends Tuple 27 | > = Extract }> 28 | 29 | type A3 = Assert< 30 | FilterTupleValuePairByPrefix< 31 | | { key: [1, 2]; value: number } 32 | | { key: [1, 3]; value: string } 33 | | { key: [2, 1]; value: null }, 34 | [1] 35 | >, 36 | { key: [1, 2]; value: number } | { key: [1, 3]; value: string } 37 | > 38 | 39 | type A33 = Assert< 40 | FilterTupleValuePairByPrefix< 41 | { key: [string, number, boolean]; value: null }, 42 | [string] 43 | >, 44 | { key: [string, number, boolean]; value: null } 45 | > 46 | 47 | export type FilterTupleValuePair< 48 | S extends KeyValuePair, 49 | P extends Tuple 50 | > = Extract 51 | 52 | type F1 = Assert< 53 | FilterTupleValuePair< 54 | | { key: [1, 2]; value: number } 55 | | { key: [1, 3]; value: string } 56 | | { key: [2, 1]; value: null }, 57 | [1, 2] 58 | >, 59 | { key: [1, 2]; value: number } 60 | > 61 | 62 | type DistributiveProp = T extends unknown ? T[K] : never 63 | 64 | export type ValueForTuple< 65 | S extends KeyValuePair, 66 | P extends Tuple 67 | > = DistributiveProp, "value"> 68 | 69 | type F2 = Assert< 70 | ValueForTuple< 71 | | { key: [1, 2]; value: number } 72 | | { key: [1, 3]; value: string } 73 | | { key: [2, 1]; value: null }, 74 | [1, 2] 75 | >, 76 | number 77 | > 78 | 79 | export type IsTuple = [] | { 0: any } 80 | type A4 = Assert<[], IsTuple> 81 | type A5 = Assert<[1, 2], IsTuple> 82 | // @ts-expect-error is not a tuple. 83 | type A6 = Assert 84 | 85 | export type TuplePrefix = T extends IsTuple 86 | ? T extends [any, ...infer U] 87 | ? [] | [T[0]] | [T[0], ...TuplePrefix] 88 | : [] 89 | : T | [] 90 | 91 | type A7 = Assert, [] | [1] | [1, 2] | [1, 2, 3]> 92 | // @ts-expect-error missing a prefix [] 93 | type A77 = Assert, [1] | [1, 2] | [1, 2, 3]> 94 | type A777 = Assert, string[]> 95 | 96 | type A7775 = Assert< 97 | TuplePrefix<[string, boolean, number]>, 98 | [] | [string] | [string, boolean] | [string, boolean, number] 99 | > 100 | 101 | export type TupleRest = T extends [any, ...infer U] 102 | ? U 103 | : never 104 | 105 | type A8 = Assert, [2, 3]> 106 | 107 | export type RemoveTuplePrefix = T extends IsTuple 108 | ? T extends [...P, ...infer U] 109 | ? U 110 | : never 111 | : T 112 | 113 | type A9 = Assert, [3]> 114 | type A10 = Assert, [2, 3]> 115 | type A11 = Assert, never> 116 | 117 | type A111 = Assert< 118 | RemoveTuplePrefix<[string, number, boolean], [string]>, 119 | [number, boolean] 120 | > 121 | 122 | type A1111 = Assert< 123 | RemoveTuplePrefix<[string, number, boolean], []>, 124 | [string, number, boolean] 125 | > 126 | 127 | type A11111 = Assert< 128 | RemoveTuplePrefix, []>, 129 | TuplePrefix<[string, number, boolean]> 130 | > 131 | 132 | export type RemoveTupleValuePairPrefix< 133 | T extends KeyValuePair, 134 | P extends any[] 135 | > = T extends { 136 | key: [...P, ...infer U] 137 | value: infer V 138 | } 139 | ? { key: U; value: V } 140 | : never 141 | 142 | type A12 = Assert< 143 | RemoveTupleValuePairPrefix<{ key: [1, 2, 3]; value: null }, [1, 2]>, 144 | { key: [3]; value: null } 145 | > 146 | type A13 = Assert< 147 | RemoveTupleValuePairPrefix<{ key: [1, 2, 3]; value: string }, [1]>, 148 | { key: [2, 3]; value: string } 149 | > 150 | type A14 = Assert< 151 | RemoveTupleValuePairPrefix<{ key: [1, 2, 3]; value: string }, [2]>, 152 | never 153 | > 154 | 155 | // Using the DistributiveProp trick here too. 156 | export type SchemaSubspace< 157 | P extends Tuple, 158 | T extends KeyValuePair 159 | > = T extends unknown 160 | ? { 161 | key: [...P, ...T["key"]] 162 | value: T["value"] 163 | } 164 | : never 165 | 166 | type A15 = Assert< 167 | SchemaSubspace<["int"], { key: [1]; value: 1 } | { key: [2]; value: 2 }>, 168 | { key: ["int", 1]; value: 1 } | { key: ["int", 2]; value: 2 } 169 | > 170 | -------------------------------------------------------------------------------- /src/database/sync/types.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | This file is generated from async/asyncTypes.ts 4 | 5 | */ 6 | 7 | type Identity = T 8 | 9 | import { KeyValuePair, ScanStorageArgs, WriteOps } from "../../storage/types" 10 | import { 11 | FilterTupleValuePairByPrefix, 12 | RemoveTupleValuePairPrefix, 13 | TuplePrefix, 14 | ValueForTuple, 15 | } from "../typeHelpers" 16 | import { ScanArgs, TxId, Unsubscribe } from "../types" 17 | 18 | /** The low-level API for implementing new storage layers. */ 19 | export type TupleStorageApi = { 20 | scan: (args?: ScanStorageArgs) => Identity 21 | commit: (writes: WriteOps) => Identity 22 | close: () => Identity 23 | } 24 | 25 | /** Wraps TupleStorageApi with reactivity and MVCC */ 26 | export type TupleDatabaseApi = { 27 | scan: (args?: ScanStorageArgs, txId?: TxId) => Identity 28 | commit: (writes: WriteOps, txId?: TxId) => Identity 29 | cancel: (txId: string) => Identity 30 | subscribe: ( 31 | args: ScanStorageArgs, 32 | callback: Callback 33 | ) => Identity 34 | close: () => Identity 35 | } 36 | 37 | /** Wraps TupleDatabaseApi with types, subspaces, transaction objects, and additional read apis. */ 38 | export type TupleDatabaseClientApi = { 39 | // Types 40 | commit: (writes: WriteOps, txId?: TxId) => Identity 41 | cancel: (txId: string) => Identity 42 | scan: >( 43 | args?: ScanArgs, 44 | txId?: TxId 45 | ) => Identity[]> 46 | subscribe: >( 47 | args: ScanArgs, 48 | callback: Callback> 49 | ) => Identity 50 | close: () => Identity 51 | 52 | // ReadApis 53 | get: ( 54 | tuple: T, 55 | txId?: TxId 56 | ) => Identity | undefined> 57 | exists: (tuple: T, txId?: TxId) => Identity 58 | 59 | // Subspace 60 | subspace:

>( 61 | prefix: P 62 | ) => TupleDatabaseClientApi> 63 | 64 | // Transaction 65 | /** Arguments to transact() are for internal use only. */ 66 | transact: (txId?: TxId, writes?: WriteOps) => TupleRootTransactionApi 67 | } 68 | 69 | export type TupleRootTransactionApi = { 70 | // ReadApis 71 | // Same as TupleDatabaseClientApi without the txId argument. 72 | scan: >( 73 | args?: ScanArgs 74 | ) => Identity[]> 75 | get: ( 76 | tuple: T 77 | ) => Identity | undefined> 78 | exists: (tuple: T) => Identity 79 | 80 | // Subspace 81 | // Demotes to a non-root transaction so you cannot commit, cancel, or inspect 82 | // the transaction. 83 | subspace:

>( 84 | prefix: P 85 | ) => TupleTransactionApi> 86 | 87 | // WriteApis 88 | set: ( 89 | tuple: Key, 90 | value: ValueForTuple 91 | ) => TupleRootTransactionApi 92 | remove: (tuple: S["key"]) => TupleRootTransactionApi 93 | write: (writes: WriteOps) => TupleRootTransactionApi 94 | 95 | // RootTransactionApis 96 | commit: () => Identity 97 | cancel: () => Identity 98 | id: TxId 99 | writes: Required> 100 | } 101 | 102 | export type TupleTransactionApi = { 103 | // ReadApis 104 | // Same as TupleDatabaseClientApi without the txId argument. 105 | scan: >( 106 | args?: ScanArgs 107 | ) => Identity[]> 108 | get: ( 109 | tuple: T 110 | ) => Identity | undefined> 111 | exists: (tuple: T) => Identity 112 | 113 | // Subspace 114 | subspace:

>( 115 | prefix: P 116 | ) => TupleTransactionApi> 117 | 118 | // WriteApis 119 | set: ( 120 | tuple: Key, 121 | value: ValueForTuple 122 | ) => TupleTransactionApi 123 | remove: (tuple: S["key"]) => TupleTransactionApi 124 | write: (writes: WriteOps) => TupleTransactionApi 125 | } 126 | 127 | /** Useful for indicating that a function does not commit any writes. */ 128 | export type ReadOnlyTupleDatabaseClientApi< 129 | S extends KeyValuePair = KeyValuePair 130 | > = { 131 | scan: >( 132 | args?: ScanArgs, 133 | txId?: TxId 134 | ) => Identity[]> 135 | get: ( 136 | tuple: T, 137 | txId?: TxId 138 | ) => Identity | undefined> 139 | exists: (tuple: T, txId?: TxId) => Identity 140 | subspace:

>( 141 | prefix: P 142 | ) => ReadOnlyTupleDatabaseClientApi> 143 | 144 | // subscribe? 145 | } 146 | 147 | export type Callback = ( 148 | writes: WriteOps, 149 | txId: TxId 150 | ) => void | Identity 151 | -------------------------------------------------------------------------------- /src/database/async/asyncTypes.ts: -------------------------------------------------------------------------------- 1 | import { KeyValuePair, ScanStorageArgs, WriteOps } from "../../storage/types" 2 | import { 3 | FilterTupleValuePairByPrefix, 4 | RemoveTupleValuePairPrefix, 5 | TuplePrefix, 6 | ValueForTuple, 7 | } from "../typeHelpers" 8 | import { ScanArgs, TxId, Unsubscribe } from "../types" 9 | 10 | /** The low-level API for implementing new storage layers. */ 11 | export type AsyncTupleStorageApi = { 12 | scan: (args?: ScanStorageArgs) => Promise 13 | commit: (writes: WriteOps) => Promise 14 | close: () => Promise 15 | } 16 | 17 | /** Wraps AsyncTupleStorageApi with reactivity and MVCC */ 18 | export type AsyncTupleDatabaseApi = { 19 | scan: (args?: ScanStorageArgs, txId?: TxId) => Promise 20 | commit: (writes: WriteOps, txId?: TxId) => Promise 21 | cancel: (txId: string) => Promise 22 | subscribe: ( 23 | args: ScanStorageArgs, 24 | callback: AsyncCallback 25 | ) => Promise 26 | close: () => Promise 27 | } 28 | 29 | /** Wraps AsyncTupleDatabaseApi with types, subspaces, transaction objects, and additional read apis. */ 30 | export type AsyncTupleDatabaseClientApi = 31 | { 32 | // Types 33 | commit: (writes: WriteOps, txId?: TxId) => Promise 34 | cancel: (txId: string) => Promise 35 | scan: >( 36 | args?: ScanArgs, 37 | txId?: TxId 38 | ) => Promise[]> 39 | subscribe: >( 40 | args: ScanArgs, 41 | callback: AsyncCallback> 42 | ) => Promise 43 | close: () => Promise 44 | 45 | // ReadApis 46 | get: ( 47 | tuple: T, 48 | txId?: TxId 49 | ) => Promise | undefined> 50 | exists: (tuple: T, txId?: TxId) => Promise 51 | 52 | // Subspace 53 | subspace:

>( 54 | prefix: P 55 | ) => AsyncTupleDatabaseClientApi> 56 | 57 | // Transaction 58 | /** Arguments to transact() are for internal use only. */ 59 | transact: ( 60 | txId?: TxId, 61 | writes?: WriteOps 62 | ) => AsyncTupleRootTransactionApi 63 | } 64 | 65 | export type AsyncTupleRootTransactionApi< 66 | S extends KeyValuePair = KeyValuePair 67 | > = { 68 | // ReadApis 69 | // Same as AsyncTupleDatabaseClientApi without the txId argument. 70 | scan: >( 71 | args?: ScanArgs 72 | ) => Promise[]> 73 | get: ( 74 | tuple: T 75 | ) => Promise | undefined> 76 | exists: (tuple: T) => Promise 77 | 78 | // Subspace 79 | // Demotes to a non-root transaction so you cannot commit, cancel, or inspect 80 | // the transaction. 81 | subspace:

>( 82 | prefix: P 83 | ) => AsyncTupleTransactionApi> 84 | 85 | // WriteApis 86 | set: ( 87 | tuple: Key, 88 | value: ValueForTuple 89 | ) => AsyncTupleRootTransactionApi 90 | remove: (tuple: S["key"]) => AsyncTupleRootTransactionApi 91 | write: (writes: WriteOps) => AsyncTupleRootTransactionApi 92 | 93 | // RootTransactionApis 94 | commit: () => Promise 95 | cancel: () => Promise 96 | id: TxId 97 | writes: Required> 98 | } 99 | 100 | export type AsyncTupleTransactionApi = { 101 | // ReadApis 102 | // Same as AsyncTupleDatabaseClientApi without the txId argument. 103 | scan: >( 104 | args?: ScanArgs 105 | ) => Promise[]> 106 | get: ( 107 | tuple: T 108 | ) => Promise | undefined> 109 | exists: (tuple: T) => Promise 110 | 111 | // Subspace 112 | subspace:

>( 113 | prefix: P 114 | ) => AsyncTupleTransactionApi> 115 | 116 | // WriteApis 117 | set: ( 118 | tuple: Key, 119 | value: ValueForTuple 120 | ) => AsyncTupleTransactionApi 121 | remove: (tuple: S["key"]) => AsyncTupleTransactionApi 122 | write: (writes: WriteOps) => AsyncTupleTransactionApi 123 | } 124 | 125 | /** Useful for indicating that a function does not commit any writes. */ 126 | export type ReadOnlyAsyncTupleDatabaseClientApi< 127 | S extends KeyValuePair = KeyValuePair 128 | > = { 129 | scan: >( 130 | args?: ScanArgs, 131 | txId?: TxId 132 | ) => Promise[]> 133 | get: ( 134 | tuple: T, 135 | txId?: TxId 136 | ) => Promise | undefined> 137 | exists: (tuple: T, txId?: TxId) => Promise 138 | subspace:

>( 139 | prefix: P 140 | ) => ReadOnlyAsyncTupleDatabaseClientApi> 141 | 142 | // subscribe? 143 | } 144 | 145 | export type AsyncCallback = ( 146 | writes: WriteOps, 147 | txId: TxId 148 | ) => void | Promise 149 | -------------------------------------------------------------------------------- /src/helpers/codec.ts: -------------------------------------------------------------------------------- 1 | // This codec is should create a component-wise lexicographically sortable array. 2 | 3 | import * as elen from "elen" 4 | import { invert, sortBy } from "remeda" 5 | import { isPlainObject } from "./isPlainObject" 6 | import { Tuple, Value } from "../storage/types" 7 | import { compare } from "./compare" 8 | import { UnreachableError } from "./Unreachable" 9 | 10 | export type EncodingOptions = { 11 | delimiter?: string 12 | escape?: string 13 | disallow?: string[] 14 | } 15 | 16 | // null < object < array < number < string < boolean 17 | export const encodingByte = { 18 | null: "b", 19 | object: "c", 20 | array: "d", 21 | number: "e", 22 | string: "f", 23 | boolean: "g", 24 | } as const 25 | 26 | export type EncodingType = keyof typeof encodingByte 27 | 28 | export const encodingRank = new Map( 29 | sortBy(Object.entries(encodingByte), ([key, value]) => value).map( 30 | ([key], i) => [key as EncodingType, i] 31 | ) 32 | ) 33 | 34 | export function encodeValue(value: Value, options?: EncodingOptions): string { 35 | if (value === null) { 36 | return encodingByte.null 37 | } 38 | if (value === true || value === false) { 39 | return encodingByte.boolean + value 40 | } 41 | if (typeof value === "string") { 42 | for (const disallowed of options?.disallow ?? []) { 43 | if (value.includes(disallowed)) { 44 | throw new Error(`Disallowed character found: ${disallowed}.`) 45 | } 46 | } 47 | return encodingByte.string + value 48 | } 49 | if (typeof value === "number") { 50 | return encodingByte.number + elen.encode(value) 51 | } 52 | if (Array.isArray(value)) { 53 | return encodingByte.array + encodeTuple(value, options) 54 | } 55 | if (typeof value === "object") { 56 | return encodingByte.object + encodeObjectValue(value, options) 57 | } 58 | throw new UnreachableError(value, "Unknown value type") 59 | } 60 | 61 | export function encodingTypeOf(value: Value): EncodingType { 62 | if (value === null) { 63 | return "null" 64 | } 65 | if (value === true || value === false) { 66 | return "boolean" 67 | } 68 | if (typeof value === "string") { 69 | return "string" 70 | } 71 | if (typeof value === "number") { 72 | return "number" 73 | } 74 | if (Array.isArray(value)) { 75 | return "array" 76 | } 77 | if (typeof value === "object") { 78 | return "object" 79 | } 80 | throw new UnreachableError(value, "Unknown value type") 81 | } 82 | 83 | const decodeType = invert(encodingByte) as { 84 | [key: string]: keyof typeof encodingByte 85 | } 86 | 87 | export function decodeValue(str: string, options?: EncodingOptions): Value { 88 | const encoding: EncodingType = decodeType[str[0]] 89 | const rest = str.slice(1) 90 | 91 | if (encoding === "null") { 92 | return null 93 | } 94 | if (encoding === "boolean") { 95 | return JSON.parse(rest) 96 | } 97 | if (encoding === "string") { 98 | return rest 99 | } 100 | if (encoding === "number") { 101 | return elen.decode(rest) 102 | } 103 | if (encoding === "array") { 104 | return decodeTuple(rest, options) 105 | } 106 | if (encoding === "object") { 107 | return decodeObjectValue(rest, options) 108 | } 109 | throw new UnreachableError(encoding, "Invalid encoding byte") 110 | } 111 | 112 | export function encodeTuple(tuple: Tuple, options?: EncodingOptions) { 113 | const delimiter = options?.delimiter ?? "\x00" 114 | const escape = options?.escape ?? "\x01" 115 | const reEscapeByte = new RegExp(`${escape}`, "g") 116 | const reDelimiterByte = new RegExp(`${delimiter}`, "g") 117 | return tuple 118 | .map((value, i) => { 119 | const encoded = encodeValue(value, options) 120 | return ( 121 | encoded 122 | // B -> BB or \ -> \\ 123 | .replace(reEscapeByte, escape + escape) 124 | // A -> BA or x -> \x 125 | .replace(reDelimiterByte, escape + delimiter) + delimiter 126 | ) 127 | }) 128 | .join("") 129 | } 130 | 131 | export function decodeTuple(str: string, options?: EncodingOptions) { 132 | if (str === "") { 133 | return [] 134 | } 135 | 136 | const delimiter = options?.delimiter ?? "\x00" 137 | const escape = options?.escape ?? "\x01" 138 | 139 | // Capture all of the escaped BB and BA pairs and wait 140 | // til we find an exposed A. 141 | const matcher = new RegExp( 142 | `(${escape}(${escape}|${delimiter})|${delimiter})`, 143 | "g" 144 | ) 145 | const reEncodedEscape = new RegExp(escape + escape, "g") 146 | const reEncodedDelimiter = new RegExp(escape + delimiter, "g") 147 | const tuple: Tuple = [] 148 | let start = 0 149 | while (true) { 150 | const match = matcher.exec(str) 151 | if (match === null) { 152 | return tuple 153 | } 154 | if (match[0][0] === escape) { 155 | // If we match a escape+escape or escape+delimiter then keep going. 156 | continue 157 | } 158 | const end = match.index 159 | const escaped = str.slice(start, end) 160 | if (typeof escaped !== "string") { 161 | console.log(escaped) 162 | } 163 | const unescaped = escaped 164 | // BB -> B 165 | .replace(reEncodedEscape, escape) 166 | // BA -> A 167 | .replace(reEncodedDelimiter, delimiter) 168 | const decoded = decodeValue(unescaped, options) 169 | tuple.push(decoded) 170 | // Skip over the \x00. 171 | start = end + 1 172 | } 173 | } 174 | 175 | function encodeObjectValue(obj: object, options?: EncodingOptions) { 176 | if (!isPlainObject(obj)) { 177 | throw new Error("Cannot serialize this object.") 178 | } 179 | const entries = Object.entries(obj) 180 | .sort(([k1], [k2]) => compare(k1, k2)) 181 | // We allow undefined values in objects, but we want to strip them out before 182 | // serializing. 183 | .filter(([key, value]) => value !== undefined) 184 | return encodeTuple(entries as Tuple, options) 185 | } 186 | 187 | function decodeObjectValue(str: string, options?: EncodingOptions) { 188 | const entries = decodeTuple(str, options) as Array<[string, Value]> 189 | const obj = {} 190 | for (const [key, value] of entries) { 191 | obj[key] = value 192 | } 193 | return obj 194 | } 195 | -------------------------------------------------------------------------------- /src/examples/endUserDatabase.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "bun:test" 2 | import { transactionalReadWrite } from "../database/sync/transactionalReadWrite" 3 | import { TupleDatabase } from "../database/sync/TupleDatabase" 4 | import { TupleDatabaseClient } from "../database/sync/TupleDatabaseClient" 5 | import { compareTuple } from "../helpers/compareTuple" 6 | import { ReadOnlyTupleDatabaseClientApi, SchemaSubspace } from "../main" 7 | import { InMemoryTupleStorage } from "../storage/InMemoryTupleStorage" 8 | import { 9 | $, 10 | evaluateQuery, 11 | Fact, 12 | Query, 13 | substituteBinding, 14 | TriplestoreSchema, 15 | Value, 16 | writeFact, 17 | } from "./triplestore" 18 | 19 | // We're going to build off of the triplestore example. 20 | // So read triplestore.ts and triplestore.test.ts first. 21 | 22 | type Obj = { id: string; [key: string]: Value | Value[] } 23 | 24 | // Represent objects that we're typically used to as triples. 25 | function objectToFacts(obj: Obj) { 26 | const facts: Fact[] = [] 27 | const { id, ...rest } = obj 28 | for (const [key, value] of Object.entries(rest)) { 29 | if (Array.isArray(value)) { 30 | for (const item of value) { 31 | facts.push([id, key, item]) 32 | } 33 | } else { 34 | facts.push([id, key, value]) 35 | } 36 | } 37 | facts.sort(compareTuple) 38 | return facts 39 | } 40 | 41 | describe("objectToFacts", () => { 42 | it("works", () => { 43 | expect( 44 | objectToFacts({ 45 | id: "1", 46 | name: "Chet", 47 | age: 31, 48 | tags: ["engineer", "musician"], 49 | }) 50 | ).toEqual([ 51 | ["1", "age", 31], 52 | ["1", "name", "Chet"], 53 | ["1", "tags", "engineer"], 54 | ["1", "tags", "musician"], 55 | ]) 56 | }) 57 | }) 58 | 59 | // A user-defined query filters objects based on 1 or more properties. 60 | type UserFilter = { id: string; [prop: string]: Value } 61 | 62 | function userFilterToQuery(filter: UserFilter): Query { 63 | const { id, ...props } = filter 64 | return Object.entries(props).map(([a, v]) => [$("id"), a, v]) 65 | } 66 | 67 | type Schema = 68 | | SchemaSubspace<["data"], TriplestoreSchema> 69 | // A list of user-defined filters. 70 | | { key: ["filter", string]; value: UserFilter } 71 | // And index for all objects ids that pass the filter. 72 | | { key: ["index", string, string]; value: null } 73 | 74 | const reindexFact = transactionalReadWrite()((tx, fact: Fact) => { 75 | const [e, a, v] = fact 76 | 77 | // Get all the user-defined filters. 78 | const filters = tx.scan({ prefix: ["filter"] }).map(({ value }) => value) 79 | 80 | // Add this object id to the index if it passes the filter. 81 | filters.forEach((filter) => { 82 | // For performance, let's check some trivial cases: 83 | 84 | // This fact is irrelevant to the filter. 85 | if (!(a in filter)) { 86 | return 87 | } 88 | 89 | // This fact directly breaks the filter. 90 | if (v !== filter[a]) { 91 | tx.remove(["index", filter.id, e]) 92 | return 93 | } 94 | 95 | // Evaluate if this object passes the whole filter: 96 | const query = userFilterToQuery(filter) 97 | const testQuery = substituteBinding(query, { id: e }) 98 | const result = evaluateQuery(tx.subspace(["data"]), testQuery) 99 | if (result.length === 0) { 100 | tx.remove(["index", filter.id, e]) 101 | } else { 102 | tx.set(["index", filter.id, e], null) 103 | } 104 | }) 105 | }) 106 | 107 | const writeObjectFact = transactionalReadWrite()((tx, fact: Fact) => { 108 | writeFact(tx.subspace(["data"]), fact) 109 | reindexFact(tx, fact) 110 | }) 111 | 112 | const writeObject = transactionalReadWrite()((tx, obj: Obj) => { 113 | for (const fact of objectToFacts(obj)) { 114 | writeObjectFact(tx, fact) 115 | } 116 | }) 117 | 118 | const createFilter = transactionalReadWrite()( 119 | (tx, filter: UserFilter) => { 120 | tx.set(["filter", filter.id], filter) 121 | 122 | // Evaluate the filter. 123 | const query = userFilterToQuery(filter) 124 | const ids = evaluateQuery(tx.subspace(["data"]), query).map( 125 | ({ id }) => id as string 126 | ) 127 | 128 | // Write those ids to the index. 129 | ids.forEach((id) => { 130 | tx.set(["index", filter.id, id], null) 131 | }) 132 | } 133 | ) 134 | 135 | function readFilterIndex( 136 | db: ReadOnlyTupleDatabaseClientApi, 137 | filterId: string 138 | ) { 139 | return db.scan({ prefix: ["index", filterId] }).map(({ key }) => key[2]) 140 | } 141 | 142 | describe("End-user Database", () => { 143 | it("works", () => { 144 | // Lets try it out. 145 | const db = new TupleDatabaseClient( 146 | new TupleDatabase(new InMemoryTupleStorage()) 147 | ) 148 | 149 | writeObject(db, { 150 | id: "person1", 151 | name: "Chet", 152 | age: 31, 153 | tags: ["engineer", "musician"], 154 | }) 155 | 156 | writeObject(db, { 157 | id: "person2", 158 | name: "Meghan", 159 | age: 30, 160 | tags: ["engineer", "botanist"], 161 | }) 162 | 163 | writeObject(db, { 164 | id: "person3", 165 | name: "Saul", 166 | age: 31, 167 | tags: ["musician"], 168 | }) 169 | 170 | writeObject(db, { 171 | id: "person4", 172 | name: "Tanishq", 173 | age: 22, 174 | tags: [], 175 | }) 176 | 177 | // Create a filter with only one property. 178 | createFilter(db, { id: "filter1", tags: "engineer" }) 179 | expect(readFilterIndex(db, "filter1")).toEqual(["person1", "person2"]) 180 | 181 | // Test that this filter gets maintained. 182 | writeObjectFact(db, ["person4", "tags", "engineer"]) 183 | expect(readFilterIndex(db, "filter1")).toEqual([ 184 | "person1", 185 | "person2", 186 | "person4", 187 | ]) 188 | 189 | // Lets create a filter with two properties. 190 | createFilter(db, { 191 | id: "filter2", 192 | tags: "musician", 193 | age: 31, 194 | }) 195 | expect(readFilterIndex(db, "filter2")).toEqual(["person1", "person3"]) 196 | 197 | // Test that this filter gets maintained. 198 | writeObject(db, { 199 | id: "person5", 200 | name: "Sean", 201 | age: 31, 202 | tags: ["musician", "botanist"], 203 | }) 204 | 205 | expect(readFilterIndex(db, "filter2")).toEqual([ 206 | "person1", 207 | "person3", 208 | "person5", 209 | ]) 210 | }) 211 | }) 212 | -------------------------------------------------------------------------------- /src/storage/AdapterSqliteStorage.ts: -------------------------------------------------------------------------------- 1 | import { 2 | encodeTuple as _encodeTuple, 3 | decodeTuple as _decodeTuple, 4 | } from "../helpers/codec" 5 | import { TupleStorageApi } from "../database/sync/types" 6 | import { AsyncTupleStorageApi } from "../database/async/asyncTypes" 7 | import { KeyValuePair, ScanStorageArgs, WriteOps } from "./types" 8 | 9 | const NULL_BYTE = "\x00" 10 | const DELIMITER_BYTE = "\x01" 11 | const ESCAPE_BYTE = "\x02" 12 | 13 | const codecOptions = { 14 | delimiter: DELIMITER_BYTE, 15 | escape: ESCAPE_BYTE, 16 | disallow: [NULL_BYTE], 17 | } 18 | 19 | function encodeKey(tuple: any[]) { 20 | return _encodeTuple(tuple, codecOptions) 21 | } 22 | 23 | function decodeKey(encoded: string) { 24 | return _decodeTuple(encoded, codecOptions) 25 | } 26 | 27 | function encodeValue(value: any) { 28 | return JSON.stringify(value) 29 | } 30 | 31 | function decodeValue(encoded: string) { 32 | return JSON.parse(encoded) 33 | } 34 | 35 | export interface SQLiteExecutor { 36 | execute(sql: string, args?: any[]): any 37 | } 38 | 39 | export interface SQLiteAdapter extends SQLiteExecutor { 40 | transact(fn: (executor: SQLiteExecutor) => void): void 41 | close(): void 42 | normalizeResults(results: any): { key: string; value: string }[] 43 | } 44 | 45 | export interface AsyncSQLiteExecutor { 46 | execute(sql: string, args?: any[]): Promise 47 | } 48 | export interface AsyncSQLiteAdapter extends AsyncSQLiteExecutor { 49 | transact(fn: (executor: AsyncSQLiteExecutor) => Promise): Promise 50 | close(): Promise 51 | normalizeResults(results: any): { key: string; value: string }[] 52 | } 53 | 54 | export type AdapterSQLiteOptions = { 55 | tableName?: string 56 | } 57 | 58 | type RequiredAdapterSQLiteOptions = Required 59 | 60 | export class AdapterSQLiteStorage implements TupleStorageApi { 61 | private options: RequiredAdapterSQLiteOptions 62 | 63 | constructor( 64 | private adapter: SQLiteAdapter, 65 | options: AdapterSQLiteOptions = {} 66 | ) { 67 | this.options = optionsWithDefaults(options) 68 | this.adapter.execute( 69 | `CREATE TABLE IF NOT EXISTS ${this.options.tableName} (key text primary key, value text)` 70 | ) 71 | } 72 | scan(args: ScanStorageArgs = {}): KeyValuePair[] { 73 | const { sqlQuery, sqlArgs } = scanArgsToSQLQuery(args, this.options) 74 | const result = this.adapter.execute(sqlQuery, sqlArgs) 75 | const data = this.adapter.normalizeResults(result) 76 | return data.map((kv) => { 77 | return { 78 | key: decodeKey(kv.key), 79 | value: decodeValue(kv.value), 80 | } as KeyValuePair 81 | }) 82 | } 83 | commit(writes: WriteOps): void { 84 | this.adapter.transact((tx) => { 85 | for (const { key, value } of writes.set ?? []) { 86 | tx.execute( 87 | `INSERT OR REPLACE INTO ${this.options.tableName} (key, value) VALUES (?, ?)`, 88 | [encodeKey(key), encodeValue(value)] 89 | ) 90 | } 91 | for (const key of writes.remove ?? []) { 92 | tx.execute(`DELETE FROM ${this.options.tableName} WHERE key = ?`, [ 93 | encodeKey(key), 94 | ]) 95 | } 96 | }) 97 | } 98 | close(): void { 99 | this.adapter.close() 100 | } 101 | } 102 | 103 | export class AsyncAdapterSQLiteStorage implements AsyncTupleStorageApi { 104 | private options: RequiredAdapterSQLiteOptions 105 | private dbReady: Promise 106 | 107 | constructor( 108 | private adapter: AsyncSQLiteAdapter, 109 | options: AdapterSQLiteOptions = {} 110 | ) { 111 | this.options = optionsWithDefaults(options) 112 | this.dbReady = this.adapter.execute( 113 | `CREATE TABLE IF NOT EXISTS ${this.options.tableName} (key text primary key, value text)` 114 | ) 115 | } 116 | async scan(args: ScanStorageArgs = {}): Promise { 117 | await this.dbReady 118 | const { sqlQuery, sqlArgs } = scanArgsToSQLQuery(args, this.options) 119 | const result = await this.adapter.execute(sqlQuery, sqlArgs) 120 | const data = this.adapter.normalizeResults(result) 121 | return data.map((kv) => { 122 | return { 123 | key: decodeKey(kv.key), 124 | value: decodeValue(kv.value), 125 | } as KeyValuePair 126 | }) 127 | } 128 | async commit(writes: WriteOps): Promise { 129 | await this.dbReady 130 | await this.adapter.transact(async (tx) => { 131 | for (const { key, value } of writes.set ?? []) { 132 | await tx.execute( 133 | `INSERT OR REPLACE INTO ${this.options.tableName} (key, value) VALUES (?, ?)`, 134 | [encodeKey(key), encodeValue(value)] 135 | ) 136 | } 137 | for (const key of writes.remove ?? []) { 138 | await tx.execute( 139 | `DELETE FROM ${this.options.tableName} WHERE key = ?`, 140 | [encodeKey(key)] 141 | ) 142 | } 143 | }) 144 | } 145 | async close(): Promise { 146 | await this.dbReady 147 | await this.adapter.close() 148 | } 149 | } 150 | 151 | function scanArgsToSQLQuery( 152 | args: ScanStorageArgs, 153 | options: RequiredAdapterSQLiteOptions 154 | ): { 155 | sqlQuery: string 156 | sqlArgs: (string | number)[] 157 | } { 158 | // Bounds. 159 | let start = args.gte ? encodeKey(args.gte) : undefined 160 | let startAfter: string | undefined = args.gt ? encodeKey(args.gt) : undefined 161 | let end: string | undefined = args.lte ? encodeKey(args.lte) : undefined 162 | let endBefore: string | undefined = args.lt ? encodeKey(args.lt) : undefined 163 | 164 | const sqlArgs = [start, startAfter, end, endBefore, args.limit].filter( 165 | Boolean 166 | ) as (string | number)[] 167 | const where = [ 168 | start ? "key >= ?" : undefined, 169 | startAfter ? "key > ?" : undefined, 170 | end ? "key <= ?" : undefined, 171 | endBefore ? "key < ?" : undefined, 172 | ] 173 | .filter(Boolean) 174 | .join(" and ") 175 | 176 | let sqlQuery = `select * from ${options.tableName}` 177 | if (where) { 178 | sqlQuery += " where " 179 | sqlQuery += where 180 | } 181 | sqlQuery += " order by key" 182 | if (args.reverse) { 183 | sqlQuery += " desc" 184 | } 185 | if (args.limit) { 186 | sqlQuery += ` limit ?` 187 | } 188 | return { sqlQuery, sqlArgs } 189 | } 190 | 191 | function optionsWithDefaults( 192 | options: AdapterSQLiteOptions 193 | ): RequiredAdapterSQLiteOptions { 194 | return { 195 | tableName: "data", 196 | ...options, 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /src/examples/classScheduling.test.ts: -------------------------------------------------------------------------------- 1 | // Based on the FoundationDb tutorial: 2 | // https://apple.github.io/foundationdb/class-scheduling.html 3 | 4 | import { flatten, range } from "remeda" 5 | import { describe, it, expect } from "bun:test" 6 | import { transactionalReadWrite } from "../database/sync/transactionalReadWrite" 7 | import { ReadOnlyTupleDatabaseClientApi } from "../database/sync/types" 8 | import { SchemaSubspace } from "../database/typeHelpers" 9 | import { 10 | InMemoryTupleStorage, 11 | TupleDatabase, 12 | TupleDatabaseClient, 13 | } from "../main" 14 | 15 | // Generate 1,620 classes like '9:00 chem for dummies' 16 | const levels = [ 17 | "intro", 18 | "for dummies", 19 | "remedial", 20 | "101", 21 | "201", 22 | "301", 23 | "mastery", 24 | "lab", 25 | "seminar", 26 | ] 27 | 28 | const types = [ 29 | "chem", 30 | "bio", 31 | "cs", 32 | "geometry", 33 | "calc", 34 | "alg", 35 | "film", 36 | "music", 37 | "art", 38 | "dance", 39 | ] 40 | 41 | const times = range(2, 20).map((t) => `${t}:00`) 42 | 43 | const classNames = flatten( 44 | flatten( 45 | levels.map((level) => 46 | types.map((type) => times.map((time) => [level, type, time].join(" "))) 47 | ) 48 | ) 49 | ) 50 | 51 | type SchoolSchema = 52 | | { key: ["class", string]; value: number } 53 | | { key: ["attends", string, string]; value: null } 54 | 55 | const addClass = transactionalReadWrite()( 56 | (tr, className: string, remainingSeats: number) => { 57 | const course = tr.subspace(["class"]) 58 | course.set([className], remainingSeats) 59 | } 60 | ) 61 | 62 | const init = transactionalReadWrite()((tr) => { 63 | // Clear the directory. 64 | for (const { key } of tr.scan()) { 65 | tr.remove(key) 66 | } 67 | 68 | for (const className of classNames) { 69 | addClass(tr, className, 4) 70 | } 71 | }) 72 | 73 | function availableClasses(db: ReadOnlyTupleDatabaseClientApi) { 74 | return db 75 | .subspace(["class"]) 76 | .scan() 77 | .filter(({ value }) => value > 0) 78 | .map(({ key }) => { 79 | const className = key[0] 80 | return className 81 | }) 82 | } 83 | 84 | const signup = transactionalReadWrite()( 85 | (tr, student: string, className: string) => { 86 | const attends = tr.subspace(["attends"]) 87 | const course = tr.subspace(["class"]) 88 | 89 | if (attends.exists([student, className])) return // Already signed up. 90 | 91 | const remainingSeats = course.get([className])! 92 | if (remainingSeats <= 0) throw new Error("No remaining seats.") 93 | 94 | const classes = attends.scan({ prefix: [student] }) 95 | if (classes.length >= 5) throw new Error("Too many classes.") 96 | 97 | course.set([className], remainingSeats - 1) 98 | attends.set([student, className], null) 99 | } 100 | ) 101 | 102 | const drop = transactionalReadWrite()( 103 | (tr, student: string, className: string) => { 104 | const attends = tr.subspace(["attends"]) 105 | const course = tr.subspace(["class"]) 106 | 107 | if (!attends.exists([student, className])) return // Not taking this class. 108 | 109 | const remainingSeats = course.get([className])! 110 | course.set([className], remainingSeats + 1) 111 | attends.remove([student, className]) 112 | } 113 | ) 114 | 115 | const switchClasses = transactionalReadWrite()( 116 | (tr, student: string, classes: { old: string; new: string }) => { 117 | drop(tr, student, classes.old) 118 | signup(tr, student, classes.new) 119 | } 120 | ) 121 | 122 | function getClasses( 123 | db: ReadOnlyTupleDatabaseClientApi, 124 | student: string 125 | ) { 126 | const attends = db.subspace(["attends"]) 127 | const classes = attends.scan({ prefix: [student] }).map(({ key }) => key[1]) 128 | return classes 129 | } 130 | 131 | describe("Class Scheduling Example", () => { 132 | const [class1, class2, class3, class4, class5, class6] = classNames 133 | const [student1, student2, student3, student4, student5] = range(0, 5).map( 134 | (i) => `student${i}` 135 | ) 136 | 137 | function createStorage() { 138 | // The class scheduling application is just a subspace! 139 | type Schema = SchemaSubspace<["scheduling"], SchoolSchema> 140 | const db = new TupleDatabaseClient( 141 | new TupleDatabase(new InMemoryTupleStorage()) 142 | ) 143 | const scheduling = db.subspace(["scheduling"]) 144 | return scheduling 145 | } 146 | 147 | it("signup", () => { 148 | const db = createStorage() 149 | init(db) 150 | 151 | expect(getClasses(db, student1).length).toBe(0) 152 | signup(db, student1, class1) 153 | expect(getClasses(db, student1).length).toBe(1) 154 | }) 155 | 156 | it("signup - already signed up", () => { 157 | const db = createStorage() 158 | init(db) 159 | 160 | expect(getClasses(db, student1).length).toBe(0) 161 | signup(db, student1, class1) 162 | expect(getClasses(db, student1).length).toBe(1) 163 | signup(db, student1, class1) 164 | expect(getClasses(db, student1).length).toBe(1) 165 | }) 166 | 167 | it("signup more than one", () => { 168 | const db = createStorage() 169 | init(db) 170 | 171 | expect(getClasses(db, student1).length).toBe(0) 172 | expect(getClasses(db, student2).length).toBe(0) 173 | 174 | const course = db.subspace(["class"]) 175 | 176 | expect(course.get([class1])).toBe(4) 177 | expect(course.get([class2])).toBe(4) 178 | 179 | signup(db, student1, class1) 180 | expect(getClasses(db, student1).length).toBe(1) 181 | expect(course.get([class1])).toBe(3) 182 | 183 | signup(db, student1, class2) 184 | expect(getClasses(db, student1).length).toBe(2) 185 | expect(course.get([class2])).toBe(3) 186 | 187 | signup(db, student2, class2) 188 | 189 | expect(getClasses(db, student1).length).toBe(2) 190 | expect(getClasses(db, student2).length).toBe(1) 191 | 192 | expect(course.get([class2])).toBe(2) 193 | }) 194 | 195 | it("drop", () => { 196 | const db = createStorage() 197 | init(db) 198 | 199 | expect(getClasses(db, student1).length).toBe(0) 200 | signup(db, student1, class1) 201 | expect(getClasses(db, student1).length).toBe(1) 202 | drop(db, student1, class1) 203 | expect(getClasses(db, student1).length).toBe(0) 204 | }) 205 | 206 | it("drop - not taking this class", () => { 207 | const db = createStorage() 208 | init(db) 209 | 210 | expect(getClasses(db, student1).length).toBe(0) 211 | signup(db, student1, class1) 212 | expect(getClasses(db, student1).length).toBe(1) 213 | drop(db, student1, class2) 214 | expect(getClasses(db, student1).length).toBe(1) 215 | }) 216 | 217 | it("signup - max attendance", () => { 218 | const db = createStorage() 219 | init(db) 220 | 221 | signup(db, student1, class1) 222 | signup(db, student2, class1) 223 | signup(db, student3, class1) 224 | signup(db, student4, class1) 225 | 226 | const course = db.subspace(["class"]) 227 | 228 | expect(course.get([class1])).toBe(0) 229 | expect(() => signup(db, student5, class1)).toThrow() 230 | }) 231 | 232 | it("signup - too many classes", () => { 233 | const db = createStorage() 234 | init(db) 235 | 236 | signup(db, student1, class1) 237 | signup(db, student1, class2) 238 | signup(db, student1, class3) 239 | signup(db, student1, class4) 240 | signup(db, student1, class5) 241 | 242 | expect(getClasses(db, student1).length).toBe(5) 243 | 244 | expect(() => signup(db, student1, class6)).toThrow() 245 | }) 246 | 247 | it("switchClasses", () => { 248 | const db = createStorage() 249 | init(db) 250 | 251 | signup(db, student1, class1) 252 | signup(db, student1, class2) 253 | signup(db, student1, class3) 254 | signup(db, student1, class4) 255 | signup(db, student1, class5) 256 | 257 | expect(getClasses(db, student1).length).toBe(5) 258 | 259 | switchClasses(db, student1, { old: class5, new: class6 }) 260 | const classes = getClasses(db, student1) 261 | expect(classes.length).toBe(5) 262 | expect(classes.includes(class6)).toBeTruthy() 263 | expect(!classes.includes(class5)).toBeTruthy() 264 | }) 265 | 266 | it("availableClasses", () => { 267 | const db = createStorage() 268 | init(db) 269 | 270 | expect(availableClasses(db).includes(class1)).toBeTruthy() 271 | 272 | signup(db, student1, class1) 273 | signup(db, student2, class1) 274 | signup(db, student3, class1) 275 | signup(db, student4, class1) 276 | 277 | expect(!availableClasses(db).includes(class1)).toBeTruthy() 278 | }) 279 | }) 280 | -------------------------------------------------------------------------------- /src/tools/benchmark.ts: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | ./node_modules/.bin/ts-node src/tools/benchmark.ts 4 | 5 | */ 6 | 7 | import sqlite from "better-sqlite3" 8 | import * as fs from "fs-extra" 9 | import { Level } from "level" 10 | import { range } from "remeda" 11 | import * as path from "path" 12 | import { AsyncTupleDatabase } from "../database/async/AsyncTupleDatabase" 13 | import { AsyncTupleDatabaseClientApi } from "../database/async/asyncTypes" 14 | import { transactionalReadWriteAsync } from "../database/async/transactionalReadWriteAsync" 15 | import { AsyncTupleDatabaseClient, InMemoryTupleStorage } from "../main" 16 | import { LevelTupleStorage } from "../storage/LevelTupleStorage" 17 | import { SQLiteTupleStorage } from "../storage/SQLiteTupleStorage" 18 | import * as LMDB from "lmdb" 19 | import { LMDBTupleStorage } from "../storage/LMDBTupleStorage" 20 | import { MemoryBTreeStorage } from "../storage/MemoryBTreeTupleStorage" 21 | 22 | const iterations = 1000 23 | const writeIters = 100 24 | const readSize = 10 25 | const readIters = writeIters / readSize 26 | const tupleSize = 4 27 | 28 | function randomTuple() { 29 | return range(0, tupleSize).map(() => Math.random()) 30 | } 31 | 32 | function randomObjectTuple() { 33 | return range(0, tupleSize).map(() => ({ value: Math.random() })) 34 | } 35 | 36 | function randomArrayTuple() { 37 | return range(0, tupleSize).map(() => [Math.random(), Math.random()]) 38 | } 39 | 40 | const NUM_TUPLES = 10000 41 | 42 | const seedReadRemoveWriteBench = transactionalReadWriteAsync()(async (tx) => { 43 | for (const i of range(0, NUM_TUPLES)) { 44 | tx.set(randomTuple(), null) 45 | } 46 | }) 47 | 48 | const readRemoveWrite = transactionalReadWriteAsync()(async (tx) => { 49 | for (const i of range(0, readIters)) { 50 | const results = await tx.scan({ gt: randomTuple(), limit: 10 }) 51 | for (const { key } of results) { 52 | tx.remove(key) 53 | } 54 | } 55 | for (const i of range(0, writeIters)) { 56 | tx.set(randomTuple(), null) 57 | } 58 | }) 59 | 60 | const seedReadPerformanceBench = transactionalReadWriteAsync()(async (tx) => { 61 | // seed simple tuples 62 | for (const i of range(0, NUM_TUPLES)) { 63 | tx.set(["simpleTuple", ...randomTuple()], null) 64 | } 65 | // seed complex tuples 66 | for (const i of range(0, NUM_TUPLES)) { 67 | tx.set(["objectTuple", ...randomObjectTuple()], null) 68 | } 69 | 70 | // seed complex tuples 71 | for (const i of range(0, NUM_TUPLES)) { 72 | tx.set(["arrayTuple", ...randomArrayTuple()], null) 73 | } 74 | }) 75 | 76 | const readSimpleTuples = transactionalReadWriteAsync()(async (tx) => { 77 | await tx.scan({ prefix: ["simpleTuple"], gte: [0], lt: [1] }) 78 | }) 79 | 80 | const readObjectTuples = transactionalReadWriteAsync()(async (tx) => { 81 | await tx.scan({ 82 | prefix: ["objectTuple"], 83 | gte: [{ value: 0 }], 84 | lt: [{ value: 1 }], 85 | }) 86 | }) 87 | 88 | const readArrayTuples = transactionalReadWriteAsync()(async (tx) => { 89 | await tx.scan({ 90 | prefix: ["arrayTuple"], 91 | gte: [[0, 0]], 92 | lt: [[1, 1]], 93 | }) 94 | }) 95 | 96 | async function timeIt(label: string, fn: () => Promise) { 97 | const start = performance.now() 98 | await fn() 99 | const end = performance.now() 100 | console.log(label, end - start) 101 | } 102 | 103 | async function asyncReadRemoveWriteBenchmark( 104 | label: string, 105 | db: AsyncTupleDatabaseClientApi 106 | ) { 107 | await timeIt(label + ":seedReadRemoveWriteBench", () => 108 | seedReadRemoveWriteBench(db) 109 | ) 110 | 111 | await timeIt(label + ":readRemoveWrite", async () => { 112 | for (const i of range(0, iterations)) { 113 | await readRemoveWrite(db) 114 | } 115 | }) 116 | } 117 | 118 | export function asyncWriteOnlyBenchmark( 119 | label: string, 120 | db: AsyncTupleDatabaseClientApi 121 | ) { 122 | return timeIt(label + ":writeOnly", async () => { 123 | const tx = db.transact() 124 | for (const i of range(0, iterations)) { 125 | tx.set(randomTuple(), null) 126 | } 127 | await tx.commit() 128 | }) 129 | } 130 | 131 | async function asyncReadPerformanceBenchmark( 132 | label: string, 133 | db: AsyncTupleDatabaseClientApi 134 | ) { 135 | await timeIt(label + ":seedReadPerformanceBench", () => 136 | seedReadPerformanceBench(db) 137 | ) 138 | 139 | await timeIt(label + ":readSimpleTuples", async () => { 140 | for (const i of range(0, iterations)) { 141 | await readSimpleTuples(db) 142 | } 143 | }) 144 | 145 | await timeIt(label + ":readObjectTuples", async () => { 146 | for (const i of range(0, iterations)) { 147 | await readObjectTuples(db) 148 | } 149 | }) 150 | 151 | await timeIt(label + ":readArrayTuples", async () => { 152 | for (const i of range(0, iterations)) { 153 | await readArrayTuples(db) 154 | } 155 | }) 156 | } 157 | 158 | const tmpDir = path.resolve(__dirname, "../../tmp") 159 | 160 | async function main() { 161 | await fs.mkdirp(tmpDir) 162 | 163 | // Memory 164 | await asyncWriteOnlyBenchmark( 165 | "Memory", 166 | new AsyncTupleDatabaseClient( 167 | new AsyncTupleDatabase(new InMemoryTupleStorage()) 168 | ) 169 | ) 170 | await asyncReadPerformanceBenchmark( 171 | "Memory", 172 | new AsyncTupleDatabaseClient( 173 | new AsyncTupleDatabase(new InMemoryTupleStorage()) 174 | ) 175 | ) 176 | await asyncReadRemoveWriteBenchmark( 177 | "Memory", 178 | new AsyncTupleDatabaseClient( 179 | new AsyncTupleDatabase(new InMemoryTupleStorage()) 180 | ) 181 | ) 182 | 183 | // Memory BTree 184 | await asyncWriteOnlyBenchmark( 185 | "Memory BTree", 186 | new AsyncTupleDatabaseClient( 187 | new AsyncTupleDatabase(new MemoryBTreeStorage()) 188 | ) 189 | ) 190 | await asyncReadPerformanceBenchmark( 191 | "Memory BTree", 192 | new AsyncTupleDatabaseClient( 193 | new AsyncTupleDatabase(new MemoryBTreeStorage()) 194 | ) 195 | ) 196 | await asyncReadRemoveWriteBenchmark( 197 | "Memory BTree", 198 | new AsyncTupleDatabaseClient( 199 | new AsyncTupleDatabase(new MemoryBTreeStorage()) 200 | ) 201 | ) 202 | 203 | // LevelDB 204 | // await asyncWriteOnlyBenchmark( 205 | // "Level", 206 | // new AsyncTupleDatabaseClient( 207 | // new AsyncTupleDatabase( 208 | // new LevelTupleStorage( 209 | // new Level(path.join(tmpDir, "benchmark-level.db")) 210 | // ) 211 | // ) 212 | // ) 213 | // ) 214 | // await asyncReadPerformanceBenchmark( 215 | // "Level", 216 | // new AsyncTupleDatabaseClient( 217 | // new AsyncTupleDatabase( 218 | // new LevelTupleStorage( 219 | // new Level(path.join(tmpDir, "benchmark-level.db")) 220 | // ) 221 | // ) 222 | // ) 223 | // ) 224 | // await asyncReadRemoveWriteBenchmark( 225 | // "Level", 226 | // new AsyncTupleDatabaseClient( 227 | // new AsyncTupleDatabase( 228 | // new LevelTupleStorage( 229 | // new Level(path.join(tmpDir, "benchmark-level.db")) 230 | // ) 231 | // ) 232 | // ) 233 | // ) 234 | 235 | // SQLite 236 | await asyncWriteOnlyBenchmark( 237 | "SQLite", 238 | new AsyncTupleDatabaseClient( 239 | new AsyncTupleDatabase( 240 | new SQLiteTupleStorage(sqlite(path.join(tmpDir, "benchmark-sqlite.db"))) 241 | ) 242 | ) 243 | ) 244 | await asyncReadPerformanceBenchmark( 245 | "SQLite", 246 | new AsyncTupleDatabaseClient( 247 | new AsyncTupleDatabase( 248 | new SQLiteTupleStorage(sqlite(path.join(tmpDir, "benchmark-sqlite.db"))) 249 | ) 250 | ) 251 | ) 252 | await asyncReadRemoveWriteBenchmark( 253 | "SQLite", 254 | new AsyncTupleDatabaseClient( 255 | new AsyncTupleDatabase( 256 | new SQLiteTupleStorage(sqlite(path.join(tmpDir, "benchmark-sqlite.db"))) 257 | ) 258 | ) 259 | ) 260 | 261 | // LMDB 262 | await asyncWriteOnlyBenchmark( 263 | "LMDB", 264 | new AsyncTupleDatabaseClient( 265 | new AsyncTupleDatabase( 266 | new LMDBTupleStorage((options) => 267 | LMDB.open(path.join(tmpDir, "benchmark-lmdb-write.db"), { 268 | ...options, 269 | }) 270 | ) 271 | ) 272 | ) 273 | ) 274 | await asyncReadPerformanceBenchmark( 275 | "LMDB", 276 | new AsyncTupleDatabaseClient( 277 | new AsyncTupleDatabase( 278 | new LMDBTupleStorage((options) => 279 | LMDB.open(path.join(tmpDir, "benchmark-lmdb.db"), { ...options }) 280 | ) 281 | ) 282 | ) 283 | ) 284 | await asyncReadRemoveWriteBenchmark( 285 | "LMDB", 286 | new AsyncTupleDatabaseClient( 287 | new AsyncTupleDatabase( 288 | new LMDBTupleStorage((options) => 289 | LMDB.open(path.join(tmpDir, "benchmark-lmdb.db"), { ...options }) 290 | ) 291 | ) 292 | ) 293 | ) 294 | } 295 | 296 | main() 297 | -------------------------------------------------------------------------------- /src/helpers/sortedTupleArray.test.ts: -------------------------------------------------------------------------------- 1 | import * as _ from "remeda" 2 | import { describe, it, expect } from "bun:test" 3 | import { MAX, MIN, Tuple } from "../storage/types" 4 | import { 5 | getPrefixContainingBounds, 6 | isTupleWithinBounds, 7 | MaxTuple, 8 | normalizeTupleBounds, 9 | scan, 10 | set, 11 | } from "./sortedTupleArray" 12 | 13 | describe("sortedTupleArray", () => { 14 | describe("prefix basics", () => { 15 | const items: Tuple[] = [ 16 | [], 17 | ["a"], 18 | ["a", "a"], 19 | ["a", "b"], 20 | ["b"], 21 | ["b", "a"], 22 | ["b", "b"], 23 | ] 24 | 25 | it("sorts prefixes in the correct order", () => { 26 | const data: Tuple[] = [] 27 | for (const item of _.shuffle(items)) { 28 | set(data, item) 29 | } 30 | expect(data).toEqual(items) 31 | }) 32 | 33 | it("prefix", () => { 34 | const result = scan(items, { prefix: ["a"] }) 35 | expect(result).toEqual([["a"], ["a", "a"], ["a", "b"]]) 36 | }) 37 | 38 | it("prefix limit", () => { 39 | const result = scan(items, { prefix: ["a"], limit: 2 }) 40 | expect(result).toEqual([["a"], ["a", "a"]]) 41 | }) 42 | 43 | it("prefix limit truncated", () => { 44 | const result = scan(items, { prefix: ["a"], limit: 10 }) 45 | expect(result).toEqual([["a"], ["a", "a"], ["a", "b"]]) 46 | }) 47 | 48 | it("prefix reverse", () => { 49 | const result = scan(items, { prefix: ["a"], reverse: true }) 50 | expect(result).toEqual([["a", "b"], ["a", "a"], ["a"]]) 51 | }) 52 | 53 | it("prefix reverse limit", () => { 54 | const result = scan(items, { prefix: ["a"], limit: 2, reverse: true }) 55 | expect(result).toEqual([ 56 | ["a", "b"], 57 | ["a", "a"], 58 | ]) 59 | }) 60 | 61 | it("prefix reverse limit truncated", () => { 62 | const result = scan(items, { prefix: ["a"], limit: 10, reverse: true }) 63 | expect(result).toEqual([["a", "b"], ["a", "a"], ["a"]]) 64 | }) 65 | }) 66 | 67 | describe("prefix composition", () => { 68 | const items: Tuple[] = [ 69 | ["a", "a", "a"], 70 | ["a", "a", "b"], 71 | ["a", "a", "c"], 72 | ["a", "b", "a"], 73 | ["a", "b", "b"], 74 | ["a", "b", "c"], 75 | ["a", "c", "a"], 76 | ["a", "c", "b"], 77 | ["a", "c", "c"], 78 | ["b", "a", "a"], 79 | ["b", "a", "b"], 80 | ["b", "a", "c"], 81 | ["b", "b", "a"], 82 | ["b", "b", "b"], 83 | ["b", "b", "c"], 84 | ["b", "c", "a"], 85 | ["b", "c", "b"], 86 | ["b", "c", "c"], 87 | ] 88 | 89 | it("prefix gt", () => { 90 | const result = scan(items, { prefix: ["a"], gt: ["a", MAX] }) 91 | expect(result).toEqual([ 92 | ["a", "b", "a"], 93 | ["a", "b", "b"], 94 | ["a", "b", "c"], 95 | ["a", "c", "a"], 96 | ["a", "c", "b"], 97 | ["a", "c", "c"], 98 | ]) 99 | }) 100 | 101 | it("prefix gt reverse", () => { 102 | const result = scan(items, { 103 | prefix: ["a"], 104 | gt: ["a", MAX], 105 | reverse: true, 106 | }) 107 | expect(result).toEqual( 108 | [ 109 | ["a", "b", "a"], 110 | ["a", "b", "b"], 111 | ["a", "b", "c"], 112 | ["a", "c", "a"], 113 | ["a", "c", "b"], 114 | ["a", "c", "c"], 115 | ].reverse() 116 | ) 117 | }) 118 | 119 | it("prefix lt", () => { 120 | const result = scan(items, { prefix: ["a"], lt: ["b"] }) 121 | expect(result).toEqual([ 122 | ["a", "a", "a"], 123 | ["a", "a", "b"], 124 | ["a", "a", "c"], 125 | ]) 126 | }) 127 | 128 | it("prefix lt reverse", () => { 129 | const result = scan(items, { prefix: ["a"], lt: ["b"], reverse: true }) 130 | expect(result).toEqual( 131 | [ 132 | ["a", "a", "a"], 133 | ["a", "a", "b"], 134 | ["a", "a", "c"], 135 | ].reverse() 136 | ) 137 | }) 138 | 139 | it("prefix gt/lt", () => { 140 | const result = scan(items, { prefix: ["a"], gt: ["a", MAX], lt: ["c"] }) 141 | expect(result).toEqual([ 142 | ["a", "b", "a"], 143 | ["a", "b", "b"], 144 | ["a", "b", "c"], 145 | ]) 146 | }) 147 | 148 | it("prefix gt/lt reverse", () => { 149 | const result = scan(items, { 150 | prefix: ["a"], 151 | gt: ["a", MAX], 152 | lt: ["c"], 153 | reverse: true, 154 | }) 155 | expect(result).toEqual( 156 | [ 157 | ["a", "b", "a"], 158 | ["a", "b", "b"], 159 | ["a", "b", "c"], 160 | ].reverse() 161 | ) 162 | }) 163 | 164 | it("prefix gte", () => { 165 | const result = scan(items, { prefix: ["a"], gte: ["b"] }) 166 | expect(result).toEqual([ 167 | ["a", "b", "a"], 168 | ["a", "b", "b"], 169 | ["a", "b", "c"], 170 | ["a", "c", "a"], 171 | ["a", "c", "b"], 172 | ["a", "c", "c"], 173 | ]) 174 | }) 175 | 176 | it("prefix gte reverse", () => { 177 | const result = scan(items, { prefix: ["a"], gte: ["b"], reverse: true }) 178 | expect(result).toEqual( 179 | [ 180 | ["a", "b", "a"], 181 | ["a", "b", "b"], 182 | ["a", "b", "c"], 183 | ["a", "c", "a"], 184 | ["a", "c", "b"], 185 | ["a", "c", "c"], 186 | ].reverse() 187 | ) 188 | }) 189 | 190 | it("prefix lte", () => { 191 | const result = scan(items, { prefix: ["a"], lte: ["a", MAX] }) 192 | expect(result).toEqual([ 193 | ["a", "a", "a"], 194 | ["a", "a", "b"], 195 | ["a", "a", "c"], 196 | ]) 197 | }) 198 | 199 | it("prefix lte reverse", () => { 200 | const result = scan(items, { 201 | prefix: ["a"], 202 | lte: ["a", MAX], 203 | reverse: true, 204 | }) 205 | expect(result).toEqual( 206 | [ 207 | ["a", "a", "a"], 208 | ["a", "a", "b"], 209 | ["a", "a", "c"], 210 | ].reverse() 211 | ) 212 | }) 213 | 214 | it("prefix gte/lte", () => { 215 | const result = scan(items, { prefix: ["a"], gte: ["b"], lte: ["c", MAX] }) 216 | expect(result).toEqual([ 217 | ["a", "b", "a"], 218 | ["a", "b", "b"], 219 | ["a", "b", "c"], 220 | ["a", "c", "a"], 221 | ["a", "c", "b"], 222 | ["a", "c", "c"], 223 | ]) 224 | }) 225 | 226 | it("prefix gte/lte reverse", () => { 227 | const result = scan(items, { 228 | prefix: ["a"], 229 | gte: ["b"], 230 | lte: ["c", MAX], 231 | reverse: true, 232 | }) 233 | expect(result).toEqual( 234 | [ 235 | ["a", "b", "a"], 236 | ["a", "b", "b"], 237 | ["a", "b", "c"], 238 | ["a", "c", "a"], 239 | ["a", "c", "b"], 240 | ["a", "c", "c"], 241 | ].reverse() 242 | ) 243 | }) 244 | }) 245 | 246 | describe("bounds", () => { 247 | const items: Tuple[] = [ 248 | [], 249 | ["a"], 250 | ["a", "a"], 251 | ["a", "b"], 252 | ["a", "c"], 253 | ["b"], 254 | ["b", "a"], 255 | ["b", "b"], 256 | ["b", "c"], 257 | ] 258 | 259 | it("prefix gt MIN", () => { 260 | const result = scan(items, { prefix: ["a"], gt: [MIN] }) 261 | expect(result).toEqual([ 262 | ["a", "a"], 263 | ["a", "b"], 264 | ["a", "c"], 265 | ]) 266 | }) 267 | 268 | it("prefix gt MIN reverse", () => { 269 | const result = scan(items, { prefix: ["a"], gt: [MIN], reverse: true }) 270 | expect(result).toEqual([ 271 | ["a", "c"], 272 | ["a", "b"], 273 | ["a", "a"], 274 | ]) 275 | }) 276 | 277 | it("prefix gt", () => { 278 | const result = scan(items, { prefix: ["a"], gt: ["a"] }) 279 | expect(result).toEqual([ 280 | ["a", "b"], 281 | ["a", "c"], 282 | ]) 283 | }) 284 | 285 | it("prefix gt reverse", () => { 286 | const result = scan(items, { prefix: ["a"], gt: ["a"], reverse: true }) 287 | expect(result).toEqual([ 288 | ["a", "c"], 289 | ["a", "b"], 290 | ]) 291 | }) 292 | 293 | it("prefix lt MAX", () => { 294 | const result = scan(items, { prefix: ["a"], lt: [MAX] }) 295 | expect(result).toEqual([["a"], ["a", "a"], ["a", "b"], ["a", "c"]]) 296 | }) 297 | 298 | it("prefix lt MAX reverse", () => { 299 | const result = scan(items, { prefix: ["a"], lt: [MAX], reverse: true }) 300 | expect(result).toEqual([["a", "c"], ["a", "b"], ["a", "a"], ["a"]]) 301 | }) 302 | 303 | it("prefix lt", () => { 304 | const result = scan(items, { prefix: ["a"], lt: ["c"] }) 305 | expect(result).toEqual([["a"], ["a", "a"], ["a", "b"]]) 306 | }) 307 | 308 | it("prefix lt reverse", () => { 309 | const result = scan(items, { prefix: ["a"], lt: ["c"], reverse: true }) 310 | expect(result).toEqual([["a", "b"], ["a", "a"], ["a"]]) 311 | }) 312 | }) 313 | 314 | describe("normalizeTupleBounds", () => { 315 | it("normalized prefix", () => { 316 | expect(normalizeTupleBounds({ prefix: ["a"] })).toEqual({ 317 | gte: ["a"], // NOTE: this is not ["a", MIN] 318 | lte: ["a", ...MaxTuple], 319 | }) 320 | }) 321 | 322 | it("prepends prefix to constraints", () => { 323 | expect(normalizeTupleBounds({ prefix: ["a"], gte: ["b"] })).toEqual({ 324 | gte: ["a", "b"], 325 | lte: ["a", ...MaxTuple], 326 | }) 327 | }) 328 | }) 329 | 330 | describe("prefixTupleBounds", () => { 331 | it("Computes trivial bounds prefix", () => { 332 | expect( 333 | getPrefixContainingBounds({ gte: ["a", 1], lte: ["a", 10] }) 334 | ).toEqual(["a"]) 335 | }) 336 | 337 | it("Handles entirely disjoint tuples", () => { 338 | expect( 339 | getPrefixContainingBounds({ gte: ["a", 1], lte: ["b", 10] }) 340 | ).toEqual([]) 341 | }) 342 | }) 343 | 344 | describe("isTupleWithinBounds", () => { 345 | it("Works for exact equality range", () => { 346 | expect(isTupleWithinBounds(["a"], { gte: ["a"], lte: ["a"] })).toEqual( 347 | true 348 | ) 349 | expect(isTupleWithinBounds(["a", 1], { gte: ["a"], lte: ["a"] })).toEqual( 350 | false 351 | ) 352 | }) 353 | 354 | it("Works for non-trivial range", () => { 355 | expect(isTupleWithinBounds(["a"], { gt: ["a"], lte: ["b"] })).toEqual( 356 | false 357 | ) 358 | expect(isTupleWithinBounds(["a", 1], { gt: ["a"], lte: ["b"] })).toEqual( 359 | true 360 | ) 361 | expect(isTupleWithinBounds(["b"], { gt: ["a"], lte: ["b"] })).toEqual( 362 | true 363 | ) 364 | expect(isTupleWithinBounds(["b", 1], { gt: ["a"], lte: ["b"] })).toEqual( 365 | false 366 | ) 367 | }) 368 | }) 369 | }) 370 | --------------------------------------------------------------------------------