├── src ├── version.ts ├── fireproof.ts ├── write-queue.ts ├── store.ts ├── loader-helpers.ts ├── crdt.ts ├── store-fs.ts ├── types.d.ts ├── database.ts ├── store-browser.ts ├── encrypt-helpers.ts ├── encrypted-block.ts ├── crypto.ts ├── transaction.ts ├── crdt-helpers.ts ├── loader.ts ├── indexer-helpers.ts └── index.ts ├── .gitignore ├── test ├── globals.d.ts ├── helpers.js ├── www │ └── iife.html ├── hello.test.js ├── store-fs.test.js ├── transaction.test.js ├── database.test.js ├── crdt.test.js ├── loader.test.js ├── fireproof.test.js └── indexer.test.js ├── .vscode └── launch.json ├── scripts ├── test.js ├── serve.js ├── build.js ├── analyze.js ├── types.js ├── browser-test.js ├── server.js └── settings.js ├── .github └── workflows │ └── test.yml ├── tsconfig.json ├── .eslintrc.cjs ├── README.md └── package.json /src/version.ts: -------------------------------------------------------------------------------- 1 | export const PACKAGE_VERSION = "0.10.67"; 2 | -------------------------------------------------------------------------------- /src/fireproof.ts: -------------------------------------------------------------------------------- 1 | export * from './database' 2 | export * from './index' 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | .DS_Store 4 | dist 5 | build 6 | examples 7 | *.zip -------------------------------------------------------------------------------- /test/globals.d.ts: -------------------------------------------------------------------------------- 1 | declare global { 2 | function describe(description: string, callback: () => void): void; 3 | function it(description: string, callback: () => void): void; 4 | } 5 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "msedge", 6 | "request": "launch", 7 | "name": "Launch Edge against localhost", 8 | "url": "http://localhost:5505", 9 | "webRoot": "${workspaceFolder}/www" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /scripts/test.js: -------------------------------------------------------------------------------- 1 | import { spawn } from 'child_process' 2 | const args = process.argv.slice(2) 3 | 4 | let command = 'mocha test/*.js' 5 | 6 | if (args.length > 0) { 7 | command += ` --grep '${args.join(' ')}'` 8 | } 9 | 10 | const mocha = spawn(command, { stdio: 'inherit', shell: true }) 11 | 12 | mocha.on('close', (code) => { 13 | process.exit(code) 14 | }) 15 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | 7 | jobs: 8 | build: 9 | 10 | runs-on: ubuntu-latest 11 | 12 | strategy: 13 | matrix: 14 | node-version: [16.x] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Use Node.js ${{ matrix.node-version }} 19 | uses: actions/setup-node@v2 20 | with: 21 | node-version: ${{ matrix.node-version }} 22 | - run: npm ci 23 | - run: npm test 24 | env: 25 | CI: true 26 | -------------------------------------------------------------------------------- /scripts/serve.js: -------------------------------------------------------------------------------- 1 | import esbuild from 'esbuild' 2 | import { createBuildSettings } from './settings.js' 3 | 4 | const settings = createBuildSettings({ 5 | sourcemap: true, 6 | banner: { 7 | js: 'new EventSource(\'/esbuild\').addEventListener(\'change\', () => location.reload())' 8 | } 9 | }) 10 | 11 | const ctx = await esbuild.context(settings) 12 | 13 | await ctx.watch() 14 | 15 | const { host, port } = await ctx.serve({ 16 | port: 5505, 17 | servedir: 'www' 18 | }) 19 | 20 | console.log(`Serving app at ${host}:${port}.`) 21 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": ["src", "test", ".eslintrc.cjs", "scripts"], 3 | "compilerOptions": { 4 | "module": "esnext", 5 | "target": "esnext", 6 | "lib": ["dom", "dom.iterable", "esnext"], 7 | "strict": true, 8 | "moduleResolution": "node", 9 | "jsx": "react", 10 | "skipLibCheck": true, 11 | "resolveJsonModule": true, 12 | "esModuleInterop": true, 13 | "emitDeclarationOnly": true, 14 | "declarationDir": "dist/types", 15 | "declaration": true, 16 | "outDir": "dist", 17 | "rootDir": "src", 18 | "forceConsistentCasingInFileNames": true 19 | } 20 | } -------------------------------------------------------------------------------- /scripts/build.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/require-await */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-member-access */ 3 | import { build } from 'esbuild' 4 | import { createBuildSettings } from './settings.js' 5 | 6 | async function buildProject() { 7 | const buildConfigs = createBuildSettings() 8 | 9 | for (const config of buildConfigs) { 10 | console.log('Building', config.outfile) 11 | build(config).catch(() => { 12 | console.log('Error', config.outfile) 13 | }) 14 | } 15 | } 16 | 17 | buildProject().catch((err) => { 18 | console.error(err) 19 | process.exit(1) 20 | }) 21 | -------------------------------------------------------------------------------- /scripts/analyze.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-member-access */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-argument */ 3 | import * as esbuild from 'esbuild' 4 | import fs from 'fs' 5 | import { createBuildSettings } from './settings.js' 6 | 7 | const mode = process.env.npm_config_mode 8 | 9 | async function analyzeProject() { 10 | const buildConfigs = createBuildSettings({ minify: true, metafile: true }) 11 | 12 | for (const config of buildConfigs) { 13 | if (!/fireproof/.test(config.outfile)) continue 14 | try { 15 | const result = await esbuild.build(config) 16 | 17 | if (mode === 'write') { 18 | fs.writeFileSync(`build-meta-${result.format}.json`, JSON.stringify(result.metafile)) 19 | } else { 20 | console.log(await esbuild.analyzeMetafile(result.metafile, { 21 | verbose: false 22 | })) 23 | } 24 | } catch (err) { 25 | console.error(err) 26 | } 27 | } 28 | } 29 | 30 | analyzeProject().catch((err) => { 31 | console.error(err) 32 | process.exit(1) 33 | }) 34 | -------------------------------------------------------------------------------- /src/write-queue.ts: -------------------------------------------------------------------------------- 1 | import { BulkResult, DocUpdate } from './types' 2 | 3 | type WorkerFunction = (tasks: DocUpdate[]) => Promise; 4 | 5 | export type WriteQueue = { 6 | push(task: DocUpdate): Promise; 7 | }; 8 | 9 | export function writeQueue(worker: WorkerFunction, payload: number = Infinity): WriteQueue { 10 | const queue: { task: DocUpdate; resolve: (result: BulkResult) => void; }[] = [] 11 | let isProcessing = false 12 | 13 | async function process() { 14 | if (isProcessing || queue.length === 0) return 15 | isProcessing = true 16 | 17 | const tasksToProcess = queue.splice(0, payload) 18 | const updates = tasksToProcess.map(item => item.task) 19 | 20 | const result = await worker(updates) 21 | 22 | tasksToProcess.forEach(task => task.resolve(result)) 23 | 24 | isProcessing = false 25 | void process() 26 | } 27 | 28 | return { 29 | push(task: DocUpdate): Promise { 30 | return new Promise((resolve) => { 31 | queue.push({ task, resolve }) 32 | void process() 33 | }) 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /scripts/types.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import path from 'path' 3 | // import { createBuildSettings } from './settings.js' // import your build settings 4 | 5 | // Get the entry points from your build settings 6 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-member-access 7 | // const entryPoints = ['database', 'index', 'types']// createBuildSettings({}).map(config => path.basename(config.entryPoints[0], '.ts')) 8 | 9 | function generateIndexFile() { 10 | const typesDir = path.resolve('dist/types') 11 | const srcTypesPath = path.resolve('src/types.d.ts') 12 | const typeIndexPath = path.resolve('dist/types/fireproof.d.ts') 13 | 14 | // Read the contents of src/types.d.ts 15 | const srcTypesContent = fs.readFileSync(srcTypesPath, 'utf8') 16 | 17 | // Write the contents to dist/types/types.d.ts 18 | fs.writeFileSync(path.join(typesDir, 'types.d.ts'), srcTypesContent, 'utf8') 19 | 20 | const toAppend = '\nexport * from \'./types\';\n' 21 | const typeIndexContent = fs.readFileSync(typeIndexPath, 'utf8') 22 | fs.writeFileSync(typeIndexPath, typeIndexContent + toAppend, 'utf8') 23 | } 24 | 25 | // Call this function as part of your build process 26 | generateIndexFile() 27 | -------------------------------------------------------------------------------- /.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line no-undef 2 | module.exports = { 3 | env: { 4 | mocha: true 5 | }, 6 | plugins: [ 7 | 'mocha' 8 | ], 9 | ignorePatterns: ['dist/'], 10 | parser: '@typescript-eslint/parser', 11 | extends: [ 12 | 'standard', 13 | 'plugin:@typescript-eslint/recommended', 14 | 'plugin:@typescript-eslint/recommended-requiring-type-checking', 15 | 'plugin:mocha/recommended' 16 | ], 17 | parserOptions: { 18 | ecmaVersion: 2020, 19 | sourceType: 'module', 20 | project: './tsconfig.json' 21 | }, 22 | rules: { 23 | '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }], 24 | '@typescript-eslint/no-use-before-define': ['error', { functions: false, classes: false, variables: true, typedefs: false }], 25 | '@typescript-eslint/no-explicit-any': 'off', 26 | '@typescript-eslint/ban-ts-comment': 'off', 27 | '@typescript-eslint/explicit-function-return-type': 'off', 28 | '@typescript-eslint/type-annotation-spacing': 'error', 29 | 'no-use-before-define': 'off', 30 | 'no-void': 'off', 31 | 'space-before-function-paren': ['error', { 32 | anonymous: 'always', 33 | named: 'never', 34 | asyncArrow: 'always' 35 | }], 36 | semi: ['error', 'never'] 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /test/helpers.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-argument */ 2 | import assert from 'assert' 3 | import { join } from 'path' 4 | import { promises } from 'fs' 5 | 6 | const { mkdir, readdir, rm } = promises 7 | 8 | export { assert } 9 | 10 | export function equals(actual, expected) { 11 | assert(actual === expected, `Expected '${actual}' to equal '${expected}'`) 12 | } 13 | 14 | export function equalsJSON(actual, expected) { 15 | equals(JSON.stringify(actual), JSON.stringify(expected)) 16 | } 17 | 18 | export function notEquals(actual, expected) { 19 | assert(actual !== expected, `Expected '${actual} 'to not equal '${expected}'`) 20 | } 21 | 22 | export function matches(actual, expected) { 23 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 24 | assert(actual.toString().match(expected), `Expected '${actual}' to match ${expected}`) 25 | } 26 | 27 | export async function resetDirectory(dir, name) { 28 | await doResetDirectory(dir, name) 29 | await doResetDirectory(dir, name + '.idx') 30 | } 31 | 32 | export async function doResetDirectory(dir, name) { 33 | const path = join(dir, name) 34 | await mkdir(path, { recursive: true }) 35 | 36 | const files = await readdir(path) 37 | 38 | for (const file of files) { 39 | await rm(join(path, file), { recursive: false, force: true }) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/store.ts: -------------------------------------------------------------------------------- 1 | import { format, parse, ToString } from '@ipld/dag-json' 2 | import { AnyBlock, AnyLink, DbMeta } from './types' 3 | 4 | import { PACKAGE_VERSION } from './version' 5 | const match = PACKAGE_VERSION.match(/^([^.]*\.[^.]*)/) 6 | if (!match) throw new Error('invalid version: ' + PACKAGE_VERSION) 7 | export const STORAGE_VERSION = match[0] 8 | 9 | abstract class VersionedStore { 10 | STORAGE_VERSION: string = STORAGE_VERSION 11 | } 12 | 13 | export abstract class HeaderStore extends VersionedStore { 14 | tag: string = 'header-base' 15 | name: string 16 | constructor(name: string) { 17 | super() 18 | this.name = name 19 | } 20 | 21 | makeHeader({ car, key }: DbMeta): ToString { 22 | const encoded = format({ car, key } as DbMeta) 23 | return encoded 24 | } 25 | 26 | parseHeader(headerData: ToString): DbMeta { 27 | const got = parse(headerData) 28 | return got 29 | } 30 | 31 | abstract load(branch?: string): Promise 32 | abstract save(dbMeta: DbMeta, branch?: string): Promise 33 | } 34 | 35 | export abstract class CarStore extends VersionedStore { 36 | tag: string = 'car-base' 37 | name: string 38 | constructor(name: string) { 39 | super() 40 | this.name = name 41 | } 42 | 43 | abstract load(cid: AnyLink): Promise 44 | abstract save(car: AnyBlock): Promise 45 | abstract remove(cid: AnyLink): Promise 46 | } 47 | -------------------------------------------------------------------------------- /test/www/iife.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Fireproof Test 7 | 8 | 42 | 43 | 44 | 45 |
    46 | 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Fireproof 2 | 3 | ## Live database for the web 4 | 5 | Fireproof is beta software. You can browse the alpha proof of concept at https://github.com/fireproof-storage/fireproof 6 | 7 | This code will replace the alpha code soon, at that repository. 8 | 9 | ## Implementation 10 | 11 | Fireproof integrates [Pail](https://github.com/alanshaw/pail) with [Prolly Trees](https://github.com/mikeal/prolly-trees) and vector indexes with a database API. 12 | 13 | ## Build and Lint Process 14 | 15 | The project uses `esbuild` and `esbuild-plugin-tsc` for compiling TypeScript to JavaScript, the results of which are bundled into different module formats for different environments. The `build.js` script in the `scripts` directory orchestrates this process. You can trigger the build process by running `npm run build` in your terminal. 16 | 17 | For code quality assurance, we utilize ESLint with `@typescript-eslint/parser` and `@typescript-eslint/eslint-plugin`. ESLint checks both the JavaScript and TypeScript code in the project. The configuration for ESLint is stored in the `.eslintrc.js` file. For the test files located in the `test` directory, there are additional linting configurations to accommodate Mocha-specific code. 18 | 19 | Tests in this project are written with Mocha and can be run in both Node.js and a browser environment through the use of `polendina`. Test files are located in the `test` directory and are recognizable by their `.test.js` extension. You can run the tests by executing the command `npm test`, which also triggers the build process and checks for any linting errors. -------------------------------------------------------------------------------- /scripts/browser-test.js: -------------------------------------------------------------------------------- 1 | import { launch } from 'puppeteer' 2 | import { join, dirname } from 'path' 3 | import { fileURLToPath } from 'url' 4 | 5 | void (async () => { 6 | const browser = await launch({ headless: 'new' }) 7 | const page = await browser.newPage() 8 | 9 | // Add an event listener to echo console messages from the page 10 | page.on('console', (msg) => { 11 | console.log('console', msg.args().length) 12 | for (let i = 0; i < msg.args().length; ++i) { 13 | console.log(`${i}: ${msg.args()[i].toString()} ${JSON.stringify(msg.args()[i])}`) 14 | } 15 | }) 16 | 17 | // Get the directory of the current module 18 | const currentDir = dirname(fileURLToPath(import.meta.url)) 19 | 20 | // Construct the absolute path to your iife.html file 21 | const filePath = join(currentDir, '../test/www/iife.html') 22 | const url = `file://${filePath}` 23 | 24 | await page.goto(url) 25 | 26 | // Wait for some time to ensure the database operation is complete 27 | // await page.waitForTimeout(1000) 28 | 29 | // Click the button to run onButtonClick 30 | await page.click('button') 31 | 32 | // await page.waitForTimeout(1000) 33 | await page.waitForSelector('li', { timeout: 5000 }) 34 | 35 | // Reload the page to trigger initialize 36 | await page.reload() 37 | 38 | await page.waitForSelector('li') 39 | 40 | // Check if the list contains at least one item 41 | const result = await page.evaluate(() => { 42 | const listItems = document.querySelectorAll('ul > li') 43 | return listItems.length > 0 ? 'success' : 'failure' 44 | }) 45 | 46 | if (result === 'success') { 47 | console.log('Test passed') 48 | process.exit(0) 49 | } else { 50 | console.log('Test failed') 51 | process.exit(1) 52 | } 53 | 54 | await browser.close() 55 | })() 56 | -------------------------------------------------------------------------------- /test/hello.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-return */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-member-access */ 3 | /* eslint-disable @typescript-eslint/no-unsafe-argument */ 4 | /* eslint-disable @typescript-eslint/no-unsafe-call */ 5 | /* eslint-disable @typescript-eslint/no-unsafe-assignment */ 6 | /* eslint-disable mocha/max-top-level-suites */ 7 | import { assert, equals, resetDirectory } from './helpers.js' 8 | import { database, Database } from '../dist/test/database.esm.js' 9 | import { index, Index } from '../dist/test/index.esm.js' 10 | import { testConfig } from '../dist/test/store-fs.esm.js' 11 | 12 | describe('Hello World Test', function () { 13 | it('should pass the hello world test', function () { 14 | const result = database('hello') // call to your library function 15 | assert(result.name === 'hello') 16 | }) 17 | }) 18 | 19 | describe('public API', function () { 20 | beforeEach(async function () { 21 | await resetDirectory(testConfig.dataDir, 'test-api') 22 | this.db = database('test-api') 23 | this.index = index(this.db, 'test-index', (doc) => doc.foo) 24 | this.ok = await this.db.put({ _id: 'test', foo: 'bar' }) 25 | this.doc = await this.db.get('test') 26 | this.query = await this.index.query() 27 | }) 28 | it('should have a database', function () { 29 | assert(this.db) 30 | assert(this.db instanceof Database) 31 | }) 32 | it('should have an index', function () { 33 | assert(this.index) 34 | assert(this.index instanceof Index) 35 | }) 36 | it('should put', function () { 37 | assert(this.ok) 38 | equals(this.ok.id, 'test') 39 | }) 40 | it('should get', function () { 41 | equals(this.doc.foo, 'bar') 42 | }) 43 | it('should query', function () { 44 | assert(this.query) 45 | assert(this.query.rows) 46 | equals(this.query.rows.length, 1) 47 | equals(this.query.rows[0].key, 'bar') 48 | }) 49 | }) 50 | -------------------------------------------------------------------------------- /src/loader-helpers.ts: -------------------------------------------------------------------------------- 1 | import { CID } from 'multiformats' 2 | import { Block, encode, decode } from 'multiformats/block' 3 | import { sha256 as hasher } from 'multiformats/hashes/sha2' 4 | import * as raw from 'multiformats/codecs/raw' 5 | import * as CBW from '@ipld/car/buffer-writer' 6 | import * as codec from '@ipld/dag-cbor' 7 | import { CarReader } from '@ipld/car' 8 | 9 | import { AnyBlock, AnyCarHeader, AnyLink, CarMakeable } from './types' 10 | import { Transaction } from './transaction' 11 | 12 | export async function innerMakeCarFile(fp: AnyCarHeader, t: Transaction): Promise { 13 | const { cid, bytes } = await encodeCarHeader(fp) 14 | await t.put(cid, bytes) 15 | return encodeCarFile(cid, t) 16 | } 17 | 18 | export async function encodeCarFile(carHeaderBlockCid: AnyLink, t: CarMakeable): Promise { 19 | let size = 0 20 | const headerSize = CBW.headerLength({ roots: [carHeaderBlockCid as CID] }) 21 | size += headerSize 22 | for (const { cid, bytes } of t.entries()) { 23 | size += CBW.blockLength({ cid, bytes } as Block) 24 | } 25 | const buffer = new Uint8Array(size) 26 | const writer = CBW.createWriter(buffer, { headerSize }) 27 | 28 | writer.addRoot(carHeaderBlockCid as CID) 29 | 30 | for (const { cid, bytes } of t.entries()) { 31 | writer.write({ cid, bytes } as Block) 32 | } 33 | writer.close() 34 | return await encode({ value: writer.bytes, hasher, codec: raw }) 35 | } 36 | 37 | export async function encodeCarHeader(fp: AnyCarHeader) { 38 | return (await encode({ 39 | value: { fp }, 40 | hasher, 41 | codec 42 | })) as AnyBlock 43 | } 44 | 45 | export async function parseCarFile(reader: CarReader): Promise { 46 | const roots = await reader.getRoots() 47 | const header = await reader.get(roots[0]) 48 | if (!header) throw new Error('missing header block') 49 | const { value } = await decode({ bytes: header.bytes, hasher, codec }) 50 | // @ts-ignore 51 | if (value && value.fp === undefined) throw new Error('missing fp') 52 | const { fp } = value as { fp: AnyCarHeader } 53 | return fp 54 | } 55 | -------------------------------------------------------------------------------- /src/crdt.ts: -------------------------------------------------------------------------------- 1 | import { TransactionBlockstore, IndexBlockstore } from './transaction' 2 | import { clockChangesSince, applyBulkUpdateToCrdt, getValueFromCrdt, doCompact } from './crdt-helpers' 3 | import type { DocUpdate, BulkResult, ClockHead, DbCarHeader, FireproofOptions } from './types' 4 | import type { Index } from './index' 5 | 6 | export class CRDT { 7 | name: string | null 8 | opts: FireproofOptions = {} 9 | ready: Promise 10 | blocks: TransactionBlockstore 11 | indexBlocks: IndexBlockstore 12 | 13 | indexers: Map = new Map() 14 | 15 | private _head: ClockHead = [] 16 | 17 | constructor(name?: string, opts?: FireproofOptions) { 18 | this.name = name || null 19 | this.opts = opts || this.opts 20 | this.blocks = new TransactionBlockstore(name, this.opts) 21 | this.indexBlocks = new IndexBlockstore(name ? name + '.idx' : undefined, this.opts) 22 | this.ready = this.blocks.ready.then((header: DbCarHeader) => { 23 | // @ts-ignore 24 | if (header.indexes) throw new Error('cannot have indexes in crdt header') 25 | if (header.head) { this._head = header.head } // todo multi head support here 26 | }) 27 | } 28 | 29 | async bulk(updates: DocUpdate[], options?: object): Promise { 30 | await this.ready 31 | const tResult = await this.blocks.transaction(async (tblocks): Promise => { 32 | const { head } = await applyBulkUpdateToCrdt(tblocks, this._head, updates, options) 33 | this._head = head // we need multi head support here if allowing calls to bulk in parallel 34 | return { head } 35 | }) 36 | return tResult 37 | } 38 | 39 | // async getAll(rootCache: any = null): Promise<{root: any, cids: CIDCounter, clockCIDs: CIDCounter, result: T[]}> { 40 | 41 | async get(key: string) { 42 | await this.ready 43 | const result = await getValueFromCrdt(this.blocks, this._head, key) 44 | if (result.del) return null 45 | return result 46 | } 47 | 48 | async changes(since: ClockHead = []) { 49 | await this.ready 50 | return await clockChangesSince(this.blocks, this._head, since) 51 | } 52 | 53 | async compact() { 54 | await this.ready 55 | return await doCompact(this.blocks, this._head) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/store-fs.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/first */ 2 | console.log('import store-node-fs') 3 | 4 | import { join, dirname } from 'path' 5 | import { homedir } from 'os' 6 | import { mkdir, readFile, writeFile, unlink } from 'fs/promises' 7 | import type { AnyBlock, AnyLink, DbMeta } from './types' 8 | import { STORAGE_VERSION, HeaderStore as HeaderStoreBase, CarStore as CarStoreBase } from './store' 9 | 10 | export class HeaderStore extends HeaderStoreBase { 11 | tag: string = 'header-node-fs' 12 | keyId: string = 'public' 13 | static dataDir: string = join(homedir(), '.fireproof', 'v' + STORAGE_VERSION) 14 | 15 | async load(branch: string = 'main'): Promise { 16 | const filepath = join(HeaderStore.dataDir, this.name, branch + '.json') 17 | const bytes = await readFile(filepath).catch((e: Error & { code: string }) => { 18 | if (e.code === 'ENOENT') return null 19 | throw e 20 | }) 21 | return bytes ? this.parseHeader(bytes.toString()) : null 22 | } 23 | 24 | async save(meta: DbMeta, branch: string = 'main'): Promise { 25 | const filepath = join(HeaderStore.dataDir, this.name, branch + '.json') 26 | const bytes = this.makeHeader(meta) 27 | await writePathFile(filepath, bytes) 28 | } 29 | } 30 | 31 | export const testConfig = { 32 | dataDir: HeaderStore.dataDir 33 | } 34 | 35 | export class CarStore extends CarStoreBase { 36 | tag: string = 'car-node-fs' 37 | static dataDir: string = join(homedir(), '.fireproof', 'v' + STORAGE_VERSION) 38 | 39 | async save(car: AnyBlock): Promise { 40 | const filepath = join(CarStore.dataDir, this.name, car.cid.toString() + '.car') 41 | await writePathFile(filepath, car.bytes) 42 | } 43 | 44 | async load(cid: AnyLink): Promise { 45 | const filepath = join(CarStore.dataDir, this.name, cid.toString() + '.car') 46 | const bytes = await readFile(filepath) 47 | return { cid, bytes: new Uint8Array(bytes) } 48 | } 49 | 50 | async remove(cid: AnyLink): Promise { 51 | const filepath = join(CarStore.dataDir, this.name, cid.toString() + '.car') 52 | await unlink(filepath) 53 | } 54 | } 55 | 56 | async function writePathFile(path: string, data: Uint8Array | string) { 57 | await mkdir(dirname(path), { recursive: true }) 58 | return await writeFile(path, data) 59 | } 60 | -------------------------------------------------------------------------------- /src/types.d.ts: -------------------------------------------------------------------------------- 1 | import type { Link } from 'multiformats' 2 | import type { EventLink } from '@alanshaw/pail/clock' 3 | import type { EventData } from '@alanshaw/pail/crdt' 4 | 5 | export type FireproofOptions = { 6 | public?: boolean 7 | } 8 | 9 | export type ClockHead = EventLink[] 10 | 11 | export type DocFragment = string | number | boolean | null | DocFragment[] | { [key: string]: DocFragment } 12 | 13 | export type Doc = DocBody & { 14 | _id?: string 15 | } 16 | 17 | type DocBody = { 18 | [key: string]: DocFragment 19 | } 20 | 21 | type DocMeta = { 22 | proof?: DocFragment 23 | clock?: ClockHead 24 | } 25 | 26 | export type DocUpdate = { 27 | key: string 28 | value?: { [key: string]: any} 29 | del?: boolean 30 | } 31 | 32 | export type DocValue = { 33 | doc?: DocBody 34 | del?: boolean 35 | } 36 | 37 | type IndexCars = { 38 | [key: string]: AnyLink 39 | } 40 | 41 | export type IndexKey = [string, string] | string 42 | 43 | export type IndexUpdate = { 44 | key: IndexKey 45 | value?: DocFragment 46 | del?: boolean 47 | } 48 | 49 | export type IndexRow = { 50 | id: string 51 | key: IndexKey 52 | doc?: Doc | null 53 | value?: DocFragment 54 | del?: boolean 55 | } 56 | 57 | type CarCommit = { 58 | car?: AnyLink 59 | } 60 | 61 | export type BulkResult = { 62 | head: ClockHead 63 | } 64 | 65 | type CarHeader = { 66 | cars: AnyLink[] 67 | compact: AnyLink[] 68 | } 69 | 70 | export type IdxMeta = { 71 | byId: AnyLink 72 | byKey: AnyLink 73 | map: string 74 | name: string 75 | head: ClockHead 76 | } 77 | 78 | type IdxMetaMap = { 79 | indexes: Map 80 | } 81 | 82 | export type IdxCarHeader = CarHeader & IdxMetaMap 83 | 84 | export type DbCarHeader = CarHeader & { 85 | head: ClockHead 86 | } 87 | 88 | export type AnyCarHeader = DbCarHeader | IdxCarHeader 89 | 90 | export type QueryOpts = { 91 | descending?: boolean 92 | limit?: number 93 | includeDocs?: boolean 94 | range?: [IndexKey, IndexKey] 95 | key?: string // these two can be richer than keys... 96 | prefix?: string | [string] 97 | } 98 | 99 | export type AnyLink = Link 100 | export type AnyBlock = { cid: AnyLink; bytes: Uint8Array } 101 | export type AnyDecodedBlock = { cid: AnyLink; bytes: Uint8Array, value: any } 102 | 103 | export type BlockFetcher = { get: (link: AnyLink) => Promise } 104 | 105 | type CallbackFn = (k: string, v?: DocFragment) => void 106 | 107 | export type MapFn = (doc: Doc, map: CallbackFn) => DocFragment | void 108 | 109 | export type DbMeta = { car: AnyLink, key: string | null } 110 | 111 | export interface CarMakeable { 112 | entries(): Iterable 113 | get(cid: AnyLink): Promise 114 | } 115 | -------------------------------------------------------------------------------- /src/database.ts: -------------------------------------------------------------------------------- 1 | import { WriteQueue, writeQueue } from './write-queue' 2 | import { CRDT } from './crdt' 3 | import type { BulkResult, DocUpdate, ClockHead, Doc, FireproofOptions } from './types' 4 | 5 | export class Database { 6 | static databases: Map = new Map() 7 | 8 | name: string 9 | opts: FireproofOptions = {} 10 | 11 | _listeners: Set = new Set() 12 | _crdt: CRDT 13 | // _indexes: Map = new Map() 14 | _writeQueue: WriteQueue 15 | 16 | constructor(name: string, opts?: FireproofOptions) { 17 | this.name = name 18 | this.opts = opts || this.opts 19 | this._crdt = new CRDT(name, this.opts) 20 | this._writeQueue = writeQueue(async (updates: DocUpdate[]) => { 21 | const r = await this._crdt.bulk(updates) 22 | await this._notify(updates) 23 | return r 24 | }) 25 | } 26 | 27 | async get(id: string): Promise { 28 | const got = await this._crdt.get(id).catch(e => { 29 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access 30 | e.message = `Not found: ${id} - ` + e.message 31 | throw e 32 | }) 33 | if (!got) throw new Error(`Not found: ${id}`) 34 | const { doc } = got 35 | return { _id: id, ...doc } 36 | } 37 | 38 | async put(doc: Doc): Promise { 39 | const { _id, ...value } = doc 40 | const docId = _id || 'f' + Math.random().toString(36).slice(2) // todo uuid v7 41 | const result: BulkResult = await this._writeQueue.push({ key: docId, value } as DocUpdate) 42 | return { id: docId, clock: result?.head } as DbResponse 43 | } 44 | 45 | async del(id: string): Promise { 46 | const result = await this._writeQueue.push({ key: id, del: true }) 47 | return { id, clock: result?.head } as DbResponse 48 | } 49 | 50 | async changes(since: ClockHead = []): Promise { 51 | const { result, head } = await this._crdt.changes(since) 52 | const rows = result.map(({ key, value }) => ({ 53 | key, value: { _id: key, ...value } as Doc 54 | })) 55 | return { rows, clock: head } 56 | } 57 | 58 | subscribe(listener: ListenerFn): () => void { 59 | this._listeners.add(listener) 60 | return () => { 61 | this._listeners.delete(listener) 62 | } 63 | } 64 | 65 | async _notify(updates: DocUpdate[]) { 66 | if (this._listeners.size) { 67 | const docs = updates.map(({ key, value }) => ({ _id: key, ...value })) 68 | for (const listener of this._listeners) { 69 | await listener(docs) 70 | } 71 | } 72 | } 73 | } 74 | 75 | type ChangesResponse = { 76 | clock: ClockHead 77 | rows: { key: string; value: Doc }[] 78 | } 79 | 80 | type DbResponse = { 81 | id: string 82 | clock: ClockHead 83 | } 84 | 85 | type ListenerFn = (docs: Doc[]) => Promise | void 86 | 87 | export function database(name: string, opts?: FireproofOptions): Database { 88 | if (!Database.databases.has(name)) { 89 | Database.databases.set(name, new Database(name, opts)) 90 | } 91 | return Database.databases.get(name)! 92 | } 93 | -------------------------------------------------------------------------------- /src/store-browser.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/first */ 2 | console.log('import store-browser') 3 | 4 | import { openDB, IDBPDatabase } from 'idb' 5 | import { AnyBlock, AnyLink, DbMeta } from './types' 6 | import { CarStore as CarStoreBase, HeaderStore as HeaderStoreBase } from './store' 7 | 8 | export class CarStore extends CarStoreBase { 9 | tag: string = 'car-browser-idb' 10 | keyId: string = 'public' 11 | idb: IDBPDatabase | null = null 12 | 13 | async _withDB(dbWorkFun: (arg0: any) => any) { 14 | if (!this.idb) { 15 | const dbName = `fp.${this.STORAGE_VERSION}.${this.keyId}.${this.name}` 16 | this.idb = await openDB(dbName, 1, { 17 | upgrade(db): void { 18 | db.createObjectStore('cars') 19 | } 20 | }) 21 | } 22 | // eslint-disable-next-line @typescript-eslint/no-unsafe-return 23 | return await dbWorkFun(this.idb) 24 | } 25 | 26 | async load(cid: AnyLink): Promise { 27 | // eslint-disable-next-line @typescript-eslint/no-unsafe-return 28 | return await this._withDB(async (db: IDBPDatabase) => { 29 | const tx = db.transaction(['cars'], 'readonly') 30 | const bytes = (await tx.objectStore('cars').get(cid.toString())) as Uint8Array 31 | if (!bytes) throw new Error(`missing idb block ${cid.toString()}`) 32 | return { cid, bytes } 33 | }) 34 | } 35 | 36 | async save(car: AnyBlock): Promise { 37 | // eslint-disable-next-line @typescript-eslint/no-unsafe-return 38 | return await this._withDB(async (db: IDBPDatabase) => { 39 | const tx = db.transaction(['cars'], 'readwrite') 40 | await tx.objectStore('cars').put(car.bytes, car.cid.toString()) 41 | return await tx.done 42 | }) 43 | } 44 | 45 | async remove(cid: AnyLink): Promise { 46 | // eslint-disable-next-line @typescript-eslint/no-unsafe-return 47 | return await this._withDB(async (db: IDBPDatabase) => { 48 | const tx = db.transaction(['cars'], 'readwrite') 49 | await tx.objectStore('cars').delete(cid.toString()) 50 | return await tx.done 51 | }) 52 | } 53 | } 54 | 55 | export class HeaderStore extends HeaderStoreBase { 56 | tag: string = 'header-browser-ls' 57 | keyId: string = 'public' 58 | decoder: TextDecoder 59 | encoder: TextEncoder 60 | 61 | constructor(name: string) { 62 | super(name) 63 | this.decoder = new TextDecoder() 64 | this.encoder = new TextEncoder() 65 | } 66 | 67 | headerKey(branch: string) { 68 | return `fp.${this.STORAGE_VERSION}.${this.keyId}.${this.name}.${branch}` 69 | } 70 | 71 | // eslint-disable-next-line @typescript-eslint/require-await 72 | async load(branch: string = 'main'): Promise { 73 | try { 74 | const bytesString = localStorage.getItem(this.headerKey(branch)) 75 | if (!bytesString) return null 76 | return this.parseHeader(bytesString) 77 | } catch (e) { 78 | return null 79 | } 80 | } 81 | 82 | // eslint-disable-next-line @typescript-eslint/require-await 83 | async save(meta: DbMeta, branch: string = 'main'): Promise { 84 | try { 85 | const headerKey = this.headerKey(branch) 86 | const bytes = this.makeHeader(meta) 87 | return localStorage.setItem(headerKey, bytes) 88 | } catch (e) {} 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/encrypt-helpers.ts: -------------------------------------------------------------------------------- 1 | import type { CarReader } from '@ipld/car' 2 | 3 | import { sha256 } from 'multiformats/hashes/sha2' 4 | import { encrypt, decrypt } from './crypto' 5 | import { Buffer } from 'buffer' 6 | // @ts-ignore 7 | import { bf } from 'prolly-trees/utils' 8 | // @ts-ignore 9 | import { nocache as cache } from 'prolly-trees/cache' 10 | import { encodeCarHeader, encodeCarFile } from './loader-helpers' // Import the existing function 11 | import type { AnyBlock, CarMakeable, AnyCarHeader, AnyLink, BlockFetcher } from './types' 12 | import type { Transaction } from './transaction' 13 | import { MemoryBlockstore } from '@alanshaw/pail/block' 14 | 15 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call 16 | const chunker = bf(30) 17 | 18 | export async function encryptedMakeCarFile(key: string, fp: AnyCarHeader, t: Transaction): Promise { 19 | const { cid, bytes } = await encodeCarHeader(fp) 20 | await t.put(cid, bytes) 21 | return encryptedEncodeCarFile(key, cid, t) 22 | } 23 | 24 | async function encryptedEncodeCarFile(key: string, rootCid: AnyLink, t: CarMakeable): Promise { 25 | const encryptionKeyBuffer = Buffer.from(key, 'hex') 26 | const encryptionKey = encryptionKeyBuffer.buffer.slice(encryptionKeyBuffer.byteOffset, encryptionKeyBuffer.byteOffset + encryptionKeyBuffer.byteLength) 27 | const encryptedBlocks = new MemoryBlockstore() 28 | const cidsToEncrypt = [] as AnyLink[] 29 | for (const { cid } of t.entries()) { 30 | cidsToEncrypt.push(cid) 31 | } 32 | let last: AnyBlock | null = null 33 | for await (const block of encrypt({ 34 | cids: cidsToEncrypt, 35 | get: t.get.bind(t), 36 | key: encryptionKey, 37 | hasher: sha256, 38 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment 39 | chunker, 40 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment 41 | cache, 42 | root: rootCid 43 | }) as AsyncGenerator) { 44 | await encryptedBlocks.put(block.cid, block.bytes) 45 | last = block 46 | } 47 | if (!last) throw new Error('no blocks encrypted') 48 | const encryptedCar = await encodeCarFile(last.cid, encryptedBlocks) 49 | return encryptedCar 50 | } 51 | 52 | export async function decodeEncryptedCar(key: string, reader: CarReader) { 53 | const roots = await reader.getRoots() 54 | const root = roots[0] 55 | return await decodeCarBlocks(root, reader.get.bind(reader), key) 56 | } 57 | async function decodeCarBlocks( 58 | root: AnyLink, 59 | get: (cid: any) => Promise, 60 | keyMaterial: string 61 | ): Promise<{ blocks: BlockFetcher; root: AnyLink }> { 62 | const decryptionKeyBuffer = Buffer.from(keyMaterial, 'hex') 63 | const decryptionKey = decryptionKeyBuffer.buffer.slice(decryptionKeyBuffer.byteOffset, decryptionKeyBuffer.byteOffset + decryptionKeyBuffer.byteLength) 64 | 65 | const decryptedBlocks = new MemoryBlockstore() 66 | let last: AnyBlock | null = null 67 | for await (const block of decrypt({ 68 | root, 69 | get, 70 | key: decryptionKey, 71 | hasher: sha256, 72 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment 73 | chunker, 74 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment 75 | cache 76 | })) { 77 | await decryptedBlocks.put(block.cid, block.bytes) 78 | last = block 79 | } 80 | if (!last) throw new Error('no blocks decrypted') 81 | return { blocks: decryptedBlocks, root: last.cid } 82 | } 83 | -------------------------------------------------------------------------------- /scripts/server.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-assignment */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-call */ 3 | /* eslint-disable @typescript-eslint/no-unsafe-argument */ 4 | /* eslint-disable @typescript-eslint/no-unsafe-member-access */ 5 | /* eslint-disable @typescript-eslint/no-misused-promises */ 6 | import * as fs from 'node:fs' 7 | import * as http from 'node:http' 8 | import * as path from 'node:path' 9 | 10 | import { HeaderStore } from '../dist/test/store-fs.esm.js' 11 | 12 | const PORT = 8000 13 | 14 | /** 15 | * This server is for illustration purposes. It trusts the client. 16 | * Before using in production it requires customization: 17 | * - Validate car files 18 | * - Validate paths & mime types 19 | * - Authenticate requests and enforce that users can only update their own header file (userid in header filename) 20 | * - Deploy in a secure environment 21 | * To connect with a managed service, see https://fireproof.storage 22 | */ 23 | 24 | const MIME_TYPES = { 25 | default: 'application/octet-stream', 26 | json: 'application/json', 27 | car: 'application/car' 28 | } 29 | 30 | const DATA_PATH = HeaderStore.dataDir 31 | 32 | const toBool = [() => true, () => false] 33 | 34 | const prepareFile = async url => { 35 | const paths = [DATA_PATH, url] 36 | if (url.endsWith('/')) paths.push('index.html') 37 | const filePath = path.join(...paths) 38 | const pathTraversal = !filePath.startsWith(DATA_PATH) 39 | const exists = await fs.promises.access(filePath).then(...toBool) 40 | const found = !pathTraversal && exists 41 | const ext = found ? path.extname(filePath).substring(1).toLowerCase() : null 42 | const stream = found ? fs.createReadStream(filePath) : null 43 | return { found, ext, stream } 44 | } 45 | 46 | export function startServer(quiet = true) { 47 | const log = quiet ? () => {} : console.log 48 | 49 | const server = http.createServer(async (req, res) => { 50 | res.setHeader('Access-Control-Allow-Origin', '*') 51 | res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE') 52 | res.setHeader('Access-Control-Allow-Headers', 'Content-Type') 53 | 54 | if (req.method === 'PUT') { 55 | const filePath = path.join(DATA_PATH, req.url) 56 | await fs.promises.mkdir(path.dirname(filePath), { recursive: true }) 57 | const writeStream = fs.createWriteStream(filePath) 58 | req.pipe(writeStream) 59 | 60 | req.on('end', () => { 61 | res.writeHead(200, { 'Content-Type': 'text/plain' }) 62 | res.end('File written successfully') 63 | log(`PUT ${req.url} 200`) 64 | }) 65 | 66 | req.on('error', err => { 67 | res.writeHead(500, { 'Content-Type': 'text/plain' }) 68 | res.end('Internal Server Error') 69 | log(`PUT ${req.url} 500 - ${err.message}`) 70 | }) 71 | } else if (req.method === 'OPTIONS') { 72 | // Pre-flight request. Reply successfully: 73 | res.writeHead(200) 74 | res.end() 75 | } else { 76 | const file = await prepareFile(req.url) 77 | if (file.found) { 78 | const mimeType = MIME_TYPES[file.ext] || MIME_TYPES.default 79 | res.writeHead(200, { 'Content-Type': mimeType }) 80 | file.stream.pipe(res) 81 | log(`GET ${req.url} 200`) 82 | } else { 83 | res.writeHead(404, { 'Content-Type': 'text/plain' }) 84 | res.end('Not found') 85 | log(`GET ${req.url} 404`) 86 | } 87 | } 88 | }) 89 | server.listen(PORT) 90 | void fs.promises.mkdir(DATA_PATH, { recursive: true }).then(() => { 91 | log(`Server running at http://127.0.0.1:${PORT}/`) 92 | }) 93 | return server 94 | } 95 | 96 | // if the script is run directly (not imported as a module), start the server: 97 | if (import.meta.url === 'file://' + process.argv[1]) startServer(false) 98 | -------------------------------------------------------------------------------- /scripts/settings.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-return */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-assignment */ 3 | import esbuildPluginTsc from 'esbuild-plugin-tsc' 4 | // import alias from 'esbuild-plugin-alias' 5 | import fs from 'fs' 6 | import path from 'path' 7 | import { polyfillNode } from 'esbuild-plugin-polyfill-node' 8 | 9 | // Obtain all .ts files in the src directory 10 | const entryPoints = fs 11 | .readdirSync('src') 12 | .filter(file => path.extname(file) === '.ts') 13 | .map(file => path.join('src', file)) 14 | 15 | export function createBuildSettings(options) { 16 | const commonSettings = { 17 | entryPoints, 18 | bundle: true, 19 | sourcemap: true, 20 | plugins: [ 21 | esbuildPluginTsc({ 22 | force: true 23 | }) 24 | ], 25 | ...options 26 | } 27 | 28 | // Generate build configs for each entry point 29 | const configs = entryPoints.map(entryPoint => { 30 | const filename = path.basename(entryPoint, '.ts') 31 | 32 | const builds = [] 33 | 34 | const esmConfig = { 35 | ...commonSettings, 36 | outfile: `dist/test/${filename}.esm.js`, 37 | format: 'esm', 38 | platform: 'node', 39 | entryPoints: [entryPoint], 40 | banner: { 41 | js: ` 42 | console.log('esm/node build'); 43 | import { createRequire } from 'module'; 44 | const require = createRequire(import.meta.url); 45 | ` 46 | } 47 | } 48 | 49 | builds.push(esmConfig) 50 | 51 | if (/fireproof\.|database\.|index\./.test(entryPoint)) { 52 | const esmPublishConfig = { 53 | ...esmConfig, 54 | outfile: `dist/node/${filename}.esm.js`, 55 | entryPoints: [entryPoint] 56 | } 57 | builds.push(esmPublishConfig) 58 | 59 | const cjsConfig = { 60 | ...commonSettings, 61 | outfile: `dist/node/${filename}.cjs`, 62 | format: 'cjs', 63 | platform: 'node', 64 | entryPoints: [entryPoint], 65 | banner: { 66 | js: ` 67 | console.log('cjs/node build'); 68 | ` 69 | } 70 | } 71 | builds.push(cjsConfig) 72 | 73 | const browserIIFEConfig = { 74 | ...commonSettings, 75 | outfile: `dist/browser/${filename}.iife.js`, 76 | format: 'iife', 77 | globalName: 'Fireproof', 78 | platform: 'browser', 79 | target: 'es2015', 80 | entryPoints: [entryPoint], 81 | banner: { 82 | js: ` 83 | console.log('browser/es2015 build'); 84 | ` 85 | }, 86 | plugins: [ 87 | polyfillNode({ 88 | polyfills: { crypto: true, fs: true, process: 'empty' } 89 | }), 90 | // alias({ 91 | // crypto: 'crypto-browserify' 92 | // }), 93 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access 94 | ...commonSettings.plugins 95 | ] 96 | } 97 | 98 | builds.push(browserIIFEConfig) 99 | 100 | const browserESMConfig = { 101 | ...browserIIFEConfig, 102 | outfile: `dist/browser/${filename}.esm.js`, 103 | format: 'esm', 104 | banner: { // should this include createRequire? 105 | js: ` 106 | console.log('esm/es2015 build'); 107 | ` 108 | } 109 | } 110 | 111 | builds.push(browserESMConfig) 112 | 113 | const browserCJSConfig = { 114 | ...browserIIFEConfig, 115 | outfile: `dist/browser/${filename}.cjs`, 116 | format: 'cjs', 117 | banner: { 118 | js: ` 119 | console.log('cjs/es2015 build'); 120 | ` 121 | } 122 | } 123 | builds.push(browserCJSConfig) 124 | } 125 | 126 | return builds 127 | }) 128 | 129 | return configs.flat() 130 | } 131 | -------------------------------------------------------------------------------- /src/encrypted-block.ts: -------------------------------------------------------------------------------- 1 | // from https://github.com/mikeal/encrypted-block 2 | import { Crypto } from '@peculiar/webcrypto' 3 | import { CID } from 'multiformats' 4 | import { Buffer } from 'buffer' 5 | import type { AnyLink } from './types' 6 | 7 | // const crypto = new Crypto() 8 | 9 | export function getCrypto() { 10 | try { 11 | return new Crypto() 12 | } catch (e) { 13 | return null 14 | } 15 | } 16 | 17 | const crypto = getCrypto() 18 | 19 | export function randomBytes(size: number) { 20 | const bytes = Buffer.allocUnsafe(size) 21 | if (size > 0) { 22 | crypto!.getRandomValues(bytes) 23 | } 24 | return bytes 25 | } 26 | 27 | const enc32 = (value: number) => { 28 | value = +value 29 | const buff = new Uint8Array(4) 30 | buff[3] = (value >>> 24) 31 | buff[2] = (value >>> 16) 32 | buff[1] = (value >>> 8) 33 | buff[0] = (value & 0xff) 34 | return buff 35 | } 36 | 37 | const readUInt32LE = (buffer: Uint8Array) => { 38 | const offset = buffer.byteLength - 4 39 | return ((buffer[offset]) | 40 | (buffer[offset + 1] << 8) | 41 | (buffer[offset + 2] << 16)) + 42 | (buffer[offset + 3] * 0x1000000) 43 | } 44 | 45 | const concat = (buffers: Array) => { 46 | const uint8Arrays = buffers.map(b => b instanceof ArrayBuffer ? new Uint8Array(b) : b) 47 | const totalLength = uint8Arrays.reduce((sum, arr) => sum + arr.length, 0) 48 | const result = new Uint8Array(totalLength) 49 | 50 | let offset = 0 51 | for (const arr of uint8Arrays) { 52 | result.set(arr, offset) 53 | offset += arr.length 54 | } 55 | 56 | return result 57 | } 58 | 59 | const encode = ({ iv, bytes }: {iv: Uint8Array, bytes: Uint8Array}) => concat([iv, bytes]) 60 | const decode = (bytes: Uint8Array) => { 61 | const iv = bytes.subarray(0, 12) 62 | bytes = bytes.slice(12) 63 | return { iv, bytes } 64 | } 65 | 66 | const code = 0x300000 + 1337 67 | 68 | async function subtleKey(key: ArrayBuffer) { 69 | return await crypto!.subtle.importKey( 70 | 'raw', // raw or jwk 71 | key, // raw data 72 | 'AES-GCM', 73 | false, // extractable 74 | ['encrypt', 'decrypt'] 75 | ) 76 | } 77 | 78 | const decrypt = async ({ key, value }: 79 | {key: ArrayBuffer, value: { bytes: Uint8Array, iv: Uint8Array} 80 | }): Promise<{ cid: AnyLink, bytes: Uint8Array }> => { 81 | let { bytes, iv } = value 82 | const cryKey = await subtleKey(key) 83 | const deBytes = await crypto!.subtle.decrypt( 84 | { 85 | name: 'AES-GCM', 86 | iv, 87 | tagLength: 128 88 | }, 89 | cryKey, 90 | bytes 91 | ) 92 | bytes = new Uint8Array(deBytes) 93 | const len = readUInt32LE(bytes.subarray(0, 4)) 94 | const cid = CID.decode(bytes.subarray(4, 4 + len)) 95 | bytes = bytes.subarray(4 + len) 96 | return { cid, bytes } 97 | } 98 | const encrypt = async ({ key, cid, bytes }: { key: ArrayBuffer, cid: AnyLink, bytes: Uint8Array }) => { 99 | const len = enc32(cid.bytes.byteLength) 100 | const iv = randomBytes(12) 101 | const msg = concat([len, cid.bytes, bytes]) 102 | try { 103 | const cryKey = await subtleKey(key) 104 | const deBytes = await crypto!.subtle.encrypt( 105 | { 106 | name: 'AES-GCM', 107 | iv, 108 | tagLength: 128 109 | }, 110 | cryKey, 111 | msg 112 | ) 113 | bytes = new Uint8Array(deBytes) 114 | } catch (e) { 115 | console.log('e', e) 116 | throw e 117 | } 118 | return { value: { bytes, iv } } 119 | } 120 | 121 | const cryptoFn = (key: Uint8Array) => { 122 | // @ts-ignore 123 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument 124 | return { encrypt: opts => encrypt({ key, ...opts }), decrypt: opts => decrypt({ key, ...opts }) } 125 | } 126 | 127 | const name = 'jchris@encrypted-block:aes-gcm' 128 | 129 | export { encode, decode, code, name, encrypt, decrypt, cryptoFn as crypto } 130 | -------------------------------------------------------------------------------- /src/crypto.ts: -------------------------------------------------------------------------------- 1 | import * as codec from './encrypted-block.js' 2 | import * as dagcbor from '@ipld/dag-cbor' 3 | // @ts-ignore 4 | import { create, load } from 'prolly-trees/cid-set' 5 | import { CID } from 'multiformats' 6 | import { encode, decode, create as mfCreate } from 'multiformats/block' 7 | import type { AnyBlock, AnyDecodedBlock, AnyLink } from './types.js' 8 | import type { MultihashHasher, ToString } from 'multiformats' 9 | 10 | const encrypt = async function * ({ 11 | get, cids, hasher, 12 | key, cache, chunker, root 13 | }: 14 | { 15 | get: (cid: AnyLink) => Promise, 16 | key: ArrayBuffer, cids: AnyLink[], hasher: MultihashHasher 17 | chunker: (bytes: Uint8Array) => AsyncGenerator, 18 | cache: (cid: AnyLink) => Promise, 19 | root: AnyLink 20 | }): AsyncGenerator { 21 | const set = new Set>() 22 | let eroot 23 | for (const cid of cids) { 24 | const unencrypted = await get(cid) 25 | if (!unencrypted) throw new Error('missing cid: ' + cid.toString()) 26 | const encrypted = await codec.encrypt({ ...unencrypted, key }) 27 | const block = await encode({ ...encrypted, codec, hasher }) 28 | yield block 29 | set.add(block.cid.toString()) 30 | if (unencrypted.cid.equals(root)) eroot = block.cid 31 | } 32 | if (!eroot) throw new Error('cids does not include root') 33 | const list = [...set].map(s => CID.parse(s)) 34 | let last 35 | // eslint-disable-next-line @typescript-eslint/no-unsafe-call 36 | for await (const node of create({ list, get, cache, chunker, hasher, codec: dagcbor })) { 37 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access 38 | const block = await node.block as AnyBlock 39 | yield block 40 | last = block 41 | } 42 | if (!last) throw new Error('missing last block') 43 | const head = [eroot, last.cid] 44 | const block = await encode({ value: head, codec: dagcbor, hasher }) 45 | yield block 46 | } 47 | 48 | const decrypt = async function * ({ root, get, key, cache, chunker, hasher }: { 49 | root: AnyLink, 50 | get: (cid: AnyLink) => Promise, 51 | key: ArrayBuffer, 52 | cache: (cid: AnyLink) => Promise, 53 | chunker: (bytes: Uint8Array) => AsyncGenerator, 54 | hasher: MultihashHasher 55 | }): AsyncGenerator { 56 | const getWithDecode = async (cid: AnyLink) => get(cid).then(async (block) => { 57 | if (!block) return 58 | const decoded = await decode({ ...block, codec: dagcbor, hasher }) 59 | return decoded 60 | }) 61 | const getWithDecrypt = async (cid: AnyLink) => get(cid).then(async (block) => { 62 | if (!block) return 63 | const decoded = await decode({ ...block, codec, hasher }) 64 | return decoded 65 | }) 66 | const decodedRoot = await getWithDecode(root) 67 | if (!decodedRoot) throw new Error('missing root') 68 | if (!decodedRoot.bytes) throw new Error('missing bytes') 69 | const { value: [eroot, tree] } = decodedRoot as { value: [AnyLink, AnyLink] } 70 | const rootBlock = await get(eroot) as AnyDecodedBlock 71 | if (!rootBlock) throw new Error('missing root block') 72 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call 73 | const cidset = await load({ cid: tree, get: getWithDecode, cache, chunker, codec, hasher }) 74 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 75 | const { result: nodes } = await cidset.getAllEntries() as { result: { cid: CID }[] } 76 | const unwrap = async (eblock: AnyDecodedBlock | undefined) => { 77 | if (!eblock) throw new Error('missing block') 78 | if (!eblock.value) { eblock = await decode({ ...eblock, codec, hasher }) as AnyDecodedBlock } 79 | const { bytes, cid } = await codec.decrypt({ ...eblock, key }).catch(e => { 80 | throw e 81 | }) 82 | const block = await mfCreate({ cid, bytes, hasher, codec }) 83 | return block 84 | } 85 | const promises = [] 86 | for (const { cid } of nodes) { 87 | if (!rootBlock.cid.equals(cid)) promises.push(getWithDecrypt(cid).then(unwrap)) 88 | } 89 | yield * promises 90 | yield unwrap(rootBlock) 91 | } 92 | export { 93 | encrypt, 94 | decrypt 95 | } 96 | -------------------------------------------------------------------------------- /test/store-fs.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-return */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-assignment */ 3 | /* eslint-disable @typescript-eslint/no-unsafe-call */ 4 | /* eslint-disable @typescript-eslint/no-unsafe-member-access */ 5 | /* eslint-disable mocha/max-top-level-suites */ 6 | /* eslint-disable @typescript-eslint/no-unsafe-argument */ 7 | import { join } from 'path' 8 | import { promises } from 'fs' 9 | 10 | import { CID } from 'multiformats' 11 | 12 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 13 | import { assert, matches, equals } from './helpers.js' 14 | 15 | import { CarStore, testConfig, HeaderStore } from '../dist/test/store-fs.esm.js' 16 | 17 | const { readFile } = promises 18 | 19 | const decoder = new TextDecoder('utf-8') 20 | 21 | describe('CarStore', function () { 22 | /** @type {CarStore} */ 23 | let store 24 | beforeEach(function () { 25 | store = new CarStore('test') 26 | }) 27 | it('should have a name', function () { 28 | equals(store.name, 'test') 29 | }) 30 | it('should save a car', async function () { 31 | const car = { 32 | cid: 'cid', 33 | bytes: new Uint8Array([55, 56, 57]) 34 | } 35 | await store.save(car) 36 | const path = join(testConfig.dataDir, store.name, car.cid + '.car') 37 | const data = await readFile(path) 38 | equals(data.toString(), decoder.decode(car.bytes)) 39 | }) 40 | }) 41 | 42 | describe('CarStore with a saved car', function () { 43 | /** @type {CarStore} */ 44 | let store, car 45 | beforeEach(async function () { 46 | store = new CarStore('test2') 47 | car = { 48 | cid: 'cid', 49 | bytes: new Uint8Array([55, 56, 57, 80]) 50 | } 51 | await store.save(car) 52 | }) 53 | it('should have a car', async function () { 54 | const path = join(testConfig.dataDir, store.name, car.cid + '.car') 55 | const data = await readFile(path) 56 | equals(data.toString(), decoder.decode(car.bytes)) 57 | }) 58 | it('should load a car', async function () { 59 | const loaded = await store.load(car.cid) 60 | equals(loaded.cid, car.cid) 61 | equals(loaded.bytes.constructor.name, 'Uint8Array') 62 | equals(loaded.bytes.toString(), car.bytes.toString()) 63 | }) 64 | it('should remove a car', async function () { 65 | await store.remove(car.cid) 66 | const error = await store.load(car.cid).catch(e => e) 67 | matches(error.message, 'ENOENT') 68 | }) 69 | }) 70 | 71 | describe('HeaderStore', function () { 72 | /** @type {HeaderStore} */ 73 | let store 74 | beforeEach(function () { 75 | store = new HeaderStore('test') 76 | }) 77 | it('should have a name', function () { 78 | equals(store.name, 'test') 79 | }) 80 | it('should save a header', async function () { 81 | const cid = CID.parse('bafybeia4luuns6dgymy5kau5rm7r4qzrrzg6cglpzpogussprpy42cmcn4') 82 | const h = { 83 | car: cid, 84 | key: null 85 | } 86 | await store.save(h) 87 | const path = join(testConfig.dataDir, store.name, 'main.json') 88 | const file = await readFile(path) 89 | const header = JSON.parse(file.toString()) 90 | assert(header) 91 | assert(header.car) 92 | equals(header.car['/'], cid.toString()) 93 | }) 94 | }) 95 | 96 | describe('HeaderStore with a saved header', function () { 97 | /** @type {HeaderStore} */ 98 | let store, cid 99 | beforeEach(async function () { 100 | store = new HeaderStore('test-saved-header') 101 | cid = CID.parse('bafybeia4luuns6dgymy5kau5rm7r4qzrrzg6cglpzpogussprpy42cmcn4') 102 | await store.save({ car: cid, key: null }) 103 | }) 104 | it('should have a header', async function () { 105 | const path = join(testConfig.dataDir, store.name, 'main.json') 106 | const data = await readFile(path) 107 | matches(data, /car/) 108 | const header = JSON.parse(data.toString()) 109 | assert(header) 110 | assert(header.car) 111 | equals(header.car['/'], cid.toString()) 112 | }) 113 | it('should load a header', async function () { 114 | const loaded = await store.load() 115 | assert(loaded) 116 | // console.log(loaded) 117 | assert(loaded.car) 118 | equals(loaded.car.toString(), cid.toString()) 119 | }) 120 | }) 121 | -------------------------------------------------------------------------------- /src/transaction.ts: -------------------------------------------------------------------------------- 1 | import { MemoryBlockstore } from '@alanshaw/pail/block' 2 | import { 3 | BlockFetcher, AnyBlock, AnyLink, BulkResult, ClockHead, 4 | DbCarHeader, IdxCarHeader, IdxMeta, CarCommit, CarMakeable, FireproofOptions 5 | } from './types' 6 | import { DbLoader, IdxLoader } from './loader' 7 | import { CID } from 'multiformats' 8 | 9 | export class Transaction extends MemoryBlockstore implements CarMakeable { 10 | constructor(private parent: BlockFetcher) { 11 | super() 12 | this.parent = parent 13 | } 14 | 15 | async get(cid: AnyLink): Promise { 16 | return this.parent.get(cid) 17 | } 18 | 19 | async superGet(cid: AnyLink): Promise { 20 | return super.get(cid) 21 | } 22 | } 23 | 24 | abstract class FireproofBlockstore implements BlockFetcher { 25 | ready: Promise 26 | name: string | null = null 27 | 28 | loader: DbLoader | IdxLoader | null = null 29 | opts: FireproofOptions = {} 30 | 31 | private transactions: Set = new Set() 32 | 33 | constructor(name: string | null, LoaderClass: typeof DbLoader | typeof IdxLoader, opts?: FireproofOptions) { 34 | this.opts = opts || this.opts 35 | if (name) { 36 | this.name = name 37 | this.loader = new LoaderClass(name, this.opts) 38 | this.ready = this.loader.ready 39 | } else { 40 | this.ready = Promise.resolve(LoaderClass.defaultHeader as DbCarHeader | IdxCarHeader) 41 | } 42 | } 43 | 44 | abstract transaction(fn: (t: Transaction) => Promise, indexes?: Map): Promise 45 | 46 | // eslint-disable-next-line @typescript-eslint/require-await 47 | async put() { 48 | throw new Error('use a transaction to put') 49 | } 50 | 51 | async get(cid: AnyLink): Promise { 52 | for (const f of this.transactions) { 53 | const v = await f.superGet(cid) 54 | if (v) return v 55 | } 56 | if (!this.loader) return 57 | return await this.loader.getBlock(cid as CID) 58 | } 59 | 60 | async commitCompaction(t: Transaction, head: ClockHead) { 61 | this.transactions.clear() 62 | this.transactions.add(t) 63 | return await this.loader?.commit(t, { head }, true) 64 | } 65 | 66 | async * entries(): AsyncIterableIterator { 67 | const seen: Set = new Set() 68 | for (const t of this.transactions) { 69 | for await (const blk of t.entries()) { 70 | if (seen.has(blk.cid.toString())) continue 71 | seen.add(blk.cid.toString()) 72 | yield blk 73 | } 74 | } 75 | } 76 | 77 | protected async executeTransaction( 78 | fn: (t: Transaction) => Promise, 79 | commitHandler: (t: Transaction, done: T) => Promise<{ car?: AnyLink, done: R }> 80 | ): Promise { 81 | const t = new Transaction(this) 82 | this.transactions.add(t) 83 | const done: T = await fn(t) 84 | const { car, done: result } = await commitHandler(t, done) 85 | return car ? { ...result, car } : result 86 | } 87 | } 88 | 89 | export class IndexBlockstore extends FireproofBlockstore { 90 | declare ready: Promise 91 | 92 | constructor(name?: string, opts?: FireproofOptions) { 93 | super(name || null, IdxLoader, opts) 94 | } 95 | 96 | async transaction(fn: (t: Transaction) => Promise, indexes: Map): Promise { 97 | return this.executeTransaction(fn, async (t, done) => { 98 | indexes.set(done.name, done) 99 | const car = await this.loader?.commit(t, { indexes }) 100 | return { car, done } 101 | }) 102 | } 103 | } 104 | 105 | export class TransactionBlockstore extends FireproofBlockstore { 106 | declare ready: Promise 107 | 108 | constructor(name?: string, opts?: FireproofOptions) { 109 | // todo this will be a map of headers by branch name 110 | super(name || null, DbLoader, opts) 111 | } 112 | 113 | async transaction(fn: (t: Transaction) => Promise): Promise { 114 | return this.executeTransaction(fn, async (t, done) => { 115 | const car = await this.loader?.commit(t, done) 116 | return { car, done } 117 | }) 118 | } 119 | } 120 | 121 | type IdxMetaCar = IdxMeta & CarCommit 122 | type BulkResultCar = BulkResult & CarCommit 123 | -------------------------------------------------------------------------------- /test/transaction.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-return */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-call */ 3 | /* eslint-disable @typescript-eslint/no-unsafe-assignment */ 4 | /* eslint-disable @typescript-eslint/no-unsafe-member-access */ 5 | /* eslint-disable @typescript-eslint/require-await */ 6 | /* eslint-disable mocha/max-top-level-suites */ 7 | import { CID } from 'multiformats' 8 | 9 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 10 | import { assert, equals, notEquals, matches, equalsJSON } from './helpers.js' 11 | import { TransactionBlockstore as Blockstore, Transaction } from '../dist/test/transaction.esm.js' 12 | 13 | describe('Fresh TransactionBlockstore', function () { 14 | /** @type {Blockstore} */ 15 | let blocks 16 | beforeEach(function () { 17 | blocks = new Blockstore() 18 | }) 19 | it('should not have a name', function () { 20 | assert(!blocks.name) 21 | }) 22 | it('should not have a loader', function () { 23 | assert(!blocks._loader) 24 | }) 25 | it('should not put', async function () { 26 | const e = await blocks.put('key', 'value').catch(e => e) 27 | matches(e.message, /transaction/) 28 | }) 29 | it('should yield a transaction', async function () { 30 | const txR = await blocks.transaction((tblocks) => { 31 | assert(tblocks) 32 | assert(tblocks instanceof Transaction) 33 | return { head: [] } 34 | }) 35 | assert(txR) 36 | equalsJSON(txR, { head: [] }) 37 | }) 38 | }) 39 | 40 | describe('TransactionBlockstore with name', function () { 41 | /** @type {Blockstore} */ 42 | let blocks 43 | beforeEach(function () { 44 | blocks = new Blockstore('test') 45 | }) 46 | it('should have a name', function () { 47 | equals(blocks.name, 'test') 48 | }) 49 | it('should have a loader', function () { 50 | assert(blocks.loader) 51 | }) 52 | it('should get from loader', async function () { 53 | blocks.loader.getBlock = async (cid) => { 54 | return { cid, bytes: 'bytes' } 55 | } 56 | const value = await blocks.get('key') 57 | equalsJSON(value, { cid: 'key', bytes: 'bytes' }) 58 | }) 59 | }) 60 | 61 | describe('A transaction', function () { 62 | /** @type {Transaction} */ 63 | let tblocks, blocks 64 | beforeEach(async function () { 65 | blocks = new Blockstore() 66 | tblocks = new Transaction(blocks) 67 | blocks.transactions.add(tblocks) 68 | }) 69 | it('should put and get', async function () { 70 | const cid = CID.parse('bafybeia4luuns6dgymy5kau5rm7r4qzrrzg6cglpzpogussprpy42cmcn4') 71 | 72 | await tblocks.put(cid, 'bytes') 73 | assert(blocks.transactions.has(tblocks)) 74 | const got = await tblocks.get(cid) 75 | assert(got) 76 | equals(got.cid, cid) 77 | equals(got.bytes, 'bytes') 78 | }) 79 | }) 80 | 81 | describe('TransactionBlockstore with a completed transaction', function () { 82 | let blocks, cid, cid2 83 | 84 | beforeEach(async function () { 85 | cid = CID.parse('bafybeia4luuns6dgymy5kau5rm7r4qzrrzg6cglpzpogussprpy42cmcn4') 86 | cid2 = CID.parse('bafybeibgouhn5ktecpjuovt52zamzvm4dlve5ak7x6d5smms3itkhplnhm') 87 | 88 | blocks = new Blockstore() 89 | await blocks.transaction(async (tblocks) => { 90 | await tblocks.put(cid, 'value') 91 | return await tblocks.put(cid2, 'value2') 92 | }) 93 | await blocks.transaction(async (tblocks) => { 94 | await tblocks.put(cid, 'value') 95 | return await tblocks.put(cid2, 'value2') 96 | }) 97 | }) 98 | it('should have transactions', async function () { 99 | const ts = blocks.transactions 100 | equals(ts.size, 2) 101 | }) 102 | it('should get', async function () { 103 | const value = await blocks.get(cid) 104 | equals(value.cid, cid) 105 | equals(value.bytes, 'value') 106 | 107 | const value2 = await blocks.get(cid2) 108 | equals(value2.bytes, 'value2') 109 | }) 110 | it('should yield entries', async function () { 111 | const blz = [] 112 | for await (const blk of blocks.entries()) { 113 | blz.push(blk) 114 | } 115 | equals(blz.length, 2) 116 | }) 117 | it('should compact', async function () { 118 | const compactT = new Transaction(blocks) 119 | await compactT.put(cid2, 'valueX') 120 | await blocks.commitCompaction(compactT) 121 | equals(blocks.transactions.size, 1) 122 | assert(blocks.transactions.has(compactT)) 123 | }) 124 | }) 125 | -------------------------------------------------------------------------------- /src/crdt-helpers.ts: -------------------------------------------------------------------------------- 1 | import { encode, decode } from 'multiformats/block' 2 | import { sha256 as hasher } from 'multiformats/hashes/sha2' 3 | import * as codec from '@ipld/dag-cbor' 4 | import { put, get, entries, EventData } from '@alanshaw/pail/crdt' 5 | import { EventFetcher } from '@alanshaw/pail/clock' 6 | import { TransactionBlockstore, Transaction } from './transaction' 7 | import { DocUpdate, ClockHead, BlockFetcher, AnyLink, DocValue, BulkResult } from './types' 8 | 9 | export async function applyBulkUpdateToCrdt( 10 | tblocks: Transaction, 11 | head: ClockHead, 12 | updates: DocUpdate[], 13 | options?: object 14 | ): Promise { 15 | for (const update of updates) { 16 | const link = await makeLinkForDoc(tblocks, update) 17 | const result = await put(tblocks, head, update.key, link, options) 18 | for (const { cid, bytes } of [...result.additions, ...result.removals, result.event]) { 19 | tblocks.putSync(cid, bytes) 20 | } 21 | head = result.head 22 | } 23 | return { head } 24 | } 25 | 26 | async function makeLinkForDoc(blocks: Transaction, update: DocUpdate): Promise { 27 | let value: DocValue 28 | if (update.del) { 29 | value = { del: true } 30 | } else { 31 | value = { doc: update.value } 32 | } 33 | const block = await encode({ value, hasher, codec }) 34 | blocks.putSync(block.cid, block.bytes) 35 | return block.cid 36 | } 37 | 38 | export async function getValueFromCrdt(blocks: TransactionBlockstore, head: ClockHead, key: string): Promise { 39 | const link = await get(blocks, head, key) 40 | if (!link) throw new Error(`Missing key ${key}`) 41 | return await getValueFromLink(blocks, link) 42 | } 43 | 44 | async function getValueFromLink(blocks: TransactionBlockstore, link: AnyLink): Promise { 45 | const block = await blocks.get(link) 46 | if (!block) throw new Error(`Missing block ${link.toString()}`) 47 | const { value } = (await decode({ bytes: block.bytes, hasher, codec })) as { value: DocValue } 48 | return value 49 | } 50 | 51 | export async function clockChangesSince( 52 | blocks: TransactionBlockstore, 53 | head: ClockHead, 54 | since: ClockHead 55 | ): Promise<{ result: DocUpdate[], head: ClockHead }> { 56 | const eventsFetcher = new EventFetcher(blocks) 57 | const keys: Set = new Set() 58 | const updates = await gatherUpdates(blocks, eventsFetcher, head, since, [], keys) 59 | return { result: updates.reverse(), head } 60 | } 61 | 62 | async function gatherUpdates( 63 | blocks: TransactionBlockstore, 64 | eventsFetcher: EventFetcher, 65 | head: ClockHead, 66 | since: ClockHead, 67 | updates: DocUpdate[] = [], 68 | keys: Set 69 | ): Promise { 70 | const sHead = head.map(l => l.toString()) 71 | for (const link of since) { 72 | if (sHead.includes(link.toString())) { 73 | return updates 74 | } 75 | } 76 | for (const link of head) { 77 | const { value: event } = await eventsFetcher.get(link) 78 | const { key, value } = event.data 79 | if (keys.has(key)) continue 80 | keys.add(key) 81 | const docValue = await getValueFromLink(blocks, value) 82 | updates.push({ key, value: docValue.doc, del: docValue.del }) 83 | if (event.parents) { 84 | updates = await gatherUpdates(blocks, eventsFetcher, event.parents, since, updates, keys) 85 | } 86 | } 87 | return updates 88 | } 89 | 90 | export async function doCompact(blocks: TransactionBlockstore, head: ClockHead) { 91 | const blockLog = new LoggingFetcher(blocks) 92 | const newBlocks = new Transaction(blocks) 93 | 94 | for await (const [, link] of entries(blockLog, head)) { 95 | const bl = await blocks.get(link) 96 | if (!bl) throw new Error('Missing block: ' + link.toString()) 97 | await newBlocks.put(link, bl.bytes) 98 | } 99 | 100 | for (const cid of blockLog.cids) { 101 | const bl = await blocks.get(cid) 102 | if (!bl) throw new Error('Missing block: ' + cid.toString()) 103 | await newBlocks.put(cid, bl.bytes) 104 | } 105 | 106 | await blocks.commitCompaction(newBlocks, head) 107 | } 108 | 109 | class LoggingFetcher implements BlockFetcher { 110 | blocks: BlockFetcher 111 | cids: Set = new Set() 112 | constructor(blocks: BlockFetcher) { 113 | this.blocks = blocks 114 | } 115 | 116 | async get(cid: AnyLink) { 117 | this.cids.add(cid) 118 | return await this.blocks.get(cid) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@fireproof/database", 3 | "version": "0.10.67", 4 | "description": "Live database for the web", 5 | "main": "./dist/browser/fireproof.cjs", 6 | "module": "./dist/browser/fireproof.esm.js", 7 | "exports": { 8 | ".": { 9 | "import": "./dist/browser/fireproof.esm.js", 10 | "require": "./dist/browser/fireproof.cjs", 11 | "types": "./dist/types/fireproof.d.ts", 12 | "script": "./dist/browser/fireproof.iife.js", 13 | "default": "./dist/browser/fireproof.esm.js" 14 | }, 15 | "./node": { 16 | "import": "./dist/node/fireproof.esm.js", 17 | "require": "./dist/node/fireproof.cjs", 18 | "types": "./dist/types/fireproof.d.ts", 19 | "script": "./dist/browser/fireproof.iife.js", 20 | "default": "./dist/node/fireproof.esm.js" 21 | }, 22 | "./database": { 23 | "import": "./dist/node/database.esm.js", 24 | "require": "./dist/node/database.cjs", 25 | "types": "./dist/types/database.d.ts", 26 | "script": "./dist/browser/database.iife.js", 27 | "default": "./dist/node/database.esm.js" 28 | }, 29 | "./index": { 30 | "import": "./dist/node/index.esm.js", 31 | "require": "./dist/node/index.cjs", 32 | "types": "./dist/types/index.d.ts", 33 | "script": "./dist/browser/index.iife.js", 34 | "default": "./dist/node/index.esm.js" 35 | } 36 | }, 37 | "browser": "./dist/fireproof.browser.iife.js", 38 | "types": "./dist/types/fireproof.d.ts", 39 | "files": [ 40 | "src", 41 | "dist/node", 42 | "dist/browser", 43 | "dist/types" 44 | ], 45 | "type": "module", 46 | "scripts": { 47 | "build:version": "node -p \"'export const PACKAGE_VERSION = ' + JSON.stringify(require('./package.json').version) + ';'\" > src/version.ts", 48 | "build": "npm run clean && node ./scripts/build.js", 49 | "build:types": "tsc --declaration --outDir dist/types && node ./scripts/types.js", 50 | "prepublishOnly": "npm run build:all", 51 | "build:all": "npm run build && npm run build:types && npm run build:version", 52 | "clean": "rm -rf dist/*", 53 | "start": "node ./scripts/serve.js", 54 | "analyze": "node ./scripts/analyze.js", 55 | "test:watch": "nodemon -w src -w test -e ts,js --exec \"npm run build && npm run test:node\"", 56 | "test:node": "node ./scripts/test.js", 57 | "test:browser": "node ./scripts/browser-test.js", 58 | "test:coverage": "c8 --reporter=html --include='dist/*' node ./scripts/test.js && open coverage/src/index.html", 59 | "test": "npm run build && npm run test:node && npm run test:browser && tsc", 60 | "lint": "eslint 'src/**/*.{js,ts}'", 61 | "lint:exports": "ts-unused-exports tsconfig.json", 62 | "lint:fix": "eslint --fix 'src/**/*.{js,ts}'" 63 | }, 64 | "keywords": [ 65 | "database", 66 | "JSON", 67 | "document", 68 | "IPLD", 69 | "CID", 70 | "IPFS" 71 | ], 72 | "contributors": [ 73 | "J Chris Anderson", 74 | "Alan Shaw", 75 | "Travis Vachon", 76 | "Mikeal Rogers" 77 | ], 78 | "author": "J Chris Anderson", 79 | "license": "Apache-2.0 OR MIT", 80 | "homepage": "https://fireproof.storage", 81 | "repository": { 82 | "type": "git", 83 | "url": "git+https://github.com/fireproof-storage/fireproof.git" 84 | }, 85 | "bugs": { 86 | "url": "https://github.com/fireproof-storage/fireproof/issues" 87 | }, 88 | "devDependencies": { 89 | "@types/async": "^3.2.20", 90 | "@types/mocha": "^10.0.1", 91 | "@typescript-eslint/eslint-plugin": "^6.1.0", 92 | "@typescript-eslint/parser": "^6.1.0", 93 | "browser-assert": "^1.2.1", 94 | "c8": "^8.0.1", 95 | "crypto-browserify": "^3.12.0", 96 | "esbuild": "^0.18.14", 97 | "esbuild-plugin-alias": "^0.2.1", 98 | "esbuild-plugin-polyfill-node": "^0.3.0", 99 | "esbuild-plugin-tsc": "^0.4.0", 100 | "eslint": "^8.45.0", 101 | "eslint-config-standard": "^17.1.0", 102 | "eslint-plugin-import": "^2.27.5", 103 | "eslint-plugin-mocha": "^10.1.0", 104 | "eslint-plugin-node": "^11.1.0", 105 | "eslint-plugin-promise": "^6.1.1", 106 | "memfs": "^4.2.1", 107 | "mocha": "^10.2.0", 108 | "nodemon": "^3.0.1", 109 | "os-browserify": "^0.3.0", 110 | "path-browserify": "^1.0.1", 111 | "process": "^0.11.10", 112 | "puppeteer": "^21.0.3", 113 | "stream-browserify": "^3.0.0", 114 | "ts-unused-exports": "^10.0.0", 115 | "tslib": "^2.6.0", 116 | "typescript": "^5.1.6", 117 | "util": "^0.12.5" 118 | }, 119 | "dependencies": { 120 | "@alanshaw/pail": "^0.3.3", 121 | "@ipld/dag-cbor": "^9.0.3", 122 | "@ipld/dag-json": "^10.1.2", 123 | "@peculiar/webcrypto": "^1.4.3", 124 | "charwise": "^3.0.1", 125 | "idb": "^7.1.1", 126 | "multiformats": "^12.0.1", 127 | "prolly-trees": "^1.0.4" 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/loader.ts: -------------------------------------------------------------------------------- 1 | import { CarReader } from '@ipld/car' 2 | import { innerMakeCarFile, parseCarFile } from './loader-helpers' 3 | import { Transaction } from './transaction' 4 | import type { 5 | AnyBlock, AnyCarHeader, AnyLink, BulkResult, 6 | CarCommit, DbCarHeader, DbMeta, FireproofOptions, IdxCarHeader, 7 | IdxMeta, IdxMetaMap 8 | } from './types' 9 | import { CID } from 'multiformats' 10 | import { CarStore, HeaderStore } from './store' 11 | import { decodeEncryptedCar, encryptedMakeCarFile } from './encrypt-helpers' 12 | import { getCrypto, randomBytes } from './encrypted-block' 13 | 14 | abstract class Loader { 15 | name: string 16 | opts: FireproofOptions = {} 17 | 18 | headerStore: HeaderStore | undefined 19 | carStore: CarStore | undefined 20 | carLog: AnyLink[] = [] 21 | carReaders: Map = new Map() 22 | ready: Promise 23 | key: string | undefined 24 | keyId: string | undefined 25 | 26 | static defaultHeader: AnyCarHeader 27 | abstract defaultHeader: AnyCarHeader 28 | 29 | constructor(name: string, opts?: FireproofOptions) { 30 | this.name = name 31 | this.opts = opts || this.opts 32 | this.ready = this.initializeStores().then(async () => { 33 | if (!this.headerStore || !this.carStore) throw new Error('stores not initialized') 34 | const meta = await this.headerStore.load('main') 35 | return await this.ingestCarHeadFromMeta(meta) 36 | }) 37 | } 38 | 39 | async commit(t: Transaction, done: IndexerResult | BulkResult, compact: boolean = false): Promise { 40 | await this.ready 41 | const fp = this.makeCarHeader(done, this.carLog, compact) 42 | const { cid, bytes } = this.key ? await encryptedMakeCarFile(this.key, fp, t) : await innerMakeCarFile(fp, t) 43 | await this.carStore!.save({ cid, bytes }) 44 | if (compact) { 45 | for (const cid of this.carLog) { 46 | await this.carStore!.remove(cid) 47 | } 48 | this.carLog.splice(0, this.carLog.length, cid) 49 | } else { 50 | this.carLog.push(cid) 51 | } 52 | await this.headerStore!.save({ car: cid, key: this.key || null }) 53 | return cid 54 | } 55 | 56 | async getBlock(cid: CID): Promise { 57 | await this.ready 58 | for (const [, reader] of [...this.carReaders]) { 59 | const block = await reader.get(cid) 60 | if (block) { 61 | return block 62 | } 63 | } 64 | } 65 | 66 | protected async initializeStores() { 67 | const isBrowser = typeof window !== 'undefined' 68 | console.log('is browser?', isBrowser) 69 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment 70 | const module = isBrowser ? await require('./store-browser') : await require('./store-fs') 71 | if (module) { 72 | // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access 73 | this.headerStore = new module.HeaderStore(this.name) as HeaderStore 74 | // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access 75 | this.carStore = new module.CarStore(this.name) as CarStore 76 | } else { 77 | throw new Error('Failed to initialize stores.') 78 | } 79 | } 80 | 81 | protected abstract makeCarHeader(_result: BulkResult | IndexerResult, _cars: AnyLink[], _compact: boolean): AnyCarHeader; 82 | 83 | protected async loadCar(cid: AnyLink): Promise { 84 | if (!this.headerStore || !this.carStore) throw new Error('stores not initialized') 85 | if (this.carReaders.has(cid.toString())) return this.carReaders.get(cid.toString()) as CarReader 86 | const car = await this.carStore.load(cid) 87 | if (!car) throw new Error(`missing car file ${cid.toString()}`) 88 | const reader = await this.ensureDecryptedReader(await CarReader.fromBytes(car.bytes)) as CarReader 89 | this.carReaders.set(cid.toString(), reader) 90 | this.carLog.push(cid) 91 | return reader 92 | } 93 | 94 | protected async ensureDecryptedReader(reader: CarReader) { 95 | if (!this.key) return reader 96 | const { blocks, root } = await decodeEncryptedCar(this.key, reader) 97 | return { 98 | getRoots: () => [root], 99 | get: blocks.get.bind(blocks) 100 | } 101 | } 102 | 103 | protected async setKey(key: string) { 104 | if (this.key && this.key !== key) throw new Error('key already set') 105 | this.key = key 106 | const crypto = getCrypto() 107 | if (!crypto) throw new Error('missing crypto module') 108 | const subtle = crypto.subtle 109 | const encoder = new TextEncoder() 110 | const data = encoder.encode(key) 111 | const hashBuffer = await subtle.digest('SHA-256', data) 112 | const hashArray = Array.from(new Uint8Array(hashBuffer)) 113 | this.keyId = hashArray.map(b => b.toString(16).padStart(2, '0')).join('') 114 | } 115 | 116 | protected async ingestCarHeadFromMeta(meta: DbMeta | null): Promise { 117 | if (!this.headerStore || !this.carStore) throw new Error('stores not initialized') 118 | if (!meta) { 119 | // generate a random key 120 | if (!this.opts.public) { 121 | if (getCrypto()) { 122 | await this.setKey(randomBytes(32).toString('hex')) 123 | } else { 124 | console.warn('missing crypto module, using public mode') 125 | } 126 | } 127 | console.log('no meta, returning default header', this.name, this.keyId) 128 | return this.defaultHeader 129 | } 130 | const { car: cid, key } = meta 131 | console.log('ingesting car head from meta', { car: cid, key }) 132 | if (key) { 133 | await this.setKey(key) 134 | } 135 | const reader = await this.loadCar(cid) 136 | this.carLog = [cid] // this.carLog.push(cid) 137 | const carHeader = await parseCarFile(reader) 138 | await this.getMoreReaders(carHeader.cars) 139 | return carHeader 140 | } 141 | 142 | protected async getMoreReaders(cids: AnyLink[]) { 143 | await Promise.all(cids.map(cid => this.loadCar(cid))) 144 | } 145 | } 146 | 147 | export class DbLoader extends Loader { 148 | declare ready: Promise // todo this will be a map of headers by branch name 149 | 150 | static defaultHeader = { cars: [], compact: [], head: [] } 151 | defaultHeader = DbLoader.defaultHeader 152 | 153 | protected makeCarHeader({ head }: BulkResult, cars: AnyLink[], compact: boolean = false): DbCarHeader { 154 | return compact ? { head, cars: [], compact: cars } : { head, cars, compact: [] } 155 | } 156 | } 157 | 158 | export class IdxLoader extends Loader { 159 | declare ready: Promise 160 | 161 | static defaultHeader = { cars: [], compact: [], indexes: new Map() as Map } 162 | defaultHeader = IdxLoader.defaultHeader 163 | 164 | protected makeCarHeader({ indexes }: IndexerResult, cars: AnyLink[], compact: boolean = false): IdxCarHeader { 165 | return compact ? { indexes, cars: [], compact: cars } : { indexes, cars, compact: [] } 166 | } 167 | } 168 | 169 | type IndexerResult = CarCommit & IdxMetaMap 170 | -------------------------------------------------------------------------------- /src/indexer-helpers.ts: -------------------------------------------------------------------------------- 1 | import type { Block, Link } from 'multiformats' 2 | import { create } from 'multiformats/block' 3 | import { sha256 as hasher } from 'multiformats/hashes/sha2' 4 | import * as codec from '@ipld/dag-cbor' 5 | 6 | // @ts-ignore 7 | import charwise from 'charwise' 8 | // @ts-ignore 9 | import * as DbIndex from 'prolly-trees/db-index' 10 | // @ts-ignore 11 | import { bf, simpleCompare } from 'prolly-trees/utils' 12 | // @ts-ignore 13 | import { nocache as cache } from 'prolly-trees/cache' 14 | // @ts-ignore 15 | import { ProllyNode as BaseNode } from 'prolly-trees/base' 16 | 17 | import { AnyLink, DocUpdate, MapFn, DocFragment, BlockFetcher, IndexKey, IndexUpdate, QueryOpts, IndexRow, AnyBlock } from './types' 18 | import { Transaction } from './transaction' 19 | import { CRDT } from './crdt' 20 | 21 | export class IndexTree { 22 | cid: AnyLink | null = null 23 | root: ProllyNode | null = null 24 | } 25 | 26 | type CompareRef = string | number 27 | type CompareKey = [string | number, CompareRef] 28 | 29 | const refCompare = (aRef: CompareRef, bRef: CompareRef) => { 30 | if (Number.isNaN(aRef)) return -1 31 | if (Number.isNaN(bRef)) throw new Error('ref may not be Infinity or NaN') 32 | if (aRef === Infinity) return 1 33 | // if (!Number.isFinite(bRef)) throw new Error('ref may not be Infinity or NaN') 34 | // eslint-disable-next-line @typescript-eslint/no-unsafe-call 35 | return simpleCompare(aRef, bRef) as number 36 | } 37 | 38 | const compare = (a: CompareKey, b: CompareKey) => { 39 | const [aKey, aRef] = a 40 | const [bKey, bRef] = b 41 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call 42 | const comp: number = simpleCompare(aKey, bKey) 43 | if (comp !== 0) return comp 44 | return refCompare(aRef, bRef) 45 | } 46 | 47 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call 48 | export const byKeyOpts: StaticProllyOptions = { cache, chunker: bf(30), codec, hasher, compare } 49 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call 50 | export const byIdOpts: StaticProllyOptions = { cache, chunker: bf(30), codec, hasher, compare: simpleCompare } 51 | 52 | export function indexEntriesForChanges( 53 | changes: DocUpdate[], 54 | mapFn: MapFn 55 | ): { key: [string, string]; value: DocFragment }[] { 56 | const indexEntries: { key: [string, string]; value: DocFragment }[] = [] 57 | changes.forEach(({ key: _id, value, del }) => { 58 | if (del || !value) return 59 | let mapCalled = false 60 | const mapReturn = mapFn({ _id, ...value }, (k: string, v?: DocFragment) => { 61 | mapCalled = true 62 | if (typeof k === 'undefined') return 63 | indexEntries.push({ 64 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 65 | key: [charwise.encode(k) as string, _id], 66 | value: v || null 67 | }) 68 | }) 69 | if (!mapCalled && mapReturn) { 70 | indexEntries.push({ 71 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 72 | key: [charwise.encode(mapReturn) as string, _id], 73 | value: null 74 | }) 75 | } 76 | }) 77 | return indexEntries 78 | } 79 | 80 | function makeProllyGetBlock(blocks: BlockFetcher): (address: AnyLink) => Promise { 81 | return async (address: AnyLink) => { 82 | const block = await blocks.get(address) 83 | if (!block) throw new Error(`Missing block ${address.toString()}`) 84 | const { cid, bytes } = block 85 | return create({ cid, bytes, hasher, codec }) as Promise 86 | } 87 | } 88 | 89 | export async function bulkIndex(tblocks: Transaction, inIndex: IndexTree, indexEntries: IndexUpdate[], opts: StaticProllyOptions): Promise { 90 | if (!indexEntries.length) return inIndex 91 | if (!inIndex.root) { 92 | if (!inIndex.cid) { 93 | let returnRootBlock: Block | null = null 94 | let returnNode: ProllyNode | null = null 95 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 96 | for await (const node of await DbIndex.create({ get: makeProllyGetBlock(tblocks), list: indexEntries, ...opts }) as ProllyNode[]) { 97 | const block = await node.block 98 | await tblocks.put(block.cid, block.bytes) 99 | returnRootBlock = block 100 | returnNode = node 101 | } 102 | if (!returnNode || !returnRootBlock) throw new Error('failed to create index') 103 | return { root: returnNode, cid: returnRootBlock.cid } 104 | } else { 105 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 106 | inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts }) as ProllyNode 107 | } 108 | } 109 | const { root, blocks: newBlocks } = await inIndex.root.bulk(indexEntries) 110 | if (root) { 111 | for await (const block of newBlocks) { 112 | await tblocks.put(block.cid, block.bytes) 113 | } 114 | return { root, cid: (await root.block).cid } 115 | } else { 116 | return { root: null, cid: null } 117 | } 118 | } 119 | 120 | export async function loadIndex(tblocks: BlockFetcher, cid: AnyLink, opts: StaticProllyOptions): Promise { 121 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 122 | return await DbIndex.load({ cid, get: makeProllyGetBlock(tblocks), ...opts }) as ProllyNode 123 | } 124 | 125 | export async function applyQuery(crdt: CRDT, resp: { result: IndexRow[] }, query: QueryOpts) { 126 | if (query.descending) { 127 | resp.result = resp.result.reverse() 128 | } 129 | if (query.limit) { 130 | resp.result = resp.result.slice(0, query.limit) 131 | } 132 | if (query.includeDocs) { 133 | resp.result = await Promise.all( 134 | resp.result.map(async row => { 135 | const val = await crdt.get(row.id) 136 | const doc = val ? { _id: row.id, ...val.doc } : null 137 | return { ...row, doc } 138 | }) 139 | ) 140 | } 141 | return { 142 | rows: resp.result.map(row => { 143 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 144 | row.key = (charwise.decode(row.key) as IndexKey) 145 | return row 146 | }) 147 | } 148 | } 149 | 150 | export function encodeRange(range: [DocFragment, DocFragment]): [IndexKey, IndexKey] { 151 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 152 | return range.map(key => charwise.encode(key) as IndexKey) as [IndexKey, IndexKey] 153 | } 154 | 155 | export function encodeKey(key: string): string { 156 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 157 | return charwise.encode(key) as string 158 | } 159 | 160 | // ProllyNode type based on the ProllyNode from 'prolly-trees/base' 161 | export interface ProllyNode extends BaseNode { 162 | getAllEntries(): PromiseLike<{ [x: string]: any; result: IndexRow[] }> 163 | getMany(removeIds: string[]): Promise<{ [x: string]: any; result: IndexKey[] }> 164 | range(a: IndexKey, b: IndexKey): Promise<{ result: IndexRow[] }> 165 | get(key: string): Promise<{ result: IndexRow[] }> 166 | bulk(bulk: IndexUpdate[]): PromiseLike<{ root: ProllyNode | null; blocks: Block[] }> 167 | address: Promise 168 | distance: number 169 | compare: (a: any, b: any) => number 170 | cache: any 171 | block: Promise 172 | } 173 | 174 | export interface StaticProllyOptions { 175 | cache: any 176 | chunker: (entry: any, distance: number) => boolean 177 | codec: any 178 | hasher: any 179 | compare: (a: any, b: any) => number 180 | } 181 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import type { ClockHead, DocUpdate, MapFn, IndexUpdate, QueryOpts, IdxMeta, IdxCarHeader } from './types' 2 | import { IndexBlockstore } from './transaction' 3 | import { bulkIndex, indexEntriesForChanges, byIdOpts, byKeyOpts, IndexTree, applyQuery, encodeRange, encodeKey, loadIndex } from './indexer-helpers' 4 | import { CRDT } from './crdt' 5 | 6 | export function index({ _crdt }: { _crdt: CRDT}, name: string, mapFn?: MapFn, meta?: IdxMeta): Index { 7 | if (mapFn && meta) throw new Error('cannot provide both mapFn and meta') 8 | if (mapFn && mapFn.constructor.name !== 'Function') throw new Error('mapFn must be a function') 9 | if (_crdt.indexers.has(name)) { 10 | const idx = _crdt.indexers.get(name)! 11 | idx.applyMapFn(name, mapFn, meta) 12 | } else { 13 | const idx = new Index(_crdt, name, mapFn, meta) 14 | _crdt.indexers.set(name, idx) 15 | } 16 | return _crdt.indexers.get(name)! 17 | } 18 | 19 | export class Index { 20 | blocks: IndexBlockstore 21 | crdt: CRDT 22 | name: string | null = null 23 | mapFn: MapFn | null = null 24 | mapFnString: string = '' 25 | byKey = new IndexTree() 26 | byId = new IndexTree() 27 | indexHead: ClockHead | undefined = undefined 28 | includeDocsDefault: boolean = false 29 | initError: Error | null = null 30 | ready: Promise 31 | 32 | constructor(crdt: CRDT, name: string, mapFn?: MapFn, meta?: IdxMeta) { 33 | this.blocks = crdt.indexBlocks 34 | this.crdt = crdt 35 | this.applyMapFn(name, mapFn, meta) 36 | if (!(this.mapFnString || this.initError)) throw new Error('missing mapFnString') 37 | this.ready = this.blocks.ready.then((header: IdxCarHeader) => { 38 | // @ts-ignore 39 | if (header.head) throw new Error('cannot have head in idx header') 40 | if (header.indexes === undefined) throw new Error('missing indexes in idx header') 41 | for (const [name, idx] of Object.entries(header.indexes)) { 42 | index({ _crdt: crdt }, name, undefined, idx as IdxMeta) 43 | } 44 | }) 45 | } 46 | 47 | applyMapFn(name: string, mapFn?: MapFn, meta?: IdxMeta) { 48 | if (mapFn && meta) throw new Error('cannot provide both mapFn and meta') 49 | if (this.name && this.name !== name) throw new Error('cannot change name') 50 | this.name = name 51 | try { 52 | if (meta) { 53 | // hydrating from header 54 | if (this.indexHead && 55 | this.indexHead.map(c => c.toString()).join() !== meta.head.map(c => c.toString()).join()) { 56 | throw new Error('cannot apply meta to existing index') 57 | } 58 | this.byId.cid = meta.byId 59 | this.byKey.cid = meta.byKey 60 | this.indexHead = meta.head 61 | if (this.mapFnString) { 62 | // we already initialized from application code 63 | if (this.mapFnString !== meta.map) throw new Error('cannot apply different mapFn meta') 64 | } else { 65 | // we are first 66 | this.mapFnString = meta.map 67 | } 68 | } else { 69 | if (this.mapFn) { 70 | // we already initialized from application code 71 | if (mapFn) { 72 | if (this.mapFn.toString() !== mapFn.toString()) throw new Error('cannot apply different mapFn app2') 73 | } 74 | } else { 75 | // application code is creating an index 76 | if (!mapFn) { 77 | mapFn = makeMapFnFromName(name) 78 | } 79 | if (this.mapFnString) { 80 | // we already loaded from a header 81 | if (this.mapFnString !== mapFn.toString()) throw new Error('cannot apply different mapFn app') 82 | } else { 83 | // we are first 84 | this.mapFnString = mapFn.toString() 85 | } 86 | this.mapFn = mapFn 87 | } 88 | } 89 | const matches = /=>\s*(.*)/.test(this.mapFnString) 90 | this.includeDocsDefault = matches 91 | } catch (e) { 92 | this.initError = e as Error 93 | } 94 | } 95 | 96 | async query(opts: QueryOpts = {}) { 97 | await this._updateIndex() 98 | await this._hydrateIndex() 99 | if (!this.byKey.root) return await applyQuery(this.crdt, { result: [] }, opts) 100 | if (this.includeDocsDefault && opts.includeDocs === undefined) opts.includeDocs = true 101 | if (opts.range) { 102 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 103 | const { result, ...all } = await this.byKey.root.range(...encodeRange(opts.range)) 104 | return await applyQuery(this.crdt, { result, ...all }, opts) 105 | } 106 | if (opts.key) { 107 | const encodedKey = encodeKey(opts.key) 108 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call 109 | return await applyQuery(this.crdt, await this.byKey.root.get(encodedKey), opts) 110 | } 111 | if (opts.prefix) { 112 | if (!Array.isArray(opts.prefix)) opts.prefix = [opts.prefix] 113 | const start = [...opts.prefix, NaN] 114 | const end = [...opts.prefix, Infinity] 115 | const encodedR = encodeRange([start, end]) 116 | return await applyQuery(this.crdt, await this.byKey.root.range(...encodedR), opts) 117 | } 118 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call 119 | const { result, ...all } = await this.byKey.root.getAllEntries() // funky return type 120 | return await applyQuery(this.crdt, { 121 | result: result.map(({ key: [k, id], value }) => 122 | ({ key: k, id, value })), 123 | ...all 124 | }, opts) 125 | } 126 | 127 | async _hydrateIndex() { 128 | if (this.byId.root && this.byKey.root) return 129 | if (!this.byId.cid || !this.byKey.cid) return 130 | this.byId.root = await loadIndex(this.blocks, this.byId.cid, byIdOpts) 131 | this.byKey.root = await loadIndex(this.blocks, this.byKey.cid, byKeyOpts) 132 | } 133 | 134 | async _updateIndex() { 135 | await this.ready 136 | if (this.initError) throw this.initError 137 | if (!this.mapFn) throw new Error('No map function defined') 138 | const { result, head } = await this.crdt.changes(this.indexHead) 139 | if (result.length === 0) { 140 | this.indexHead = head 141 | return { byId: this.byId, byKey: this.byKey } 142 | } 143 | let staleKeyIndexEntries: IndexUpdate[] = [] 144 | let removeIdIndexEntries: IndexUpdate[] = [] 145 | if (this.byId.root) { 146 | const removeIds = result.map(({ key }) => key) 147 | const { result: oldChangeEntries } = await this.byId.root.getMany(removeIds) as { result: Array<[string, string] | string> } 148 | staleKeyIndexEntries = oldChangeEntries.map(key => ({ key, del: true })) 149 | removeIdIndexEntries = oldChangeEntries.map((key) => ({ key: key[1], del: true })) 150 | } 151 | const indexEntries = indexEntriesForChanges(result, this.mapFn) // use a getter to translate from string 152 | const byIdIndexEntries: DocUpdate[] = indexEntries.map(({ key }) => ({ key: key[1], value: key })) 153 | const indexerMeta: Map = new Map() 154 | for (const [name, indexer] of this.crdt.indexers) { 155 | if (indexer.indexHead) { 156 | indexerMeta.set(name, { 157 | byId: indexer.byId.cid, 158 | byKey: indexer.byKey.cid, 159 | head: indexer.indexHead, 160 | map: indexer.mapFnString, 161 | name: indexer.name 162 | } as IdxMeta) 163 | } 164 | } 165 | return await this.blocks.transaction(async (tblocks): Promise => { 166 | this.byId = await bulkIndex( 167 | tblocks, 168 | this.byId, 169 | removeIdIndexEntries.concat(byIdIndexEntries), 170 | byIdOpts 171 | ) 172 | this.byKey = await bulkIndex(tblocks, this.byKey, staleKeyIndexEntries.concat(indexEntries), byKeyOpts) 173 | this.indexHead = head 174 | return { byId: this.byId.cid, byKey: this.byKey.cid, head, map: this.mapFnString, name: this.name } as IdxMeta 175 | }, indexerMeta) 176 | } 177 | } 178 | 179 | function makeMapFnFromName(name: string): MapFn { 180 | return (doc) => { 181 | if (doc[name]) return doc[name] 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /test/database.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable mocha/max-top-level-suites */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-assignment */ 3 | /* eslint-disable @typescript-eslint/no-unsafe-member-access */ 4 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 5 | import { assert, equals, notEquals, matches, resetDirectory } from './helpers.js' 6 | import { Database } from '../dist/test/database.esm.js' 7 | // import { Doc } from '../dist/test/types.d.esm.js' 8 | import { HeaderStore } from '../dist/test/store-fs.esm.js' 9 | 10 | /** 11 | * @typedef {Object.} DocBody 12 | */ 13 | 14 | /** 15 | * @typedef {Object} Doc 16 | * @property {string} _id 17 | * @property {DocBody} [property] - an additional property 18 | */ 19 | 20 | describe('basic Database', function () { 21 | /** @type {Database} */ 22 | let db 23 | beforeEach(function () { 24 | db = new Database() 25 | }) 26 | it('should put', async function () { 27 | /** @type {Doc} */ 28 | const doc = { _id: 'hello', value: 'world' } 29 | const ok = await db.put(doc) 30 | equals(ok.id, 'hello') 31 | }) 32 | it('get missing should throw', async function () { 33 | // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-return 34 | const e = await (db.get('missing')).catch(e => e) 35 | matches(e.message, /Not found/) 36 | }) 37 | it('del missing should result in deleted state', async function () { 38 | await db.del('missing') 39 | // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-return 40 | const e = await (db.get('missing')).catch(e => e) 41 | matches(e.message, /Not found/) 42 | }) 43 | it('has no changes', async function () { 44 | const { rows } = await db.changes([]) 45 | equals(rows.length, 0) 46 | }) 47 | }) 48 | 49 | describe('basic Database with record', function () { 50 | /** @type {Database} */ 51 | let db 52 | beforeEach(async function () { 53 | db = new Database() 54 | /** @type {Doc} */ 55 | const doc = { _id: 'hello', value: 'world' } 56 | const ok = await db.put(doc) 57 | equals(ok.id, 'hello') 58 | }) 59 | it('should get', async function () { 60 | const doc = await db.get('hello') 61 | assert(doc) 62 | equals(doc._id, 'hello') 63 | equals(doc.value, 'world') 64 | }) 65 | it('should update', async function () { 66 | const ok = await db.put({ _id: 'hello', value: 'universe' }) 67 | equals(ok.id, 'hello') 68 | const doc = await db.get('hello') 69 | assert(doc) 70 | equals(doc._id, 'hello') 71 | equals(doc.value, 'universe') 72 | }) 73 | it('should del last record', async function () { 74 | const ok = await db.del('hello') 75 | equals(ok.id, 'hello') 76 | // eslint-disable-next-line @typescript-eslint/no-unsafe-return 77 | const e = await (db.get('hello')).catch(e => e) 78 | matches(e.message, /Not found/) 79 | }) 80 | it('has changes', async function () { 81 | const { rows } = await db.changes([]) 82 | equals(rows.length, 1) 83 | equals(rows[0].key, 'hello') 84 | equals(rows[0].value._id, 'hello') 85 | }) 86 | }) 87 | 88 | describe('named Database with record', function () { 89 | /** @type {Database} */ 90 | let db 91 | beforeEach(async function () { 92 | await resetDirectory(HeaderStore.dataDir, 'test-db-name') 93 | 94 | db = new Database('test-db-name') 95 | /** @type {Doc} */ 96 | const doc = { _id: 'hello', value: 'world' } 97 | const ok = await db.put(doc) 98 | equals(ok.id, 'hello') 99 | }) 100 | it('should get', async function () { 101 | const doc = await db.get('hello') 102 | assert(doc) 103 | equals(doc._id, 'hello') 104 | equals(doc.value, 'world') 105 | }) 106 | it('should update', async function () { 107 | const ok = await db.put({ _id: 'hello', value: 'universe' }) 108 | equals(ok.id, 'hello') 109 | const doc = await db.get('hello') 110 | assert(doc) 111 | equals(doc._id, 'hello') 112 | equals(doc.value, 'universe') 113 | }) 114 | it('should del last record', async function () { 115 | const ok = await db.del('hello') 116 | equals(ok.id, 'hello') 117 | // eslint-disable-next-line @typescript-eslint/no-unsafe-return 118 | const e = await (db.get('hello')).catch(e => e) 119 | matches(e.message, /Not found/) 120 | }) 121 | it('has changes', async function () { 122 | const { rows } = await db.changes([]) 123 | equals(rows.length, 1) 124 | equals(rows[0].key, 'hello') 125 | equals(rows[0].value._id, 'hello') 126 | }) 127 | it('should have a key', async function () { 128 | const { rows } = await db.changes([]) 129 | equals(rows.length, 1) 130 | const loader = db._crdt.blocks.loader 131 | await loader.ready 132 | equals(loader.key.length, 64) 133 | equals(loader.keyId.length, 64) 134 | notEquals(loader.key, loader.keyId) 135 | }) 136 | }) 137 | 138 | describe('basic Database parallel writes / public', function () { 139 | /** @type {Database} */ 140 | let db 141 | const writes = [] 142 | beforeEach(async function () { 143 | await resetDirectory(HeaderStore.dataDir, 'test-parallel-writes') 144 | db = new Database('test-parallel-writes', { public: true }) 145 | /** @type {Doc} */ 146 | for (let i = 0; i < 10; i++) { 147 | const doc = { _id: `id-${i}`, hello: 'world' } 148 | writes.push(db.put(doc)) 149 | } 150 | await Promise.all(writes) 151 | }) 152 | it('should have one head', function () { 153 | const crdt = db._crdt 154 | equals(crdt._head.length, 1) 155 | }) 156 | it('should write all', async function () { 157 | for (let i = 0; i < 10; i++) { 158 | const id = `id-${i}` 159 | const doc = await db.get(id) 160 | assert(doc) 161 | equals(doc._id, id) 162 | equals(doc.hello, 'world') 163 | } 164 | }) 165 | it('should del all', async function () { 166 | for (let i = 0; i < 10; i++) { 167 | const id = `id-${i}` 168 | const ok = await db.del(id) 169 | equals(ok.id, id) 170 | // eslint-disable-next-line @typescript-eslint/no-unsafe-return 171 | const e = await (db.get(id)).catch(e => e) 172 | matches(e.message, /Not found/) 173 | } 174 | }) 175 | it('should delete all in parallel', async function () { 176 | const deletes = [] 177 | for (let i = 0; i < 10; i++) { 178 | const id = `id-${i}` 179 | deletes.push(db.del(id)) 180 | } 181 | await Promise.all(deletes) 182 | for (let i = 0; i < 10; i++) { 183 | const id = `id-${i}` 184 | // eslint-disable-next-line @typescript-eslint/no-unsafe-return 185 | const e = await (db.get(id)).catch(e => e) 186 | matches(e.message, /Not found/) 187 | } 188 | }) 189 | it('has changes', async function () { 190 | const { rows } = await db.changes([]) 191 | equals(rows.length, 10) 192 | for (let i = 0; i < 10; i++) { 193 | equals(rows[i].key, 'id-' + i) 194 | } 195 | }) 196 | it('should not have a key', async function () { 197 | const { rows } = await db.changes([]) 198 | equals(rows.length, 10) 199 | assert(db.opts.public) 200 | assert(db._crdt.opts.public) 201 | const loader = db._crdt.blocks.loader 202 | await loader.ready 203 | equals(loader.key, undefined) 204 | equals(loader.keyId, undefined) 205 | }) 206 | }) 207 | 208 | describe('basic Database with subscription', function () { 209 | /** @type {Database} */ 210 | let db, didRun, unsubscribe 211 | beforeEach(function () { 212 | db = new Database() 213 | didRun = 0 214 | // eslint-disable-next-line @typescript-eslint/no-unsafe-call 215 | unsubscribe = db.subscribe((docs) => { 216 | assert(docs[0]._id) 217 | didRun++ 218 | }) 219 | }) 220 | it('should run on put', async function () { 221 | /** @type {Doc} */ 222 | const doc = { _id: 'hello', message: 'world' } 223 | const ok = await db.put(doc) 224 | equals(ok.id, 'hello') 225 | equals(didRun, 1) 226 | }) 227 | it('should unsubscribe', async function () { 228 | // eslint-disable-next-line @typescript-eslint/no-unsafe-call 229 | unsubscribe() 230 | /** @type {Doc} */ 231 | const doc = { _id: 'hello', message: 'again' } 232 | const ok = await db.put(doc) 233 | equals(ok.id, 'hello') 234 | equals(didRun, 0) 235 | }) 236 | }) 237 | -------------------------------------------------------------------------------- /test/crdt.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-call */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-assignment */ 3 | /* eslint-disable @typescript-eslint/no-unsafe-member-access */ 4 | /* eslint-disable @typescript-eslint/require-await */ 5 | /* eslint-disable mocha/max-top-level-suites */ 6 | import { assert, equals, equalsJSON, matches, notEquals } from './helpers.js' 7 | import { CRDT } from '../dist/test/crdt.esm.js' 8 | import { index } from '../dist/test/index.esm.js' 9 | 10 | describe('Fresh crdt', function () { 11 | /** @type {CRDT} */ 12 | let crdt 13 | beforeEach(function () { 14 | crdt = new CRDT() 15 | }) 16 | it('should have an empty head', async function () { 17 | const head = crdt._head 18 | equals(head.length, 0) 19 | }) 20 | it('should accept put and return results', async function () { 21 | const didPut = await crdt.bulk([{ key: 'hello', value: { hello: 'world' } }]) 22 | const head = didPut.head 23 | equals(head.length, 1) 24 | }) 25 | it('should accept multi-put and return results', async function () { 26 | const didPut = await crdt.bulk([{ key: 'ace', value: { points: 11 } }, { key: 'king', value: { points: 10 } }]) 27 | const head = didPut.head 28 | equals(head.length, 1) 29 | }) 30 | }) 31 | 32 | describe('CRDT with one record', function () { 33 | /** @type {CRDT} */ 34 | let crdt, firstPut 35 | beforeEach(async function () { 36 | crdt = new CRDT() 37 | firstPut = await crdt.bulk([{ key: 'hello', value: { hello: 'world' } }]) 38 | }) 39 | it('should have a one-element head', async function () { 40 | const head = crdt._head 41 | equals(head.length, 1) 42 | }) 43 | it('should return the head', async function () { 44 | equals(firstPut.head.length, 1) 45 | }) 46 | it('return the record on get', async function () { 47 | const got = await crdt.get('hello') 48 | assert(got) 49 | const value = got.doc 50 | equals(value.hello, 'world') 51 | }) 52 | it('should accept another put and return results', async function () { 53 | const didPut = await crdt.bulk([{ key: 'nice', value: { nice: 'data' } }]) 54 | const head = didPut.head 55 | equals(head.length, 1) 56 | const { doc } = await crdt.get('nice') 57 | equals(doc.nice, 'data') 58 | }) 59 | it('should allow for a delete', async function () { 60 | const didDel = await crdt.bulk([{ key: 'hello', del: true }]) 61 | assert(didDel.head) 62 | const got = await crdt.get('hello') 63 | assert(!got) 64 | }) 65 | it('should offer changes', async function () { 66 | const { result } = await crdt.changes([]) 67 | equals(result.length, 1) 68 | equals(result[0].key, 'hello') 69 | equals(result[0].value.hello, 'world') 70 | }) 71 | }) 72 | 73 | describe('CRDT with a multi-write', function () { 74 | /** @type {CRDT} */ 75 | let crdt, firstPut 76 | beforeEach(async function () { 77 | crdt = new CRDT() 78 | firstPut = await crdt.bulk([{ key: 'ace', value: { points: 11 } }, { key: 'king', value: { points: 10 } }]) 79 | }) 80 | it('should have a one-element head', async function () { 81 | const head = crdt._head 82 | equals(head.length, 1) 83 | equals(firstPut.head.length, 1) 84 | }) 85 | it('return the records on get', async function () { 86 | const { doc } = await crdt.get('ace') 87 | equals(doc.points, 11) 88 | 89 | const got2 = await crdt.get('king') 90 | assert(got2) 91 | equals(got2.doc.points, 10) 92 | }) 93 | it('should accept another put and return results', async function () { 94 | const didPut = await crdt.bulk([{ key: 'queen', value: { points: 10 } }]) 95 | const head = didPut.head 96 | equals(head.length, 1) 97 | const got = await crdt.get('queen') 98 | assert(got) 99 | equals(got.doc.points, 10) 100 | }) 101 | it('should offer changes', async function () { 102 | const { result } = await crdt.changes([]) 103 | equals(result.length, 2) 104 | equals(result[0].key, 'ace') 105 | equals(result[0].value.points, 11) 106 | equals(result[1].key, 'king') 107 | }) 108 | it('should offer changes since', async function () { 109 | /** @type {BulkResult} */ 110 | const secondPut = await crdt.bulk([{ key: 'queen', value: { points: 10 } }, { key: 'jack', value: { points: 10 } }]) 111 | assert(secondPut.head) 112 | const { result: r2, head: h2 } = await crdt.changes() 113 | equals(r2.length, 4) 114 | const { result: r3 } = await crdt.changes(firstPut.head) 115 | equals(r3.length, 2) 116 | const { result: r4 } = await crdt.changes(h2) 117 | equals(r4.length, 0) 118 | }) 119 | }) 120 | 121 | describe('CRDT with two multi-writes', function () { 122 | /** @type {CRDT} */ 123 | let crdt, firstPut, secondPut 124 | beforeEach(async function () { 125 | crdt = new CRDT() 126 | firstPut = await crdt.bulk([{ key: 'ace', value: { points: 11 } }, { key: 'king', value: { points: 10 } }]) 127 | secondPut = await crdt.bulk([{ key: 'queen', value: { points: 10 } }, { key: 'jack', value: { points: 10 } }]) 128 | }) 129 | it('should have a one-element head', async function () { 130 | const head = crdt._head 131 | equals(head.length, 1) 132 | equals(firstPut.head.length, 1) 133 | equals(secondPut.head.length, 1) 134 | notEquals(firstPut.head[0], secondPut.head[0]) 135 | }) 136 | it('return the records on get', async function () { 137 | const { doc } = await crdt.get('ace') 138 | equals(doc.points, 11) 139 | 140 | for (const key of ['king', 'queen', 'jack']) { 141 | const { doc } = await crdt.get(key) 142 | equals(doc.points, 10) 143 | } 144 | }) 145 | it('should offer changes', async function () { 146 | const { result } = await crdt.changes() 147 | equals(result.length, 4) 148 | equals(result[0].key, 'ace') 149 | equals(result[0].value.points, 11) 150 | equals(result[1].key, 'king') 151 | equals(result[2].key, 'queen') 152 | equals(result[3].key, 'jack') 153 | }) 154 | }) 155 | 156 | describe('Compact a CRDT with writes', function () { 157 | /** @type {CRDT} */ 158 | let crdt 159 | beforeEach(async function () { 160 | crdt = new CRDT() 161 | for (let i = 0; i < 10; i++) { 162 | const bulk = [{ key: 'ace', value: { points: 11 } }, { key: 'king', value: { points: 10 } }] 163 | await crdt.bulk(bulk) 164 | } 165 | }) 166 | it('has data', async function () { 167 | const got = await crdt.get('ace') 168 | assert(got.doc) 169 | equals(got.doc.points, 11) 170 | }) 171 | it('should start with blocks', async function () { 172 | const blz = [] 173 | for await (const blk of crdt.blocks.entries()) { 174 | blz.push(blk) 175 | } 176 | equals(blz.length, 25) 177 | }) 178 | it('should start with changes', async function () { 179 | const { result } = await crdt.changes() 180 | equals(result.length, 2) 181 | equals(result[0].key, 'ace') 182 | }) 183 | it('should have fewer blocks after compact', async function () { 184 | await crdt.compact() 185 | const blz = [] 186 | for await (const blk of crdt.blocks.entries()) { 187 | blz.push(blk) 188 | } 189 | equals(blz.length, 4) 190 | }) 191 | it('should have data after compact', async function () { 192 | await crdt.compact() 193 | const got = await crdt.get('ace') 194 | assert(got.doc) 195 | equals(got.doc.points, 11) 196 | }) 197 | it('should have changes after compact', async function () { 198 | const chs = await crdt.changes() 199 | equals(chs.result[0].key, 'ace') 200 | }) 201 | }) 202 | 203 | describe('CRDT with an index', function () { 204 | let crdt, idx 205 | beforeEach(async function () { 206 | crdt = new CRDT() 207 | await crdt.bulk([{ key: 'ace', value: { points: 11 } }, { key: 'king', value: { points: 10 } }]) 208 | idx = await index({ _crdt: crdt }, 'points') 209 | }) 210 | it('should query the data', async function () { 211 | const got = await idx.query({ range: [9, 12] }) 212 | equals(got.rows.length, 2) 213 | equals(got.rows[0].id, 'king') 214 | }) 215 | it('should register the index', async function () { 216 | const rIdx = await index({ _crdt: crdt }, 'points') 217 | assert(rIdx) 218 | equals(rIdx.name, 'points') 219 | const got = await rIdx.query({ range: [9, 12] }) 220 | equals(got.rows.length, 2) 221 | equals(got.rows[0].id, 'king') 222 | }) 223 | it('creating a different index with same name should not work', async function () { 224 | const e = await index({ _crdt: crdt }, 'points', (doc) => doc._id).query().catch((err) => err) 225 | matches(e.message, /cannot apply/) 226 | }) 227 | }) 228 | -------------------------------------------------------------------------------- /test/loader.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/require-await */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-assignment */ 3 | /* eslint-disable @typescript-eslint/no-unsafe-call */ 4 | /* eslint-disable @typescript-eslint/no-unsafe-member-access */ 5 | /* eslint-disable mocha/max-top-level-suites */ 6 | 7 | import * as codec from '@ipld/dag-cbor' 8 | import { sha256 as hasher } from 'multiformats/hashes/sha2' 9 | import { encode } from 'multiformats/block' 10 | import { CID } from 'multiformats/cid' 11 | 12 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 13 | import { assert, matches, equals, resetDirectory, notEquals } from './helpers.js' 14 | 15 | import { parseCarFile } from '../dist/test/loader-helpers.esm.js' 16 | 17 | import { IdxLoader, DbLoader } from '../dist/test/loader.esm.js' 18 | import { CRDT } from '../dist/test/crdt.esm.js' 19 | import { Transaction } from '../dist/test/transaction.esm.js' 20 | 21 | import { testConfig } from '../dist/test/store-fs.esm.js' 22 | import { MemoryBlockstore } from '@alanshaw/pail/block' 23 | 24 | describe('basic Loader', function () { 25 | let loader, block, t 26 | beforeEach(async function () { 27 | await resetDirectory(testConfig.dataDir, 'test-loader-commit') 28 | t = new Transaction(new MemoryBlockstore()) 29 | loader = new DbLoader('test-loader-commit', { public: true }) 30 | block = (await encode({ 31 | value: { hello: 'world' }, 32 | hasher, 33 | codec 34 | })) 35 | await t.put(block.cid, block.bytes) 36 | }) 37 | it('should have an empty car log', function () { 38 | equals(loader.carLog.length, 0) 39 | }) 40 | it('should commit', async function () { 41 | const carCid = await loader.commit(t, { head: [block.cid] }) 42 | equals(loader.carLog.length, 1) 43 | const reader = await loader.loadCar(carCid) 44 | assert(reader) 45 | const parsed = await parseCarFile(reader) 46 | assert(parsed.cars) 47 | equals(parsed.cars.length, 0) 48 | assert(parsed.head) 49 | }) 50 | }) 51 | 52 | describe('basic Loader with two commits', function () { 53 | let loader, block, block2, t, carCid 54 | beforeEach(async function () { 55 | await resetDirectory(testConfig.dataDir, 'test-loader-two-commit') 56 | t = new Transaction(new MemoryBlockstore()) 57 | loader = new DbLoader('test-loader-two-commit', { public: true }) 58 | block = (await encode({ 59 | value: { hello: 'world' }, 60 | hasher, 61 | codec 62 | })) 63 | await t.put(block.cid, block.bytes) 64 | await loader.commit(t, { head: [block.cid] }) 65 | 66 | block2 = (await encode({ 67 | value: { hello: 'universe' }, 68 | hasher, 69 | codec 70 | })) 71 | 72 | await t.put(block2.cid, block2.bytes) 73 | carCid = await loader.commit(t, { head: [block2.cid] }) 74 | }) 75 | it('should have a car log', function () { 76 | equals(loader.carLog.length, 2) 77 | }) 78 | it('should commit', async function () { 79 | const reader = await loader.loadCar(carCid) 80 | assert(reader) 81 | const parsed = await parseCarFile(reader) 82 | assert(parsed.cars) 83 | equals(parsed.compact.length, 0) 84 | equals(parsed.cars.length, 1) 85 | assert(parsed.head) 86 | }) 87 | it('should compact', async function () { 88 | const compactCid = await loader.commit(t, { head: [block2.cid] }, true) 89 | equals(loader.carLog.length, 1) 90 | 91 | const reader = await loader.loadCar(compactCid) 92 | assert(reader) 93 | const parsed = await parseCarFile(reader) 94 | assert(parsed.cars) 95 | equals(parsed.compact.length, 2) 96 | equals(parsed.cars.length, 0) 97 | assert(parsed.head) 98 | }) 99 | it('compact should erase old files', async function () { 100 | await loader.commit(t, { head: [block2.cid] }, true) 101 | // eslint-disable-next-line @typescript-eslint/no-unsafe-return 102 | const e = await loader.loadCar(carCid).catch(e => e) 103 | assert(e) 104 | matches(e.message, 'ENOENT') 105 | }) 106 | }) 107 | 108 | describe('Loader with a committed transaction', function () { 109 | /** @type {Loader} */ 110 | let loader, blockstore, crdt, done 111 | const dbname = 'test-loader' 112 | beforeEach(async function () { 113 | await resetDirectory(testConfig.dataDir, 'test-loader') 114 | crdt = new CRDT(dbname) 115 | blockstore = crdt.blocks 116 | loader = blockstore.loader 117 | done = await crdt.bulk([{ key: 'foo', value: { foo: 'bar' } }]) 118 | }) 119 | it('should have a name', function () { 120 | equals(loader.name, dbname) 121 | }) 122 | it('should commit a transaction', function () { 123 | assert(done.head) 124 | assert(done.car) 125 | equals(loader.carLog.length, 1) 126 | }) 127 | it('can load the car', async function () { 128 | const reader = await loader.loadCar(done.car) 129 | assert(reader) 130 | const parsed = await parseCarFile(reader) 131 | assert(parsed.cars) 132 | equals(parsed.cars.length, 0) 133 | assert(parsed.head) 134 | }) 135 | }) 136 | 137 | describe('Loader with two committed transactions', function () { 138 | /** @type {Loader} */ 139 | let loader, crdt, blockstore, done1, done2 140 | const dbname = 'test-loader' 141 | beforeEach(async function () { 142 | await resetDirectory(testConfig.dataDir, 'test-loader') 143 | crdt = new CRDT(dbname) 144 | blockstore = crdt.blocks 145 | loader = blockstore.loader 146 | done1 = await crdt.bulk([{ key: 'apple', value: { foo: 'bar' } }]) 147 | done2 = await crdt.bulk([{ key: 'orange', value: { foo: 'bar' } }]) 148 | }) 149 | it('should commit two transactions', function () { 150 | assert(done1.head) 151 | assert(done1.car) 152 | assert(done2.head) 153 | assert(done2.car) 154 | notEquals(done1.head, done2.head) 155 | notEquals(done1.car, done2.car) 156 | equals(blockstore.transactions.size, 2) 157 | equals(loader.carLog.length, 2) 158 | equals(loader.carLog.indexOf(done1.car), 0) 159 | equals(loader.carLog.indexOf(done2.car), 1) 160 | }) 161 | it('can load the car', async function () { 162 | const reader = await loader.loadCar(done2.car) 163 | assert(reader) 164 | const parsed = await parseCarFile(reader) 165 | assert(parsed.cars) 166 | equals(parsed.cars.length, 1) 167 | assert(parsed.head) 168 | }) 169 | }) 170 | 171 | describe('Loader with many committed transactions', function () { 172 | /** @type {Loader} */ 173 | let loader, blockstore, crdt, dones 174 | const dbname = 'test-loader' 175 | const count = 10 176 | beforeEach(async function () { 177 | await resetDirectory(testConfig.dataDir, 'test-loader') 178 | // loader = new DbLoader(dbname) 179 | crdt = new CRDT(dbname) 180 | blockstore = crdt.blocks 181 | loader = blockstore.loader 182 | dones = [] 183 | for (let i = 0; i < count; i++) { 184 | const did = await crdt.bulk([{ key: `apple${i}`, value: { foo: 'bar' } }]) 185 | dones.push(did) 186 | } 187 | }) 188 | it('should commit many transactions', function () { 189 | for (const done of dones) { 190 | assert(done.head) 191 | assert(done.car) 192 | } 193 | equals(blockstore.transactions.size, count) 194 | equals(loader.carLog.length, count) 195 | }) 196 | it('can load the car', async function () { 197 | const reader = await loader.loadCar(dones[5].car) 198 | assert(reader) 199 | const parsed = await parseCarFile(reader) 200 | assert(parsed.cars) 201 | equals(parsed.cars.length, 5) 202 | assert(parsed.head) 203 | }) 204 | }) 205 | 206 | describe('basic Loader with index commits', function () { 207 | let loader, block, t, indexerResult, cid 208 | beforeEach(async function () { 209 | await resetDirectory(testConfig.dataDir, 'test-loader-index') 210 | t = new Transaction(new MemoryBlockstore()) 211 | loader = new IdxLoader('test-loader-index', { public: true }) 212 | block = (await encode({ 213 | value: { hello: 'world' }, 214 | hasher, 215 | codec 216 | })) 217 | await t.put(block.cid, block.bytes) 218 | cid = CID.parse('bafybeia4luuns6dgymy5kau5rm7r4qzrrzg6cglpzpogussprpy42cmcn4') 219 | indexerResult = { 220 | indexes: { 221 | hello: { 222 | byId: cid, 223 | byKey: cid, 224 | head: [cid], 225 | name: 'hello', 226 | map: '(doc) => doc.hello' 227 | } 228 | } 229 | } 230 | }) 231 | it('should start with an empty car log', function () { 232 | equals(loader.carLog.length, 0) 233 | }) 234 | it('should commit the index metadata', async function () { 235 | const carCid = await loader.commit(t, indexerResult) 236 | 237 | const carLog = loader.carLog 238 | 239 | equals(carLog.length, 1) 240 | const reader = await loader.loadCar(carCid) 241 | assert(reader) 242 | const parsed = await parseCarFile(reader) 243 | assert(parsed.cars) 244 | equals(parsed.cars.length, 0) 245 | assert(parsed.indexes) 246 | assert(parsed.indexes.hello) 247 | equals(parsed.indexes.hello.map, '(doc) => doc.hello') 248 | equals(parsed.indexes.hello.name, 'hello') 249 | }) 250 | }) 251 | -------------------------------------------------------------------------------- /test/fireproof.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-return */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-call */ 3 | /* eslint-disable mocha/max-top-level-suites */ 4 | /* eslint-disable @typescript-eslint/no-unused-vars */ 5 | /* eslint-disable @typescript-eslint/no-unsafe-member-access */ 6 | /* eslint-disable @typescript-eslint/no-unsafe-assignment */ 7 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 8 | import { assert, equals, notEquals, matches, equalsJSON, resetDirectory } from './helpers.js' 9 | 10 | import { database, Database } from '../dist/test/database.esm.js' 11 | import { index, Index } from '../dist/test/index.esm.js' 12 | import { testConfig } from '../dist/test/store-fs.esm.js' 13 | 14 | describe('public API', function () { 15 | beforeEach(async function () { 16 | await resetDirectory(testConfig.dataDir, 'test-api') 17 | this.db = database('test-api') 18 | this.index = index(this.db, 'test-index', (doc) => doc.foo) 19 | this.ok = await this.db.put({ _id: 'test', foo: 'bar' }) 20 | this.doc = await this.db.get('test') 21 | this.query = await this.index.query() 22 | }) 23 | it('should have a database', function () { 24 | assert(this.db) 25 | assert(this.db instanceof Database) 26 | }) 27 | it('should have an index', function () { 28 | assert(this.index) 29 | assert(this.index instanceof Index) 30 | }) 31 | it('should put', function () { 32 | assert(this.ok) 33 | equals(this.ok.id, 'test') 34 | }) 35 | it('should get', function () { 36 | equals(this.doc.foo, 'bar') 37 | }) 38 | it('should query', function () { 39 | assert(this.query) 40 | assert(this.query.rows) 41 | equals(this.query.rows.length, 1) 42 | equals(this.query.rows[0].key, 'bar') 43 | }) 44 | }) 45 | 46 | describe('basic database', function () { 47 | /** @type {Database} */ 48 | let db 49 | beforeEach(async function () { 50 | // erase the existing test data 51 | await resetDirectory(testConfig.dataDir, 'test-basic') 52 | 53 | db = new Database('test-basic') 54 | }) 55 | it('can put with id', async function () { 56 | const ok = await db.put({ _id: 'test', foo: 'bar' }) 57 | assert(ok) 58 | equals(ok.id, 'test') 59 | }) 60 | it('can put without id', async function () { 61 | const ok = await db.put({ foo: 'bam' }) 62 | assert(ok) 63 | const got = await db.get(ok.id) 64 | equals(got.foo, 'bam') 65 | }) 66 | it('can define an index', async function () { 67 | const ok = await db.put({ _id: 'test', foo: 'bar' }) 68 | assert(ok) 69 | const idx = index(db, 'test-index', (doc) => doc.foo) 70 | const result = await idx.query() 71 | assert(result) 72 | assert(result.rows) 73 | equals(result.rows.length, 1) 74 | equals(result.rows[0].key, 'bar') 75 | }) 76 | it('can define an index with a default function', async function () { 77 | const ok = await db.put({ _id: 'test', foo: 'bar' }) 78 | assert(ok) 79 | 80 | const idx = index(db, 'foo') 81 | const result = await idx.query() 82 | assert(result) 83 | assert(result.rows) 84 | equals(result.rows.length, 1) 85 | equals(result.rows[0].key, 'bar') 86 | }) 87 | }) 88 | 89 | describe('Reopening a database', function () { 90 | /** @type {Database} */ 91 | let db 92 | beforeEach(async function () { 93 | // erase the existing test data 94 | await resetDirectory(testConfig.dataDir, 'test-reopen') 95 | 96 | db = new Database('test-reopen') 97 | const ok = await db.put({ _id: 'test', foo: 'bar' }) 98 | assert(ok) 99 | equals(ok.id, 'test') 100 | 101 | assert(db._crdt._head) 102 | equals(db._crdt._head.length, 1) 103 | }) 104 | 105 | it('should persist data', async function () { 106 | const doc = await db.get('test') 107 | equals(doc.foo, 'bar') 108 | }) 109 | 110 | it('should have the same data on reopen', async function () { 111 | const db2 = new Database('test-reopen') 112 | const doc = await db2.get('test') 113 | equals(doc.foo, 'bar') 114 | assert(db2._crdt._head) 115 | equals(db2._crdt._head.length, 1) 116 | equalsJSON(db2._crdt._head, db._crdt._head) 117 | }) 118 | 119 | it('should have a car in the car log', async function () { 120 | await db._crdt.ready 121 | assert(db._crdt.blocks.loader) 122 | assert(db._crdt.blocks.loader.carLog) 123 | equals(db._crdt.blocks.loader.carLog.length, 1) 124 | }) 125 | 126 | it('should have carlog after reopen', async function () { 127 | const db2 = new Database('test-reopen') 128 | await db2._crdt.ready 129 | assert(db2._crdt.blocks.loader) 130 | assert(db2._crdt.blocks.loader.carLog) 131 | equals(db2._crdt.blocks.loader.carLog.length, 1) 132 | }) 133 | 134 | it('faster, should have the same data on reopen after reopen and update', async function () { 135 | for (let i = 0; i < 4; i++) { 136 | const db = new Database('test-reopen') 137 | assert(db._crdt.ready) 138 | await db._crdt.ready 139 | equals(db._crdt.blocks.loader.carLog.length, i + 1) 140 | const ok = await db.put({ _id: `test${i}`, fire: 'proof'.repeat(50 * 1024) }) 141 | assert(ok) 142 | equals(db._crdt.blocks.loader.carLog.length, i + 2) 143 | const doc = await db.get(`test${i}`) 144 | equals(doc.fire, 'proof'.repeat(50 * 1024)) 145 | } 146 | }).timeout(20000) 147 | 148 | it.skip('passing slow, should have the same data on reopen after reopen and update', async function () { 149 | for (let i = 0; i < 100; i++) { 150 | // console.log('iteration', i) 151 | const db = new Database('test-reopen') 152 | assert(db._crdt.ready) 153 | await db._crdt.ready 154 | equals(db._crdt.blocks.loader.carLog.length, i + 1) 155 | const ok = await db.put({ _id: `test${i}`, fire: 'proof'.repeat(50 * 1024) }) 156 | assert(ok) 157 | equals(db._crdt.blocks.loader.carLog.length, i + 2) 158 | const doc = await db.get(`test${i}`) 159 | equals(doc.fire, 'proof'.repeat(50 * 1024)) 160 | } 161 | }).timeout(20000) 162 | }) 163 | 164 | describe('Reopening a database with indexes', function () { 165 | /** @type {Database} */ 166 | let db, idx, didMap, mapFn 167 | beforeEach(async function () { 168 | // erase the existing test data 169 | await resetDirectory(testConfig.dataDir, 'test-reopen-idx') 170 | await resetDirectory(testConfig.dataDir, 'test-reopen-idx.idx') 171 | 172 | db = database('test-reopen-idx') 173 | const ok = await db.put({ _id: 'test', foo: 'bar' }) 174 | equals(ok.id, 'test') 175 | 176 | didMap = false 177 | 178 | const mapFn = (doc) => { 179 | didMap = true 180 | return doc.foo 181 | } 182 | 183 | idx = index(db, 'foo', mapFn) 184 | }) 185 | 186 | it('should persist data', async function () { 187 | const doc = await db.get('test') 188 | equals(doc.foo, 'bar') 189 | const idx2 = index(db, 'foo') 190 | assert(idx2 === idx, 'same object') 191 | const result = await idx2.query() 192 | assert(result) 193 | assert(result.rows) 194 | equals(result.rows.length, 1) 195 | equals(result.rows[0].key, 'bar') 196 | assert(didMap) 197 | }) 198 | 199 | it('should reuse the index', async function () { 200 | const idx2 = index(db, 'foo', mapFn) 201 | assert(idx2 === idx, 'same object') 202 | const result = await idx2.query() 203 | assert(result) 204 | assert(result.rows) 205 | equals(result.rows.length, 1) 206 | equals(result.rows[0].key, 'bar') 207 | assert(didMap) 208 | didMap = false 209 | const r2 = await idx2.query() 210 | assert(r2) 211 | assert(r2.rows) 212 | equals(r2.rows.length, 1) 213 | equals(r2.rows[0].key, 'bar') 214 | assert(!didMap) 215 | }) 216 | 217 | it('should have the same data on reopen', async function () { 218 | const db2 = database('test-reopen-idx') 219 | const doc = await db2.get('test') 220 | equals(doc.foo, 'bar') 221 | assert(db2._crdt._head) 222 | equals(db2._crdt._head.length, 1) 223 | equalsJSON(db2._crdt._head, db._crdt._head) 224 | }) 225 | 226 | it('should have the same data on reopen after a query', async function () { 227 | const r0 = await idx.query() 228 | assert(r0) 229 | assert(r0.rows) 230 | equals(r0.rows.length, 1) 231 | equals(r0.rows[0].key, 'bar') 232 | 233 | const db2 = database('test-reopen-idx') 234 | const doc = await db2.get('test') 235 | equals(doc.foo, 'bar') 236 | assert(db2._crdt._head) 237 | equals(db2._crdt._head.length, 1) 238 | equalsJSON(db2._crdt._head, db._crdt._head) 239 | }) 240 | 241 | // it('should query the same data on reopen', async function () { 242 | // const r0 = await idx.query() 243 | // assert(r0) 244 | // assert(r0.rows) 245 | // equals(r0.rows.length, 1) 246 | // equals(r0.rows[0].key, 'bar') 247 | 248 | // const db2 = database('test-reopen-idx') 249 | // const d2 = await db2.get('test') 250 | // equals(d2.foo, 'bar') 251 | // didMap = false 252 | // const idx3 = db2.index('foo', mapFn) 253 | // const result = await idx3.query() 254 | // assert(result) 255 | // assert(result.rows) 256 | // equals(result.rows.length, 1) 257 | // equals(result.rows[0].key, 'bar') 258 | // assert(!didMap) 259 | // }) 260 | }) 261 | -------------------------------------------------------------------------------- /test/indexer.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-argument */ 2 | /* eslint-disable @typescript-eslint/no-unsafe-return */ 3 | /* eslint-disable @typescript-eslint/no-unsafe-call */ 4 | /* eslint-disable @typescript-eslint/no-unsafe-member-access */ 5 | /* eslint-disable @typescript-eslint/no-unsafe-assignment */ 6 | import { Index, index } from '../dist/test/index.esm.js' 7 | import { Database, database } from '../dist/test/database.esm.js' 8 | import { CRDT } from '../dist/test/crdt.esm.js' 9 | 10 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 11 | import { assert, matches, equals, resetDirectory, notEquals, equalsJSON } from './helpers.js' 12 | 13 | import { testConfig } from '../dist/test/store-fs.esm.js' 14 | 15 | describe('basic Index', function () { 16 | let db, indexer, didMap 17 | beforeEach(async function () { 18 | await resetDirectory(testConfig.dataDir, 'test-indexer') 19 | 20 | db = new Database('test-indexer') 21 | await db.put({ title: 'amazing' }) 22 | await db.put({ title: 'creative' }) 23 | await db.put({ title: 'bazillas' }) 24 | indexer = new Index(db._crdt, 'hello', (doc) => { 25 | didMap = true 26 | return doc.title 27 | }) 28 | }) 29 | it('should have properties', function () { 30 | equals(indexer.crdt, db._crdt) 31 | equals(indexer.name, 'hello') 32 | assert(indexer.mapFn) 33 | }) 34 | it('should call the map function on first query', async function () { 35 | didMap = false 36 | await indexer.query() 37 | assert(didMap) 38 | }) 39 | it('should not call the map function on second query', async function () { 40 | await indexer.query() 41 | didMap = false 42 | await indexer.query() 43 | assert(!didMap) 44 | }) 45 | it('should get results', async function () { 46 | const result = await indexer.query() 47 | assert(result) 48 | assert(result.rows) 49 | equals(result.rows.length, 3) 50 | }) 51 | it('should be in order', async function () { 52 | const { rows } = await indexer.query() 53 | equals(rows[0].key, 'amazing') 54 | }) 55 | it('should work with limit', async function () { 56 | const { rows } = await indexer.query({ limit: 1 }) 57 | equals(rows.length, 1) 58 | }) 59 | it('should work with descending', async function () { 60 | const { rows } = await indexer.query({ descending: true }) 61 | equals(rows[0].key, 'creative') 62 | }) 63 | it('should range query all', async function () { 64 | const { rows } = await indexer.query({ range: ['a', 'z'] }) 65 | equals(rows[0].key, 'amazing') 66 | equals(rows.length, 3) 67 | }) 68 | it('should range query', async function () { 69 | const { rows } = await indexer.query({ range: ['b', 'd'] }) 70 | equals(rows[0].key, 'bazillas') 71 | }) 72 | it('should key query', async function () { 73 | const { rows } = await indexer.query({ key: 'bazillas' }) 74 | equals(rows.length, 1) 75 | }) 76 | it('should include docs', async function () { 77 | const { rows } = await indexer.query({ includeDocs: true }) 78 | assert(rows[0].doc) 79 | equals(rows[0].doc._id, rows[0].id) 80 | }) 81 | }) 82 | 83 | // eslint-disable-next-line mocha/max-top-level-suites 84 | describe('Index query with compound key', function () { 85 | let db, indexer 86 | beforeEach(async function () { 87 | await resetDirectory(testConfig.dataDir, 'test-indexer') 88 | db = new Database('test-indexer') 89 | await db.put({ title: 'amazing', score: 1 }) 90 | await db.put({ title: 'creative', score: 2 }) 91 | await db.put({ title: 'creative', score: 20 }) 92 | await db.put({ title: 'bazillas', score: 3 }) 93 | indexer = new Index(db._crdt, 'hello', (doc) => { 94 | return [doc.title, doc.score] 95 | }) 96 | }) 97 | it('should prefix query', async function () { 98 | const { rows } = await indexer.query({ prefix: 'creative' }) 99 | equals(rows.length, 2) 100 | equalsJSON(rows[0].key, ['creative', 2]) 101 | equalsJSON(rows[1].key, ['creative', 20]) 102 | }) 103 | }) 104 | 105 | describe('basic Index with map fun', function () { 106 | let db, indexer 107 | beforeEach(async function () { 108 | await resetDirectory(testConfig.dataDir, 'test-indexer') 109 | 110 | db = new Database('test-indexer') 111 | await db.put({ title: 'amazing' }) 112 | await db.put({ title: 'creative' }) 113 | await db.put({ title: 'bazillas' }) 114 | indexer = new Index(db._crdt, 'hello', (doc, map) => { 115 | map(doc.title) 116 | }) 117 | }) 118 | it('should get results', async function () { 119 | const result = await indexer.query() 120 | assert(result) 121 | assert(result.rows) 122 | equals(result.rows.length, 3) 123 | }) 124 | }) 125 | 126 | describe('basic Index with string fun', function () { 127 | let db, indexer 128 | beforeEach(async function () { 129 | await resetDirectory(testConfig.dataDir, 'test-indexer') 130 | 131 | db = new Database('test-indexer') 132 | await db.put({ title: 'amazing' }) 133 | await db.put({ title: 'creative' }) 134 | await db.put({ title: 'bazillas' }) 135 | indexer = new Index(db._crdt, 'title') 136 | }) 137 | it('should get results', async function () { 138 | const result = await indexer.query() 139 | assert(result) 140 | assert(result.rows) 141 | equals(result.rows.length, 3) 142 | }) 143 | it('should include docs', async function () { 144 | const { rows } = await indexer.query() 145 | assert(rows[0].doc) 146 | }) 147 | }) 148 | 149 | describe('basic Index upon cold start', function () { 150 | let crdt, indexer, result, didMap, mapFn 151 | beforeEach(async function () { 152 | await resetDirectory(testConfig.dataDir, 'test-indexer-cold') 153 | await resetDirectory(testConfig.dataDir, 'test-indexer-cold.idx') 154 | 155 | // db = database() 156 | crdt = new CRDT('test-indexer-cold') 157 | await crdt.bulk([ 158 | { key: 'abc1', value: { title: 'amazing' } }, 159 | { key: 'abc2', value: { title: 'creative' } }, 160 | { key: 'abc3', value: { title: 'bazillas' } }]) 161 | didMap = 0 162 | mapFn = (doc) => { 163 | didMap++ 164 | return doc.title 165 | } 166 | indexer = await index({ _crdt: crdt }, 'hello', mapFn) 167 | // new Index(db._crdt.indexBlocks, db._crdt, 'hello', mapFn) 168 | result = await indexer.query() 169 | equalsJSON(indexer.indexHead, crdt._head) 170 | }) 171 | it('should call map on first query', function () { 172 | assert(didMap) 173 | equals(didMap, 3) 174 | }) 175 | it('should get results on first query', function () { 176 | assert(result) 177 | assert(result.rows) 178 | equals(result.rows.length, 3) 179 | }) 180 | it('should work on cold load', async function () { 181 | const crdt2 = new CRDT('test-indexer-cold') 182 | const { result, head } = await crdt2.changes() 183 | assert(result) 184 | await crdt2.ready 185 | const indexer2 = await index({ _crdt: crdt2 }, 'hello', mapFn) 186 | await indexer2.ready 187 | equalsJSON(indexer2.indexHead, head) 188 | const result2 = await indexer2.query() 189 | assert(result2) 190 | equals(result2.rows.length, 3) 191 | equalsJSON(indexer2.indexHead, head) 192 | }) 193 | it('should not rerun the map function on seen changes', async function () { 194 | didMap = 0 195 | const crdt2 = new CRDT('test-indexer-cold') 196 | const indexer2 = await index({ _crdt: crdt2 }, 'hello', mapFn) 197 | const { result, head } = await crdt2.changes([]) 198 | equals(result.length, 3) 199 | equals(head.length, 1) 200 | const { result: ch2, head: h2 } = await crdt2.changes(head) 201 | equals(ch2.length, 0) 202 | equals(h2.length, 1) 203 | equalsJSON(h2, head) 204 | const result2 = await indexer2.query() 205 | equalsJSON(indexer2.indexHead, head) 206 | assert(result2) 207 | equals(result2.rows.length, 3) 208 | equals(didMap, 0) 209 | await crdt2.bulk([ 210 | { key: 'abc4', value: { title: 'despicable' } }]) 211 | 212 | const { result: ch3, head: h3 } = await crdt2.changes(head) 213 | equals(ch3.length, 1) 214 | equals(h3.length, 1) 215 | const result3 = await indexer2.query() 216 | assert(result3) 217 | equals(result3.rows.length, 4) 218 | equals(didMap, 1) 219 | }) 220 | it('shouldnt allow map function definiton to change', async function () { 221 | const crdt2 = new CRDT('test-indexer-cold') 222 | const e = await index({ _crdt: crdt2 }, 'hello', (doc) => doc.title).query().catch((e) => e) 223 | matches(e.message, /cannot apply/) 224 | }) 225 | }) 226 | 227 | describe('basic Index with no data', function () { 228 | let db, indexer, didMap 229 | beforeEach(async function () { 230 | await resetDirectory(testConfig.dataDir, 'test-indexer') 231 | 232 | db = new Database('test-indexer') 233 | indexer = new Index(db._crdt, 'hello', (doc) => { 234 | didMap = true 235 | return doc.title 236 | }) 237 | }) 238 | it('should have properties', function () { 239 | equals(indexer.crdt, db._crdt) 240 | equals(indexer.name, 'hello') 241 | assert(indexer.mapFn) 242 | }) 243 | it('should not call the map function on first query', async function () { 244 | didMap = false 245 | await indexer.query() 246 | assert(!didMap) 247 | }) 248 | it('should not call the map function on second query', async function () { 249 | await indexer.query() 250 | didMap = false 251 | await indexer.query() 252 | assert(!didMap) 253 | }) 254 | it('should get results', async function () { 255 | const result = await indexer.query() 256 | assert(result) 257 | assert(result.rows) 258 | equals(result.rows.length, 0) 259 | }) 260 | }) 261 | --------------------------------------------------------------------------------