├── .gitignore ├── src ├── utils.ts ├── index.ts ├── errors.ts ├── namespace.ts ├── memory.ts ├── shard.ts ├── base.ts ├── sharding.ts ├── tiered.ts ├── mount.ts └── keytransform.ts ├── LICENSE ├── tsconfig.json ├── .github ├── dependabot.yml ├── workflows │ ├── automerge.yml │ ├── stale.yml │ └── js-test-and-release.yml ├── ISSUE_TEMPLATE │ ├── config.yml │ └── open_an_issue.md └── config.yml ├── test ├── memory.spec.ts ├── utils.spec.ts ├── keytransform.spec.ts ├── namespace.spec.ts ├── sharding.spec.ts ├── shard.spec.ts ├── tiered.spec.ts └── mount.spec.ts ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── package.json └── CHANGELOG.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | dist 4 | .docs 5 | .coverage 6 | node_modules 7 | package-lock.json 8 | yarn.lock 9 | .vscode 10 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | 2 | export const replaceStartWith = (s: string, r: string): string => { 3 | const matcher = new RegExp('^' + r) 4 | return s.replace(matcher, '') 5 | } 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This project is dual licensed under MIT and Apache-2.0. 2 | 3 | MIT: https://www.opensource.org/licenses/mit 4 | Apache-2.0: https://www.apache.org/licenses/license-2.0 5 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "10:00" 8 | open-pull-requests-limit: 10 9 | commit-message: 10 | prefix: "deps" 11 | prefix-development: "deps(dev)" 12 | -------------------------------------------------------------------------------- /.github/workflows/automerge.yml: -------------------------------------------------------------------------------- 1 | # File managed by web3-bot. DO NOT EDIT. 2 | # See https://github.com/protocol/.github/ for details. 3 | 4 | name: Automerge 5 | on: [ pull_request ] 6 | 7 | jobs: 8 | automerge: 9 | uses: protocol/.github/.github/workflows/automerge.yml@master 10 | with: 11 | job: 'automerge' 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Getting Help on IPFS 4 | url: https://ipfs.io/help 5 | about: All information about how and where to get help on IPFS. 6 | - name: IPFS Official Forum 7 | url: https://discuss.ipfs.io 8 | about: Please post general questions, support requests, and discussions here. 9 | -------------------------------------------------------------------------------- /test/memory.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { MemoryDatastore } from '../src/memory.js' 4 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 5 | 6 | describe('Memory', () => { 7 | describe('interface-datastore', () => { 8 | interfaceDatastoreTests({ 9 | setup () { 10 | return new MemoryDatastore() 11 | }, 12 | teardown () {} 13 | }) 14 | }) 15 | }) 16 | -------------------------------------------------------------------------------- /test/utils.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import * as utils from '../src/utils.js' 5 | 6 | describe('utils', () => { 7 | it('replaceStartWith', () => { 8 | expect( 9 | utils.replaceStartWith('helloworld', 'hello') 10 | ).to.eql( 11 | 'world' 12 | ) 13 | 14 | expect( 15 | utils.replaceStartWith('helloworld', 'world') 16 | ).to.eql( 17 | 'helloworld' 18 | ) 19 | }) 20 | }) 21 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 2 | 3 | http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/open_an_issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Open an issue 3 | about: Only for actionable issues relevant to this repository. 4 | title: '' 5 | labels: need/triage 6 | assignees: '' 7 | 8 | --- 9 | 20 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import type { Key } from 'interface-datastore' 2 | 3 | import * as Errors from './errors.js' 4 | import * as shard from './shard.js' 5 | 6 | export { BaseDatastore } from './base.js' 7 | export { MemoryDatastore } from './memory.js' 8 | export { KeyTransformDatastore } from './keytransform.js' 9 | export { ShardingDatastore } from './sharding.js' 10 | export { MountDatastore } from './mount.js' 11 | export { TieredDatastore } from './tiered.js' 12 | export { NamespaceDatastore } from './namespace.js' 13 | 14 | export { Errors } 15 | export { shard } 16 | 17 | export interface Shard { 18 | name: string 19 | param: number 20 | readonly _padding: string 21 | fun: (s: string) => string 22 | toString: () => string 23 | } 24 | 25 | export interface KeyTransform { 26 | convert: (key: Key) => Key 27 | invert: (key: Key) => Key 28 | } 29 | -------------------------------------------------------------------------------- /src/errors.ts: -------------------------------------------------------------------------------- 1 | import errCode from 'err-code' 2 | 3 | export function dbOpenFailedError (err?: Error): Error { 4 | err = err ?? new Error('Cannot open database') 5 | return errCode(err, 'ERR_DB_OPEN_FAILED') 6 | } 7 | 8 | export function dbDeleteFailedError (err?: Error): Error { 9 | err = err ?? new Error('Delete failed') 10 | return errCode(err, 'ERR_DB_DELETE_FAILED') 11 | } 12 | 13 | export function dbWriteFailedError (err?: Error): Error { 14 | err = err ?? new Error('Write failed') 15 | return errCode(err, 'ERR_DB_WRITE_FAILED') 16 | } 17 | 18 | export function dbReadFailedError (err?: Error): Error { 19 | err = err ?? new Error('Read failed') 20 | return errCode(err, 'ERR_DB_READ_FAILED') 21 | } 22 | 23 | export function notFoundError (err?: Error): Error { 24 | err = err ?? new Error('Not Found') 25 | return errCode(err, 'ERR_NOT_FOUND') 26 | } 27 | 28 | export function abortedError (err?: Error): Error { 29 | err = err ?? new Error('Aborted') 30 | return errCode(err, 'ERR_ABORTED') 31 | } 32 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Close and mark stale issue 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | 7 | jobs: 8 | stale: 9 | 10 | runs-on: ubuntu-latest 11 | permissions: 12 | issues: write 13 | pull-requests: write 14 | 15 | steps: 16 | - uses: actions/stale@v3 17 | with: 18 | repo-token: ${{ secrets.GITHUB_TOKEN }} 19 | stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' 20 | close-issue-message: 'This issue was closed because it is missing author input.' 21 | stale-issue-label: 'kind/stale' 22 | any-of-labels: 'need/author-input' 23 | exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' 24 | days-before-issue-stale: 6 25 | days-before-issue-close: 7 26 | enable-statistics: true 27 | -------------------------------------------------------------------------------- /src/namespace.ts: -------------------------------------------------------------------------------- 1 | import { Key } from 'interface-datastore' 2 | import type { Datastore } from 'interface-datastore' 3 | import { KeyTransformDatastore } from './keytransform.js' 4 | 5 | /** 6 | * Wraps a given datastore into a keytransform which 7 | * makes a given prefix transparent. 8 | * 9 | * For example, if the prefix is `new Key(/hello)` a call 10 | * to `store.put(new Key('/world'), mydata)` would store the data under 11 | * `/hello/world`. 12 | */ 13 | export class NamespaceDatastore extends KeyTransformDatastore { 14 | constructor (child: Datastore, prefix: Key) { 15 | super(child, { 16 | convert (key) { 17 | return prefix.child(key) 18 | }, 19 | invert (key) { 20 | if (prefix.toString() === '/') { 21 | return key 22 | } 23 | 24 | if (!prefix.isAncestorOf(key)) { 25 | throw new Error(`Expected prefix: (${prefix.toString()}) in key: ${key.toString()}`) 26 | } 27 | 28 | return new Key(key.toString().slice(prefix.toString().length), false) 29 | } 30 | }) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /src/memory.ts: -------------------------------------------------------------------------------- 1 | import { BaseDatastore } from './base.js' 2 | import { Key } from 'interface-datastore/key' 3 | import * as Errors from './errors.js' 4 | import type { Pair } from 'interface-datastore' 5 | import type { Await, AwaitIterable } from 'interface-store' 6 | 7 | export class MemoryDatastore extends BaseDatastore { 8 | private readonly data: Map 9 | 10 | constructor () { 11 | super() 12 | 13 | this.data = new Map() 14 | } 15 | 16 | put (key: Key, val: Uint8Array): Await { // eslint-disable-line require-await 17 | this.data.set(key.toString(), val) 18 | 19 | return key 20 | } 21 | 22 | get (key: Key): Await { 23 | const result = this.data.get(key.toString()) 24 | 25 | if (result == null) { 26 | throw Errors.notFoundError() 27 | } 28 | 29 | return result 30 | } 31 | 32 | has (key: Key): Await { // eslint-disable-line require-await 33 | return this.data.has(key.toString()) 34 | } 35 | 36 | delete (key: Key): Await { // eslint-disable-line require-await 37 | this.data.delete(key.toString()) 38 | } 39 | 40 | * _all (): AwaitIterable { 41 | for (const [key, value] of this.data.entries()) { 42 | yield { key: new Key(key), value } 43 | } 44 | } 45 | 46 | * _allKeys (): AwaitIterable { 47 | for (const key of this.data.keys()) { 48 | yield new Key(key) 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /test/keytransform.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import all from 'it-all' 5 | import { Key } from 'interface-datastore/key' 6 | import { MemoryDatastore } from '../src/memory.js' 7 | import { KeyTransformDatastore } from '../src/keytransform.js' 8 | 9 | describe('KeyTransformDatastore', () => { 10 | it('basic', async () => { 11 | const mStore = new MemoryDatastore() 12 | const transform = { 13 | convert (key: Key): Key { 14 | return new Key('/abc').child(key) 15 | }, 16 | invert (key: Key): Key { 17 | const l = key.list() 18 | if (l[0] !== 'abc') { 19 | throw new Error('missing prefix, convert failed?') 20 | } 21 | return Key.withNamespaces(l.slice(1)) 22 | } 23 | } 24 | 25 | const kStore = new KeyTransformDatastore(mStore, transform) 26 | 27 | const keys = [ 28 | 'foo', 29 | 'foo/bar', 30 | 'foo/bar/baz', 31 | 'foo/barb', 32 | 'foo/bar/bazb', 33 | 'foo/bar/baz/barb' 34 | ].map((s) => new Key(s)) 35 | await Promise.all(keys.map(async (key) => { await kStore.put(key, key.uint8Array()) })) 36 | const kResults = Promise.all(keys.map(async (key) => await kStore.get(key))) 37 | const mResults = Promise.all(keys.map(async (key) => await mStore.get(new Key('abc').child(key)))) 38 | const results = await Promise.all([kResults, mResults]) 39 | expect(results[0]).to.eql(results[1]) 40 | 41 | const mRes = await all(mStore.query({})) 42 | const kRes = await all(kStore.query({})) 43 | expect(kRes).to.have.length(mRes.length) 44 | 45 | mRes.forEach((a, i) => { 46 | const kA = a.key 47 | const kB = kRes[i].key 48 | expect(transform.invert(kA)).to.eql(kB) 49 | expect(kA).to.eql(transform.convert(kB)) 50 | }) 51 | }) 52 | }) 53 | -------------------------------------------------------------------------------- /test/namespace.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import all from 'it-all' 5 | import { Key } from 'interface-datastore/key' 6 | import { MemoryDatastore } from '../src/memory.js' 7 | import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' 8 | import { NamespaceDatastore } from '../src/namespace.js' 9 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 10 | 11 | describe('NamespaceDatastore', () => { 12 | const prefixes = [ 13 | 'abc', 14 | '' 15 | ] 16 | prefixes.forEach((prefix) => it(`basic '${prefix}'`, async () => { 17 | const mStore = new MemoryDatastore() 18 | const store = new NamespaceDatastore(mStore, new Key(prefix)) 19 | 20 | const keys = [ 21 | 'foo', 22 | 'foo/bar', 23 | 'foo/bar/baz', 24 | 'foo/barb', 25 | 'foo/bar/bazb', 26 | 'foo/bar/baz/barb' 27 | ].map((s) => new Key(s)) 28 | 29 | await Promise.all(keys.map(async key => { await store.put(key, uint8ArrayFromString(key.toString())) })) 30 | const nResults = Promise.all(keys.map(async (key) => await store.get(key))) 31 | const mResults = Promise.all(keys.map(async (key) => await mStore.get(new Key(prefix).child(key)))) 32 | const results = await Promise.all([nResults, mResults]) 33 | const mRes = await all(mStore.query({})) 34 | const nRes = await all(store.query({})) 35 | 36 | expect(nRes).to.have.length(mRes.length) 37 | 38 | mRes.forEach((a, i) => { 39 | const kA = a.key 40 | const kB = nRes[i].key 41 | expect(store.transform.invert(kA)).to.eql(kB) 42 | expect(kA).to.eql(store.transform.convert(kB)) 43 | }) 44 | 45 | expect(results[0]).to.eql(results[1]) 46 | })) 47 | 48 | prefixes.forEach((prefix) => { 49 | describe(`interface-datastore: '${prefix}'`, () => { 50 | interfaceDatastoreTests({ 51 | setup () { 52 | return new NamespaceDatastore(new MemoryDatastore(), new Key(prefix)) 53 | }, 54 | async teardown () { } 55 | }) 56 | }) 57 | }) 58 | }) 59 | -------------------------------------------------------------------------------- /test/sharding.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { Key } from 'interface-datastore/key' 5 | import { MemoryDatastore } from '../src/memory.js' 6 | import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' 7 | import { toString as uint8ArrayToString } from 'uint8arrays/to-string' 8 | import { 9 | NextToLast, 10 | SHARDING_FN 11 | } from '../src/shard.js' 12 | import { 13 | ShardingDatastore 14 | } from '../src/sharding.js' 15 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 16 | 17 | describe('ShardingDatastore', () => { 18 | it('create', async () => { 19 | const ms = new MemoryDatastore() 20 | const shard = new NextToLast(2) 21 | const store = new ShardingDatastore(ms, shard) 22 | await store.open() 23 | const res = await Promise.all([ 24 | store.get(new Key(SHARDING_FN)) 25 | ]) 26 | expect(uint8ArrayToString(res[0])).to.eql(shard.toString() + '\n') 27 | }) 28 | 29 | it('open - empty', () => { 30 | const ms = new MemoryDatastore() 31 | // @ts-expect-error 32 | const store = new ShardingDatastore(ms) 33 | return expect(store.open()) 34 | .to.eventually.be.rejected() 35 | .with.property('code', 'ERR_DB_OPEN_FAILED') 36 | }) 37 | 38 | it('open - existing', () => { 39 | const ms = new MemoryDatastore() 40 | const shard = new NextToLast(2) 41 | const store = new ShardingDatastore(ms, shard) 42 | 43 | return expect(store.open()).to.eventually.be.fulfilled() 44 | }) 45 | 46 | it('basics', async () => { 47 | const ms = new MemoryDatastore() 48 | const shard = new NextToLast(2) 49 | const store = new ShardingDatastore(ms, shard) 50 | await store.open() 51 | await store.put(new Key('hello'), uint8ArrayFromString('test')) 52 | const res = await ms.get(new Key('ll').child(new Key('hello'))) 53 | expect(res).to.eql(uint8ArrayFromString('test')) 54 | }) 55 | 56 | describe('interface-datastore', () => { 57 | interfaceDatastoreTests({ 58 | setup () { 59 | const shard = new NextToLast(2) 60 | return new ShardingDatastore(new MemoryDatastore(), shard) 61 | }, 62 | teardown () { } 63 | }) 64 | }) 65 | }) 66 | -------------------------------------------------------------------------------- /test/shard.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { 5 | Prefix, 6 | Suffix, 7 | NextToLast, 8 | parseShardFun 9 | } from '../src/shard.js' 10 | 11 | describe('shard', () => { 12 | it('prefix', () => { 13 | expect( 14 | new Prefix(2).fun('hello') 15 | ).to.eql( 16 | 'he' 17 | ) 18 | expect( 19 | new Prefix(2).fun('h') 20 | ).to.eql( 21 | 'h_' 22 | ) 23 | 24 | expect( 25 | new Prefix(2).toString() 26 | ).to.eql( 27 | '/repo/flatfs/shard/v1/prefix/2' 28 | ) 29 | }) 30 | 31 | it('suffix', () => { 32 | expect( 33 | new Suffix(2).fun('hello') 34 | ).to.eql( 35 | 'lo' 36 | ) 37 | expect( 38 | new Suffix(2).fun('h') 39 | ).to.eql( 40 | '_h' 41 | ) 42 | 43 | expect( 44 | new Suffix(2).toString() 45 | ).to.eql( 46 | '/repo/flatfs/shard/v1/suffix/2' 47 | ) 48 | }) 49 | 50 | it('next-to-last', () => { 51 | expect( 52 | new NextToLast(2).fun('hello') 53 | ).to.eql( 54 | 'll' 55 | ) 56 | expect( 57 | new NextToLast(3).fun('he') 58 | ).to.eql( 59 | '__h' 60 | ) 61 | 62 | expect( 63 | new NextToLast(2).toString() 64 | ).to.eql( 65 | '/repo/flatfs/shard/v1/next-to-last/2' 66 | ) 67 | }) 68 | }) 69 | 70 | describe('parsesShardFun', () => { 71 | it('errors', () => { 72 | const errors = [ 73 | '', 74 | 'shard/v1/next-to-last/2', 75 | '/repo/flatfs/shard/v2/next-to-last/2', 76 | '/repo/flatfs/shard/v1/other/2', 77 | '/repo/flatfs/shard/v1/next-to-last/' 78 | ] 79 | 80 | errors.forEach((input) => { 81 | expect( 82 | () => parseShardFun(input) 83 | ).to.throw() 84 | }) 85 | }) 86 | 87 | it('success', () => { 88 | const success = [ 89 | 'prefix', 90 | 'suffix', 91 | 'next-to-last' 92 | ] 93 | 94 | success.forEach((name) => { 95 | const n = Math.floor(Math.random() * 100) 96 | expect( 97 | parseShardFun( 98 | `/repo/flatfs/shard/v1/${name}/${n}` 99 | ).name 100 | ).to.eql(name) 101 | }) 102 | }) 103 | }) 104 | -------------------------------------------------------------------------------- /test/tiered.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { Key } from 'interface-datastore/key' 5 | import { MemoryDatastore } from '../src/memory.js' 6 | import { TieredDatastore } from '../src/tiered.js' 7 | import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' 8 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 9 | import type { Datastore } from 'interface-datastore' 10 | 11 | /** 12 | * @typedef {import('interface-datastore').Datastore} Datastore 13 | */ 14 | 15 | describe('Tiered', () => { 16 | describe('all stores', () => { 17 | const ms: Datastore[] = [] 18 | let store: TieredDatastore 19 | beforeEach(() => { 20 | ms.push(new MemoryDatastore()) 21 | ms.push(new MemoryDatastore()) 22 | store = new TieredDatastore(ms) 23 | }) 24 | 25 | it('put', async () => { 26 | const k = new Key('hello') 27 | const v = uint8ArrayFromString('world') 28 | await store.put(k, v) 29 | const res = await Promise.all([ms[0].get(k), ms[1].get(k)]) 30 | res.forEach((val) => { 31 | expect(val).to.be.eql(v) 32 | }) 33 | }) 34 | 35 | it('get and has, where available', async () => { 36 | const k = new Key('hello') 37 | const v = uint8ArrayFromString('world') 38 | await ms[1].put(k, v) 39 | const val = await store.get(k) 40 | expect(val).to.be.eql(v) 41 | const exists = await store.has(k) 42 | expect(exists).to.be.eql(true) 43 | }) 44 | 45 | it('has - key not found', async () => { 46 | expect(await store.has(new Key('hello1'))).to.be.eql(false) 47 | }) 48 | 49 | it('has and delete', async () => { 50 | const k = new Key('hello') 51 | const v = uint8ArrayFromString('world') 52 | await store.put(k, v) 53 | let res = await Promise.all([ms[0].has(k), ms[1].has(k)]) 54 | expect(res).to.be.eql([true, true]) 55 | await store.delete(k) 56 | res = await Promise.all([ms[0].has(k), ms[1].has(k)]) 57 | expect(res).to.be.eql([false, false]) 58 | }) 59 | }) 60 | 61 | describe('inteface-datastore-single', () => { 62 | interfaceDatastoreTests({ 63 | setup () { 64 | return new TieredDatastore([ 65 | new MemoryDatastore(), 66 | new MemoryDatastore() 67 | ]) 68 | }, 69 | teardown () { } 70 | }) 71 | }) 72 | }) 73 | -------------------------------------------------------------------------------- /.github/config.yml: -------------------------------------------------------------------------------- 1 | # Configuration for welcome - https://github.com/behaviorbot/welcome 2 | 3 | # Configuration for new-issue-welcome - https://github.com/behaviorbot/new-issue-welcome 4 | # Comment to be posted to on first time issues 5 | newIssueWelcomeComment: > 6 | Thank you for submitting your first issue to this repository! A maintainer 7 | will be here shortly to triage and review. 8 | 9 | In the meantime, please double-check that you have provided all the 10 | necessary information to make this process easy! Any information that can 11 | help save additional round trips is useful! We currently aim to give 12 | initial feedback within **two business days**. If this does not happen, feel 13 | free to leave a comment. 14 | 15 | Please keep an eye on how this issue will be labeled, as labels give an 16 | overview of priorities, assignments and additional actions requested by the 17 | maintainers: 18 | 19 | - "Priority" labels will show how urgent this is for the team. 20 | - "Status" labels will show if this is ready to be worked on, blocked, or in progress. 21 | - "Need" labels will indicate if additional input or analysis is required. 22 | 23 | Finally, remember to use https://discuss.ipfs.io if you just need general 24 | support. 25 | 26 | # Configuration for new-pr-welcome - https://github.com/behaviorbot/new-pr-welcome 27 | # Comment to be posted to on PRs from first time contributors in your repository 28 | newPRWelcomeComment: > 29 | Thank you for submitting this PR! 30 | 31 | A maintainer will be here shortly to review it. 32 | 33 | We are super grateful, but we are also overloaded! Help us by making sure 34 | that: 35 | 36 | * The context for this PR is clear, with relevant discussion, decisions 37 | and stakeholders linked/mentioned. 38 | 39 | * Your contribution itself is clear (code comments, self-review for the 40 | rest) and in its best form. Follow the [code contribution 41 | guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md#code-contribution-guidelines) 42 | if they apply. 43 | 44 | Getting other community members to do a review would be great help too on 45 | complex PRs (you can ask in the chats/forums). If you are unsure about 46 | something, just leave us a comment. 47 | 48 | Next steps: 49 | 50 | * A maintainer will triage and assign priority to this PR, commenting on 51 | any missing things and potentially assigning a reviewer for high 52 | priority items. 53 | 54 | * The PR gets reviews, discussed and approvals as needed. 55 | 56 | * The PR is merged by maintainers when it has been approved and comments addressed. 57 | 58 | We currently aim to provide initial feedback/triaging within **two business 59 | days**. Please keep an eye on any labelling actions, as these will indicate 60 | priorities and status of your contribution. 61 | 62 | We are very grateful for your contribution! 63 | 64 | 65 | # Configuration for first-pr-merge - https://github.com/behaviorbot/first-pr-merge 66 | # Comment to be posted to on pull requests merged by a first time user 67 | # Currently disabled 68 | #firstPRMergeComment: "" 69 | -------------------------------------------------------------------------------- /src/shard.ts: -------------------------------------------------------------------------------- 1 | import type { Datastore } from 'interface-datastore' 2 | import { Key } from 'interface-datastore/key' 3 | import type { Shard } from './index.js' 4 | 5 | export const PREFIX = '/repo/flatfs/shard/' 6 | export const SHARDING_FN = 'SHARDING' 7 | 8 | export class ShardBase implements Shard { 9 | public param: number 10 | public name: string 11 | public _padding: string 12 | 13 | constructor (param: number) { 14 | this.param = param 15 | this.name = 'base' 16 | this._padding = '' 17 | } 18 | 19 | fun (s: string): string { 20 | return 'implement me' 21 | } 22 | 23 | toString (): string { 24 | return `${PREFIX}v1/${this.name}/${this.param}` 25 | } 26 | } 27 | 28 | export class Prefix extends ShardBase { 29 | constructor (prefixLen: number) { 30 | super(prefixLen) 31 | this._padding = ''.padStart(prefixLen, '_') 32 | this.name = 'prefix' 33 | } 34 | 35 | fun (noslash: string): string { 36 | return (noslash + this._padding).slice(0, this.param) 37 | } 38 | } 39 | 40 | export class Suffix extends ShardBase { 41 | constructor (suffixLen: number) { 42 | super(suffixLen) 43 | 44 | this._padding = ''.padStart(suffixLen, '_') 45 | this.name = 'suffix' 46 | } 47 | 48 | fun (noslash: string): string { 49 | const s = this._padding + noslash 50 | return s.slice(s.length - this.param) 51 | } 52 | } 53 | 54 | export class NextToLast extends ShardBase { 55 | constructor (suffixLen: number) { 56 | super(suffixLen) 57 | this._padding = ''.padStart(suffixLen + 1, '_') 58 | this.name = 'next-to-last' 59 | } 60 | 61 | fun (noslash: string): string { 62 | const s = this._padding + noslash 63 | const offset = s.length - this.param - 1 64 | return s.slice(offset, offset + this.param) 65 | } 66 | } 67 | 68 | /** 69 | * Convert a given string to the matching sharding function 70 | */ 71 | export function parseShardFun (str: string): Shard { 72 | str = str.trim() 73 | 74 | if (str.length === 0) { 75 | throw new Error('empty shard string') 76 | } 77 | 78 | if (!str.startsWith(PREFIX)) { 79 | throw new Error(`invalid or no path prefix: ${str}`) 80 | } 81 | 82 | const parts = str.slice(PREFIX.length).split('/') 83 | const version = parts[0] 84 | 85 | if (version !== 'v1') { 86 | throw new Error(`expect 'v1' version, got '${version}'`) 87 | } 88 | 89 | const name = parts[1] 90 | 91 | if (parts[2] == null || parts[2] === '') { 92 | throw new Error('missing param') 93 | } 94 | 95 | const param = parseInt(parts[2], 10) 96 | 97 | switch (name) { 98 | case 'prefix': 99 | return new Prefix(param) 100 | case 'suffix': 101 | return new Suffix(param) 102 | case 'next-to-last': 103 | return new NextToLast(param) 104 | default: 105 | throw new Error(`unkown sharding function: ${name}`) 106 | } 107 | } 108 | 109 | export const readShardFun = async (path: string | Uint8Array, store: Datastore): Promise => { 110 | const key = new Key(path).child(new Key(SHARDING_FN)) 111 | // @ts-expect-error 112 | const get = typeof store.getRaw === 'function' ? store.getRaw.bind(store) : store.get.bind(store) 113 | const res = await get(key) 114 | return parseShardFun(new TextDecoder().decode(res ?? '').trim()) 115 | } 116 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ⛔️ This module is now part of https://github.com/ipfs/js-stores 2 | 3 | # datastore-core 4 | 5 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 6 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 7 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-datastore-core.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-datastore-core) 8 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-datastore-core/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/ipfs/js-datastore-core/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) 9 | 10 | > Wrapper implementation for interface-datastore 11 | 12 | ## Table of contents 13 | 14 | - [Install](#install) 15 | - [Browser ` 36 | ``` 37 | 38 | ## Implementations 39 | 40 | - Wrapper Implementations 41 | - Mount: [`src/mount`](src/mount.js) 42 | - Keytransform: [`src/keytransform`](src/keytransform.js) 43 | - Sharding: [`src/sharding`](src/sharding.js) 44 | - Tiered: [`src/tiered`](src/tirered.js) 45 | - Namespace: [`src/namespace`](src/namespace.js) 46 | 47 | ## Usage 48 | 49 | ### BaseDatastore 50 | 51 | An base store is made available to make implementing your own datastore easier: 52 | 53 | ```javascript 54 | import { BaseDatastore } from 'datastore-core' 55 | 56 | class MyDatastore extends BaseDatastore { 57 | constructor () { 58 | super() 59 | } 60 | 61 | async put (key, val) { 62 | // your implementation here 63 | } 64 | 65 | async get (key) { 66 | // your implementation here 67 | } 68 | 69 | // etc... 70 | } 71 | ``` 72 | 73 | See the [MemoryDatastore](./src/memory.js) for an example of how it is used. 74 | 75 | ### Wrapping Stores 76 | 77 | ```js 78 | import { Key } from 'interface-datastore' 79 | import { 80 | MemoryStore, 81 | MountStore 82 | } from 'datastore-core' 83 | 84 | const store = new MountStore({prefix: new Key('/a'), datastore: new MemoryStore()}) 85 | ``` 86 | 87 | ## Contribute 88 | 89 | Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/js-ipfs-unixfs-importer/issues)! 90 | 91 | This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 92 | 93 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 94 | 95 | ## License 96 | 97 | Licensed under either of 98 | 99 | - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) 100 | - MIT ([LICENSE-MIT](LICENSE-MIT) / ) 101 | 102 | ## Contribute 103 | 104 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-datastore-core/issues). 105 | 106 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 107 | 108 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 109 | 110 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 111 | 112 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 113 | -------------------------------------------------------------------------------- /src/base.ts: -------------------------------------------------------------------------------- 1 | import sort from 'it-sort' 2 | import drain from 'it-drain' 3 | import filter from 'it-filter' 4 | import take from 'it-take' 5 | import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' 6 | import type { AbortOptions, Await, AwaitIterable } from 'interface-store' 7 | 8 | export class BaseDatastore implements Datastore { 9 | put (key: Key, val: Uint8Array, options?: AbortOptions): Await { 10 | return Promise.reject(new Error('.put is not implemented')) 11 | } 12 | 13 | get (key: Key, options?: AbortOptions): Await { 14 | return Promise.reject(new Error('.get is not implemented')) 15 | } 16 | 17 | has (key: Key, options?: AbortOptions): Await { 18 | return Promise.reject(new Error('.has is not implemented')) 19 | } 20 | 21 | delete (key: Key, options?: AbortOptions): Await { 22 | return Promise.reject(new Error('.delete is not implemented')) 23 | } 24 | 25 | async * putMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { 26 | for await (const { key, value } of source) { 27 | await this.put(key, value, options) 28 | yield key 29 | } 30 | } 31 | 32 | async * getMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { 33 | for await (const key of source) { 34 | yield { 35 | key, 36 | value: await this.get(key, options) 37 | } 38 | } 39 | } 40 | 41 | async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { 42 | for await (const key of source) { 43 | await this.delete(key, options) 44 | yield key 45 | } 46 | } 47 | 48 | batch (): Batch { 49 | let puts: Pair[] = [] 50 | let dels: Key[] = [] 51 | 52 | return { 53 | put (key, value) { 54 | puts.push({ key, value }) 55 | }, 56 | 57 | delete (key) { 58 | dels.push(key) 59 | }, 60 | commit: async (options) => { 61 | await drain(this.putMany(puts, options)) 62 | puts = [] 63 | await drain(this.deleteMany(dels, options)) 64 | dels = [] 65 | } 66 | } 67 | } 68 | 69 | /** 70 | * Extending classes should override `query` or implement this method 71 | */ 72 | // eslint-disable-next-line require-yield 73 | async * _all (q: Query, options?: AbortOptions): AwaitIterable { 74 | throw new Error('._all is not implemented') 75 | } 76 | 77 | /** 78 | * Extending classes should override `queryKeys` or implement this method 79 | */ 80 | // eslint-disable-next-line require-yield 81 | async * _allKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { 82 | throw new Error('._allKeys is not implemented') 83 | } 84 | 85 | query (q: Query, options?: AbortOptions): AwaitIterable { 86 | let it = this._all(q, options) 87 | 88 | if (q.prefix != null) { 89 | const prefix = q.prefix 90 | it = filter(it, (e) => e.key.toString().startsWith(prefix)) 91 | } 92 | 93 | if (Array.isArray(q.filters)) { 94 | it = q.filters.reduce((it, f) => filter(it, f), it) 95 | } 96 | 97 | if (Array.isArray(q.orders)) { 98 | it = q.orders.reduce((it, f) => sort(it, f), it) 99 | } 100 | 101 | if (q.offset != null) { 102 | let i = 0 103 | const offset = q.offset 104 | it = filter(it, () => i++ >= offset) 105 | } 106 | 107 | if (q.limit != null) { 108 | it = take(it, q.limit) 109 | } 110 | 111 | return it 112 | } 113 | 114 | queryKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { 115 | let it = this._allKeys(q, options) 116 | 117 | if (q.prefix != null) { 118 | const prefix = q.prefix 119 | it = filter(it, (key) => 120 | key.toString().startsWith(prefix) 121 | ) 122 | } 123 | 124 | if (Array.isArray(q.filters)) { 125 | it = q.filters.reduce((it, f) => filter(it, f), it) 126 | } 127 | 128 | if (Array.isArray(q.orders)) { 129 | it = q.orders.reduce((it, f) => sort(it, f), it) 130 | } 131 | 132 | if (q.offset != null) { 133 | const offset = q.offset 134 | let i = 0 135 | it = filter(it, () => i++ >= offset) 136 | } 137 | 138 | if (q.limit != null) { 139 | it = take(it, q.limit) 140 | } 141 | 142 | return it 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /src/sharding.ts: -------------------------------------------------------------------------------- 1 | import { Batch, Key, KeyQuery, KeyQueryFilter, Pair, Query, QueryFilter } from 'interface-datastore' 2 | import { 3 | readShardFun, 4 | SHARDING_FN 5 | } from './shard.js' 6 | import { BaseDatastore } from './base.js' 7 | import { KeyTransformDatastore } from './keytransform.js' 8 | import * as Errors from './errors.js' 9 | import type { Shard } from './index.js' 10 | import type { Datastore } from 'interface-datastore' 11 | import type { AbortOptions, AwaitIterable } from 'interface-store' 12 | 13 | const shardKey = new Key(SHARDING_FN) 14 | 15 | /** 16 | * Backend independent abstraction of go-ds-flatfs. 17 | * 18 | * Wraps another datastore such that all values are stored 19 | * sharded according to the given sharding function. 20 | */ 21 | export class ShardingDatastore extends BaseDatastore { 22 | private readonly child: KeyTransformDatastore 23 | private shard: Shard 24 | 25 | constructor (store: Datastore, shard: Shard) { 26 | super() 27 | 28 | this.child = new KeyTransformDatastore(store, { 29 | convert: this._convertKey.bind(this), 30 | invert: this._invertKey.bind(this) 31 | }) 32 | this.shard = shard 33 | } 34 | 35 | async open (): Promise { 36 | this.shard = await ShardingDatastore.create(this.child, this.shard) 37 | } 38 | 39 | _convertKey (key: Key): Key { 40 | const s = key.toString() 41 | if (s === shardKey.toString()) { 42 | return key 43 | } 44 | 45 | const parent = new Key(this.shard.fun(s)) 46 | return parent.child(key) 47 | } 48 | 49 | _invertKey (key: Key): Key { 50 | const s = key.toString() 51 | if (s === shardKey.toString()) { 52 | return key 53 | } 54 | return Key.withNamespaces(key.list().slice(1)) 55 | } 56 | 57 | static async create (store: Datastore, shard?: Shard): Promise { 58 | const hasShard = await store.has(shardKey) 59 | 60 | if (!hasShard) { 61 | if (shard == null) { 62 | throw Errors.dbOpenFailedError(Error('Shard is required when datastore doesn\'t have a shard key already.')) 63 | } 64 | 65 | await store.put(shardKey, new TextEncoder().encode(shard.toString() + '\n')) 66 | } 67 | 68 | if (shard == null) { 69 | shard = await readShardFun('/', store) 70 | } 71 | 72 | // test shards 73 | const diskShard = await readShardFun('/', store) 74 | const a = diskShard.toString() 75 | const b = shard.toString() 76 | 77 | if (a !== b) { 78 | throw new Error(`specified fun ${b} does not match repo shard fun ${a}`) 79 | } 80 | 81 | return diskShard 82 | } 83 | 84 | async put (key: Key, val: Uint8Array, options?: AbortOptions): Promise { 85 | await this.child.put(key, val, options) 86 | 87 | return key 88 | } 89 | 90 | async get (key: Key, options?: AbortOptions): Promise { 91 | return await this.child.get(key, options) 92 | } 93 | 94 | async has (key: Key, options?: AbortOptions): Promise { 95 | return await this.child.has(key, options) 96 | } 97 | 98 | async delete (key: Key, options?: AbortOptions): Promise { 99 | await this.child.delete(key, options) 100 | } 101 | 102 | async * putMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 103 | yield * this.child.putMany(source, options) 104 | } 105 | 106 | async * getMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 107 | yield * this.child.getMany(source, options) 108 | } 109 | 110 | async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 111 | yield * this.child.deleteMany(source, options) 112 | } 113 | 114 | batch (): Batch { 115 | return this.child.batch() 116 | } 117 | 118 | query (q: Query, options?: AbortOptions): AsyncIterable { 119 | const omitShard: QueryFilter = ({ key }) => key.toString() !== shardKey.toString() 120 | 121 | const tq: Query = { 122 | ...q, 123 | filters: [ 124 | omitShard 125 | ].concat(q.filters ?? []) 126 | } 127 | 128 | return this.child.query(tq, options) 129 | } 130 | 131 | queryKeys (q: KeyQuery, options?: AbortOptions): AsyncIterable { 132 | const omitShard: KeyQueryFilter = (key) => key.toString() !== shardKey.toString() 133 | 134 | const tq: KeyQuery = { 135 | ...q, 136 | filters: [ 137 | omitShard 138 | ].concat(q.filters ?? []) 139 | } 140 | 141 | return this.child.queryKeys(tq, options) 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/tiered.ts: -------------------------------------------------------------------------------- 1 | import { BaseDatastore } from './base.js' 2 | import * as Errors from './errors.js' 3 | import { logger } from '@libp2p/logger' 4 | import { pushable } from 'it-pushable' 5 | import drain from 'it-drain' 6 | import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' 7 | import type { AbortOptions, AwaitIterable } from 'interface-store' 8 | 9 | const log = logger('datastore:core:tiered') 10 | 11 | /** 12 | * A datastore that can combine multiple stores. Puts and deletes 13 | * will write through to all datastores. Has and get will 14 | * try each store sequentially. Query will always try the 15 | * last one first. 16 | * 17 | */ 18 | export class TieredDatastore extends BaseDatastore { 19 | private readonly stores: Datastore[] 20 | 21 | constructor (stores: Datastore[]) { 22 | super() 23 | 24 | this.stores = stores.slice() 25 | } 26 | 27 | async put (key: Key, value: Uint8Array, options?: AbortOptions): Promise { 28 | try { 29 | await Promise.all(this.stores.map(async store => { await store.put(key, value, options) })) 30 | return key 31 | } catch (err: any) { 32 | throw Errors.dbWriteFailedError(err) 33 | } 34 | } 35 | 36 | async get (key: Key, options?: AbortOptions): Promise { 37 | for (const store of this.stores) { 38 | try { 39 | const res = await store.get(key, options) 40 | if (res != null) return res 41 | } catch (err) { 42 | log.error(err) 43 | } 44 | } 45 | throw Errors.notFoundError() 46 | } 47 | 48 | async has (key: Key, options?: AbortOptions): Promise { 49 | for (const s of this.stores) { 50 | if (await s.has(key, options)) { 51 | return true 52 | } 53 | } 54 | 55 | return false 56 | } 57 | 58 | async delete (key: Key, options?: AbortOptions): Promise { 59 | try { 60 | await Promise.all(this.stores.map(async store => { await store.delete(key, options) })) 61 | } catch (err: any) { 62 | throw Errors.dbDeleteFailedError(err) 63 | } 64 | } 65 | 66 | async * putMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 67 | let error: Error | undefined 68 | const pushables = this.stores.map(store => { 69 | const source = pushable({ 70 | objectMode: true 71 | }) 72 | 73 | drain(store.putMany(source, options)) 74 | .catch(err => { 75 | // store threw while putting, make sure we bubble the error up 76 | error = err 77 | }) 78 | 79 | return source 80 | }) 81 | 82 | try { 83 | for await (const pair of source) { 84 | if (error != null) { 85 | throw error 86 | } 87 | 88 | pushables.forEach(p => p.push(pair)) 89 | 90 | yield pair.key 91 | } 92 | } finally { 93 | pushables.forEach(p => p.end()) 94 | } 95 | } 96 | 97 | async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 98 | let error: Error | undefined 99 | const pushables = this.stores.map(store => { 100 | const source = pushable({ 101 | objectMode: true 102 | }) 103 | 104 | drain(store.deleteMany(source, options)) 105 | .catch(err => { 106 | // store threw while deleting, make sure we bubble the error up 107 | error = err 108 | }) 109 | 110 | return source 111 | }) 112 | 113 | try { 114 | for await (const key of source) { 115 | if (error != null) { 116 | throw error 117 | } 118 | 119 | pushables.forEach(p => p.push(key)) 120 | 121 | yield key 122 | } 123 | } finally { 124 | pushables.forEach(p => p.end()) 125 | } 126 | } 127 | 128 | batch (): Batch { 129 | const batches = this.stores.map(store => store.batch()) 130 | 131 | return { 132 | put: (key, value) => { 133 | batches.forEach(b => { b.put(key, value) }) 134 | }, 135 | delete: (key) => { 136 | batches.forEach(b => { b.delete(key) }) 137 | }, 138 | commit: async (options) => { 139 | for (const batch of batches) { 140 | await batch.commit(options) 141 | } 142 | } 143 | } 144 | } 145 | 146 | query (q: Query, options?: AbortOptions): AwaitIterable { 147 | return this.stores[this.stores.length - 1].query(q, options) 148 | } 149 | 150 | queryKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { 151 | return this.stores[this.stores.length - 1].queryKeys(q, options) 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /test/mount.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | /* eslint max-nested-callbacks: ["error", 8] */ 3 | 4 | import { expect, assert } from 'aegir/chai' 5 | import all from 'it-all' 6 | import { Key } from 'interface-datastore/key' 7 | import { MemoryDatastore } from '../src/memory.js' 8 | import { MountDatastore } from '../src/mount.js' 9 | import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' 10 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 11 | import { KeyTransformDatastore } from '../src/keytransform.js' 12 | import type { Datastore } from 'interface-datastore' 13 | 14 | const stripPrefixDatastore = (datastore: Datastore, prefix: Key): Datastore => { 15 | return new KeyTransformDatastore( 16 | datastore, { 17 | convert: (key) => { 18 | if (!prefix.isAncestorOf(key)) { 19 | throw new Error(`Expected prefix: (${prefix.toString()}) in key: ${key.toString()}`) 20 | } 21 | 22 | return Key.withNamespaces(key.namespaces().slice(prefix.namespaces().length)) 23 | }, 24 | invert: (key) => Key.withNamespaces([...prefix.namespaces(), ...key.namespaces()]) 25 | }) 26 | } 27 | 28 | describe('MountDatastore', () => { 29 | it('put - no mount', async () => { 30 | const m = new MountDatastore([]) 31 | try { 32 | await m.put(new Key('hello'), uint8ArrayFromString('foo')) 33 | assert(false, 'Failed to throw error on no mount') 34 | } catch (err) { 35 | expect(err).to.be.an('Error') 36 | } 37 | }) 38 | 39 | it('put - wrong mount', async () => { 40 | const m = new MountDatastore([{ 41 | datastore: stripPrefixDatastore(new MemoryDatastore(), new Key('cool')), 42 | prefix: new Key('cool') 43 | }]) 44 | try { 45 | await m.put(new Key('/fail/hello'), uint8ArrayFromString('foo')) 46 | assert(false, 'Failed to throw error on wrong mount') 47 | } catch (err) { 48 | expect(err).to.be.an('Error') 49 | } 50 | }) 51 | 52 | it('put', async () => { 53 | const mds = new MemoryDatastore() 54 | const m = new MountDatastore([{ 55 | datastore: stripPrefixDatastore(mds, new Key('cool')), 56 | prefix: new Key('cool') 57 | }]) 58 | 59 | const val = uint8ArrayFromString('hello') 60 | await m.put(new Key('/cool/hello'), val) 61 | const res = await mds.get(new Key('/hello')) 62 | expect(res).to.eql(val) 63 | }) 64 | 65 | it('get', async () => { 66 | const mds = new MemoryDatastore() 67 | const m = new MountDatastore([{ 68 | datastore: stripPrefixDatastore(mds, new Key('cool')), 69 | prefix: new Key('cool') 70 | }]) 71 | 72 | const val = uint8ArrayFromString('hello') 73 | await mds.put(new Key('/hello'), val) 74 | const res = await m.get(new Key('/cool/hello')) 75 | expect(res).to.eql(val) 76 | }) 77 | 78 | it('has', async () => { 79 | const mds = new MemoryDatastore() 80 | const m = new MountDatastore([{ 81 | datastore: stripPrefixDatastore(mds, new Key('cool')), 82 | prefix: new Key('cool') 83 | }]) 84 | 85 | const val = uint8ArrayFromString('hello') 86 | await mds.put(new Key('/hello'), val) 87 | const exists = await m.has(new Key('/cool/hello')) 88 | expect(exists).to.eql(true) 89 | }) 90 | 91 | it('delete', async () => { 92 | const mds = new MemoryDatastore() 93 | const m = new MountDatastore([{ 94 | datastore: stripPrefixDatastore(mds, new Key('cool')), 95 | prefix: new Key('cool') 96 | }]) 97 | 98 | const val = uint8ArrayFromString('hello') 99 | await m.put(new Key('/cool/hello'), val) 100 | await m.delete(new Key('/cool/hello')) 101 | let exists = await m.has(new Key('/cool/hello')) 102 | expect(exists).to.eql(false) 103 | exists = await mds.has(new Key('/hello')) 104 | expect(exists).to.eql(false) 105 | }) 106 | 107 | it('query simple', async () => { 108 | const mds = new MemoryDatastore() 109 | const m = new MountDatastore([{ 110 | datastore: stripPrefixDatastore(mds, new Key('cool')), 111 | prefix: new Key('cool') 112 | }]) 113 | 114 | const val = uint8ArrayFromString('hello') 115 | await m.put(new Key('/cool/hello'), val) 116 | const res = await all(m.query({ prefix: '/cool' })) 117 | expect(res).to.eql([{ key: new Key('/cool/hello'), value: val }]) 118 | }) 119 | 120 | describe('interface-datastore', () => { 121 | interfaceDatastoreTests({ 122 | setup () { 123 | return new MountDatastore([{ 124 | prefix: new Key('/a'), 125 | datastore: new MemoryDatastore() 126 | }, { 127 | prefix: new Key('/z'), 128 | datastore: new MemoryDatastore() 129 | }, { 130 | prefix: new Key('/q'), 131 | datastore: new MemoryDatastore() 132 | }]) 133 | }, 134 | teardown () { } 135 | }) 136 | }) 137 | }) 138 | -------------------------------------------------------------------------------- /src/mount.ts: -------------------------------------------------------------------------------- 1 | import filter from 'it-filter' 2 | import take from 'it-take' 3 | import merge from 'it-merge' 4 | import { BaseDatastore } from './base.js' 5 | import * as Errors from './errors.js' 6 | import sort from 'it-sort' 7 | import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' 8 | import type { AbortOptions } from 'interface-store' 9 | 10 | /** 11 | * A datastore that can combine multiple stores inside various 12 | * key prefixes 13 | */ 14 | export class MountDatastore extends BaseDatastore { 15 | private readonly mounts: Array<{ prefix: Key, datastore: Datastore }> 16 | 17 | constructor (mounts: Array<{ prefix: Key, datastore: Datastore }>) { 18 | super() 19 | 20 | this.mounts = mounts.slice() 21 | } 22 | 23 | /** 24 | * Lookup the matching datastore for the given key 25 | */ 26 | private _lookup (key: Key): { datastore: Datastore, mountpoint: Key } | undefined { 27 | for (const mount of this.mounts) { 28 | if (mount.prefix.toString() === key.toString() || mount.prefix.isAncestorOf(key)) { 29 | return { 30 | datastore: mount.datastore, 31 | mountpoint: mount.prefix 32 | } 33 | } 34 | } 35 | } 36 | 37 | async put (key: Key, value: Uint8Array, options?: AbortOptions): Promise { 38 | const match = this._lookup(key) 39 | if (match == null) { 40 | throw Errors.dbWriteFailedError(new Error('No datastore mounted for this key')) 41 | } 42 | 43 | await match.datastore.put(key, value, options) 44 | 45 | return key 46 | } 47 | 48 | /** 49 | * @param {Key} key 50 | * @param {Options} [options] 51 | */ 52 | async get (key: Key, options: AbortOptions = {}): Promise { 53 | const match = this._lookup(key) 54 | if (match == null) { 55 | throw Errors.notFoundError(new Error('No datastore mounted for this key')) 56 | } 57 | return await match.datastore.get(key, options) 58 | } 59 | 60 | async has (key: Key, options?: AbortOptions): Promise { 61 | const match = this._lookup(key) 62 | if (match == null) { 63 | return await Promise.resolve(false) 64 | } 65 | return await match.datastore.has(key, options) 66 | } 67 | 68 | async delete (key: Key, options?: AbortOptions): Promise { 69 | const match = this._lookup(key) 70 | if (match == null) { 71 | throw Errors.dbDeleteFailedError(new Error('No datastore mounted for this key')) 72 | } 73 | 74 | await match.datastore.delete(key, options) 75 | } 76 | 77 | batch (): Batch { 78 | const batchMounts: Record = {} 79 | 80 | const lookup = (key: Key): { batch: Batch } => { 81 | const match = this._lookup(key) 82 | if (match == null) { 83 | throw new Error('No datastore mounted for this key') 84 | } 85 | 86 | const m = match.mountpoint.toString() 87 | if (batchMounts[m] == null) { 88 | batchMounts[m] = match.datastore.batch() 89 | } 90 | 91 | return { 92 | batch: batchMounts[m] 93 | } 94 | } 95 | 96 | return { 97 | put: (key, value) => { 98 | const match = lookup(key) 99 | match.batch.put(key, value) 100 | }, 101 | delete: (key) => { 102 | const match = lookup(key) 103 | match.batch.delete(key) 104 | }, 105 | commit: async (options) => { 106 | await Promise.all(Object.keys(batchMounts).map(async p => { await batchMounts[p].commit(options) })) 107 | } 108 | } 109 | } 110 | 111 | query (q: Query, options?: AbortOptions): AsyncIterable { 112 | const qs = this.mounts.map(m => { 113 | return m.datastore.query({ 114 | prefix: q.prefix, 115 | filters: q.filters 116 | }, options) 117 | }) 118 | 119 | let it = merge(...qs) 120 | if (q.filters != null) q.filters.forEach(f => { it = filter(it, f) }) 121 | if (q.orders != null) q.orders.forEach(o => { it = sort(it, o) }) 122 | if (q.offset != null) { 123 | let i = 0 124 | const offset = q.offset 125 | it = filter(it, () => i++ >= offset) 126 | } 127 | if (q.limit != null) it = take(it, q.limit) 128 | 129 | return it 130 | } 131 | 132 | queryKeys (q: KeyQuery, options?: AbortOptions): AsyncIterable { 133 | const qs = this.mounts.map(m => { 134 | return m.datastore.queryKeys({ 135 | prefix: q.prefix, 136 | filters: q.filters 137 | }, options) 138 | }) 139 | 140 | /** @type AsyncIterable */ 141 | let it = merge(...qs) 142 | if (q.filters != null) q.filters.forEach(f => { it = filter(it, f) }) 143 | if (q.orders != null) q.orders.forEach(o => { it = sort(it, o) }) 144 | if (q.offset != null) { 145 | let i = 0 146 | const offset = q.offset 147 | it = filter(it, () => i++ >= offset) 148 | } 149 | if (q.limit != null) it = take(it, q.limit) 150 | 151 | return it 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /src/keytransform.ts: -------------------------------------------------------------------------------- 1 | import { BaseDatastore } from './base.js' 2 | import map from 'it-map' 3 | import { pipe } from 'it-pipe' 4 | import type { KeyTransform } from './index.js' 5 | import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' 6 | import type { AbortOptions, AwaitIterable } from 'interface-store' 7 | 8 | /** 9 | * A datastore shim, that wraps around a given datastore, changing 10 | * the way keys look to the user, for example namespacing 11 | * keys, reversing them, etc. 12 | */ 13 | export class KeyTransformDatastore extends BaseDatastore { 14 | private readonly child: Datastore 15 | public transform: KeyTransform 16 | 17 | constructor (child: Datastore, transform: KeyTransform) { 18 | super() 19 | 20 | this.child = child 21 | this.transform = transform 22 | } 23 | 24 | async put (key: Key, val: Uint8Array, options?: AbortOptions): Promise { 25 | await this.child.put(this.transform.convert(key), val, options) 26 | 27 | return key 28 | } 29 | 30 | async get (key: Key, options?: AbortOptions): Promise { 31 | return await this.child.get(this.transform.convert(key), options) 32 | } 33 | 34 | async has (key: Key, options?: AbortOptions): Promise { 35 | return await this.child.has(this.transform.convert(key), options) 36 | } 37 | 38 | async delete (key: Key, options?: AbortOptions): Promise { 39 | await this.child.delete(this.transform.convert(key), options) 40 | } 41 | 42 | async * putMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 43 | const transform = this.transform 44 | const child = this.child 45 | 46 | yield * pipe( 47 | source, 48 | async function * (source) { 49 | yield * map(source, ({ key, value }) => ({ 50 | key: transform.convert(key), 51 | value 52 | })) 53 | }, 54 | async function * (source) { 55 | yield * child.putMany(source, options) 56 | }, 57 | async function * (source) { 58 | yield * map(source, key => transform.invert(key)) 59 | } 60 | ) 61 | } 62 | 63 | async * getMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 64 | const transform = this.transform 65 | const child = this.child 66 | 67 | yield * pipe( 68 | source, 69 | async function * (source) { 70 | yield * map(source, key => transform.convert(key)) 71 | }, 72 | async function * (source) { 73 | yield * child.getMany(source, options) 74 | }, 75 | async function * (source) { 76 | yield * map(source, ({ key, value }) => ({ 77 | key: transform.invert(key), 78 | value 79 | })) 80 | } 81 | ) 82 | } 83 | 84 | async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 85 | const transform = this.transform 86 | const child = this.child 87 | 88 | yield * pipe( 89 | source, 90 | async function * (source) { 91 | yield * map(source, key => transform.convert(key)) 92 | }, 93 | async function * (source) { 94 | yield * child.deleteMany(source, options) 95 | }, 96 | async function * (source) { 97 | yield * map(source, key => transform.invert(key)) 98 | } 99 | ) 100 | } 101 | 102 | batch (): Batch { 103 | const b = this.child.batch() 104 | return { 105 | put: (key, value) => { 106 | b.put(this.transform.convert(key), value) 107 | }, 108 | delete: (key) => { 109 | b.delete(this.transform.convert(key)) 110 | }, 111 | commit: async (options) => { 112 | await b.commit(options) 113 | } 114 | } 115 | } 116 | 117 | query (q: Query, options?: AbortOptions): AsyncIterable { 118 | const query: Query = { 119 | ...q 120 | } 121 | 122 | query.filters = (query.filters ?? []).map(filter => { 123 | return ({ key, value }) => filter({ key: this.transform.convert(key), value }) 124 | }) 125 | 126 | const { prefix } = q 127 | if (prefix != null && prefix !== '/') { 128 | delete query.prefix 129 | query.filters.push(({ key }) => { 130 | return this.transform.invert(key).toString().startsWith(prefix) 131 | }) 132 | } 133 | 134 | if (query.orders != null) { 135 | query.orders = query.orders.map(order => { 136 | return (a, b) => order( 137 | { key: this.transform.invert(a.key), value: a.value }, 138 | { key: this.transform.invert(b.key), value: b.value } 139 | ) 140 | }) 141 | } 142 | 143 | return map(this.child.query(query, options), ({ key, value }) => { 144 | return { 145 | key: this.transform.invert(key), 146 | value 147 | } 148 | }) 149 | } 150 | 151 | queryKeys (q: KeyQuery, options?: AbortOptions): AsyncIterable { 152 | const query = { 153 | ...q 154 | } 155 | 156 | query.filters = (query.filters ?? []).map(filter => { 157 | return (key) => filter(this.transform.convert(key)) 158 | }) 159 | 160 | const { prefix } = q 161 | if (prefix != null && prefix !== '/') { 162 | delete query.prefix 163 | query.filters.push((key) => { 164 | return this.transform.invert(key).toString().startsWith(prefix) 165 | }) 166 | } 167 | 168 | if (query.orders != null) { 169 | query.orders = query.orders.map(order => { 170 | return (a, b) => order( 171 | this.transform.invert(a), 172 | this.transform.invert(b) 173 | ) 174 | }) 175 | } 176 | 177 | return map(this.child.queryKeys(query, options), key => { 178 | return this.transform.invert(key) 179 | }) 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "datastore-core", 3 | "version": "9.0.4", 4 | "description": "Wrapper implementation for interface-datastore", 5 | "author": "Friedel Ziegelmayer ", 6 | "license": "Apache-2.0 OR MIT", 7 | "homepage": "https://github.com/ipfs/js-datastore-core#readme", 8 | "repository": { 9 | "type": "git", 10 | "url": "git+https://github.com/ipfs/js-datastore-core.git" 11 | }, 12 | "bugs": { 13 | "url": "https://github.com/ipfs/js-datastore-core/issues" 14 | }, 15 | "keywords": [ 16 | "datastore", 17 | "interface", 18 | "ipfs", 19 | "key-value" 20 | ], 21 | "engines": { 22 | "node": ">=16.0.0", 23 | "npm": ">=7.0.0" 24 | }, 25 | "type": "module", 26 | "types": "./dist/src/index.d.ts", 27 | "typesVersions": { 28 | "*": { 29 | "*": [ 30 | "*", 31 | "dist/*", 32 | "dist/src/*", 33 | "dist/src/*/index" 34 | ], 35 | "src/*": [ 36 | "*", 37 | "dist/*", 38 | "dist/src/*", 39 | "dist/src/*/index" 40 | ] 41 | } 42 | }, 43 | "files": [ 44 | "src", 45 | "dist", 46 | "!dist/test", 47 | "!**/*.tsbuildinfo" 48 | ], 49 | "exports": { 50 | ".": { 51 | "types": "./dist/src/index.d.ts", 52 | "import": "./dist/src/index.js" 53 | }, 54 | "./base": { 55 | "types": "./dist/src/base.d.ts", 56 | "import": "./dist/src/base.js" 57 | }, 58 | "./errors": { 59 | "types": "./dist/src/errors.d.ts", 60 | "import": "./dist/src/errors.js" 61 | }, 62 | "./keytransform": { 63 | "types": "./dist/src/keytransform.d.ts", 64 | "import": "./dist/src/keytransform.js" 65 | }, 66 | "./memory": { 67 | "types": "./dist/src/memory.d.ts", 68 | "import": "./dist/src/memory.js" 69 | }, 70 | "./mount": { 71 | "types": "./dist/src/mount.d.ts", 72 | "import": "./dist/src/mount.js" 73 | }, 74 | "./namespace": { 75 | "types": "./dist/src/namespace.d.ts", 76 | "import": "./dist/src/namespace.js" 77 | }, 78 | "./shard": { 79 | "types": "./dist/src/shard.d.ts", 80 | "import": "./dist/src/shard.js" 81 | }, 82 | "./sharding": { 83 | "types": "./dist/src/sharding.d.ts", 84 | "import": "./dist/src/sharding.js" 85 | }, 86 | "./tiered": { 87 | "types": "./dist/src/tiered.d.ts", 88 | "import": "./dist/src/tiered.js" 89 | } 90 | }, 91 | "eslintConfig": { 92 | "extends": "ipfs", 93 | "parserOptions": { 94 | "sourceType": "module" 95 | } 96 | }, 97 | "release": { 98 | "branches": [ 99 | "master" 100 | ], 101 | "plugins": [ 102 | [ 103 | "@semantic-release/commit-analyzer", 104 | { 105 | "preset": "conventionalcommits", 106 | "releaseRules": [ 107 | { 108 | "breaking": true, 109 | "release": "major" 110 | }, 111 | { 112 | "revert": true, 113 | "release": "patch" 114 | }, 115 | { 116 | "type": "feat", 117 | "release": "minor" 118 | }, 119 | { 120 | "type": "fix", 121 | "release": "patch" 122 | }, 123 | { 124 | "type": "docs", 125 | "release": "patch" 126 | }, 127 | { 128 | "type": "test", 129 | "release": "patch" 130 | }, 131 | { 132 | "type": "deps", 133 | "release": "patch" 134 | }, 135 | { 136 | "scope": "no-release", 137 | "release": false 138 | } 139 | ] 140 | } 141 | ], 142 | [ 143 | "@semantic-release/release-notes-generator", 144 | { 145 | "preset": "conventionalcommits", 146 | "presetConfig": { 147 | "types": [ 148 | { 149 | "type": "feat", 150 | "section": "Features" 151 | }, 152 | { 153 | "type": "fix", 154 | "section": "Bug Fixes" 155 | }, 156 | { 157 | "type": "chore", 158 | "section": "Trivial Changes" 159 | }, 160 | { 161 | "type": "docs", 162 | "section": "Documentation" 163 | }, 164 | { 165 | "type": "deps", 166 | "section": "Dependencies" 167 | }, 168 | { 169 | "type": "test", 170 | "section": "Tests" 171 | } 172 | ] 173 | } 174 | } 175 | ], 176 | "@semantic-release/changelog", 177 | "@semantic-release/npm", 178 | "@semantic-release/github", 179 | "@semantic-release/git" 180 | ] 181 | }, 182 | "scripts": { 183 | "clean": "aegir clean", 184 | "lint": "aegir lint", 185 | "build": "aegir build", 186 | "release": "aegir release", 187 | "test": "aegir test", 188 | "test:node": "aegir test -t node --cov", 189 | "test:chrome": "aegir test -t browser --cov", 190 | "test:chrome-webworker": "aegir test -t webworker", 191 | "test:firefox": "aegir test -t browser -- --browser firefox", 192 | "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", 193 | "test:electron-main": "aegir test -t electron-main", 194 | "dep-check": "aegir dep-check" 195 | }, 196 | "dependencies": { 197 | "@libp2p/logger": "^2.0.0", 198 | "err-code": "^3.0.1", 199 | "interface-store": "^5.0.1", 200 | "it-all": "^2.0.0", 201 | "it-drain": "^2.0.0", 202 | "it-filter": "^2.0.0", 203 | "it-map": "^2.0.0", 204 | "it-merge": "^2.0.0", 205 | "it-pipe": "^2.0.3", 206 | "it-pushable": "^3.0.0", 207 | "it-sort": "^2.0.1", 208 | "it-take": "^2.0.0", 209 | "uint8arrays": "^4.0.2" 210 | }, 211 | "devDependencies": { 212 | "aegir": "^38.1.7", 213 | "interface-datastore": "^8.1.2", 214 | "interface-datastore-tests": "^5.0.0" 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /.github/workflows/js-test-and-release.yml: -------------------------------------------------------------------------------- 1 | # File managed by web3-bot. DO NOT EDIT. 2 | # See https://github.com/protocol/.github/ for details. 3 | 4 | name: test & maybe release 5 | on: 6 | push: 7 | branches: 8 | - master 9 | pull_request: 10 | 11 | jobs: 12 | 13 | check: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | - uses: actions/setup-node@v3 18 | with: 19 | node-version: lts/* 20 | - uses: ipfs/aegir/actions/cache-node-modules@master 21 | - run: npm run --if-present lint 22 | - run: npm run --if-present dep-check 23 | 24 | test-node: 25 | needs: check 26 | runs-on: ${{ matrix.os }} 27 | strategy: 28 | matrix: 29 | os: [windows-latest, ubuntu-latest, macos-latest] 30 | node: [lts/*] 31 | fail-fast: true 32 | steps: 33 | - uses: actions/checkout@v3 34 | - uses: actions/setup-node@v3 35 | with: 36 | node-version: ${{ matrix.node }} 37 | - uses: ipfs/aegir/actions/cache-node-modules@master 38 | - run: npm run --if-present test:node 39 | - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 40 | with: 41 | flags: node 42 | 43 | test-chrome: 44 | needs: check 45 | runs-on: ubuntu-latest 46 | steps: 47 | - uses: actions/checkout@v3 48 | - uses: actions/setup-node@v3 49 | with: 50 | node-version: lts/* 51 | - uses: ipfs/aegir/actions/cache-node-modules@master 52 | - run: npm run --if-present test:chrome 53 | - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 54 | with: 55 | flags: chrome 56 | 57 | test-chrome-webworker: 58 | needs: check 59 | runs-on: ubuntu-latest 60 | steps: 61 | - uses: actions/checkout@v3 62 | - uses: actions/setup-node@v3 63 | with: 64 | node-version: lts/* 65 | - uses: ipfs/aegir/actions/cache-node-modules@master 66 | - run: npm run --if-present test:chrome-webworker 67 | - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 68 | with: 69 | flags: chrome-webworker 70 | 71 | test-firefox: 72 | needs: check 73 | runs-on: ubuntu-latest 74 | steps: 75 | - uses: actions/checkout@v3 76 | - uses: actions/setup-node@v3 77 | with: 78 | node-version: lts/* 79 | - uses: ipfs/aegir/actions/cache-node-modules@master 80 | - run: npm run --if-present test:firefox 81 | - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 82 | with: 83 | flags: firefox 84 | 85 | test-firefox-webworker: 86 | needs: check 87 | runs-on: ubuntu-latest 88 | steps: 89 | - uses: actions/checkout@v3 90 | - uses: actions/setup-node@v3 91 | with: 92 | node-version: lts/* 93 | - uses: ipfs/aegir/actions/cache-node-modules@master 94 | - run: npm run --if-present test:firefox-webworker 95 | - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 96 | with: 97 | flags: firefox-webworker 98 | 99 | test-webkit: 100 | needs: check 101 | runs-on: ${{ matrix.os }} 102 | strategy: 103 | matrix: 104 | os: [ubuntu-latest, macos-latest] 105 | node: [lts/*] 106 | fail-fast: true 107 | steps: 108 | - uses: actions/checkout@v3 109 | - uses: actions/setup-node@v3 110 | with: 111 | node-version: lts/* 112 | - uses: ipfs/aegir/actions/cache-node-modules@master 113 | - run: npm run --if-present test:webkit 114 | - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 115 | with: 116 | flags: webkit 117 | 118 | test-webkit-webworker: 119 | needs: check 120 | runs-on: ${{ matrix.os }} 121 | strategy: 122 | matrix: 123 | os: [ubuntu-latest, macos-latest] 124 | node: [lts/*] 125 | fail-fast: true 126 | steps: 127 | - uses: actions/checkout@v3 128 | - uses: actions/setup-node@v3 129 | with: 130 | node-version: lts/* 131 | - uses: ipfs/aegir/actions/cache-node-modules@master 132 | - run: npm run --if-present test:webkit-webworker 133 | - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 134 | with: 135 | flags: webkit-webworker 136 | 137 | test-electron-main: 138 | needs: check 139 | runs-on: ubuntu-latest 140 | steps: 141 | - uses: actions/checkout@v3 142 | - uses: actions/setup-node@v3 143 | with: 144 | node-version: lts/* 145 | - uses: ipfs/aegir/actions/cache-node-modules@master 146 | - run: npx xvfb-maybe npm run --if-present test:electron-main 147 | - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 148 | with: 149 | flags: electron-main 150 | 151 | test-electron-renderer: 152 | needs: check 153 | runs-on: ubuntu-latest 154 | steps: 155 | - uses: actions/checkout@v3 156 | - uses: actions/setup-node@v3 157 | with: 158 | node-version: lts/* 159 | - uses: ipfs/aegir/actions/cache-node-modules@master 160 | - run: npx xvfb-maybe npm run --if-present test:electron-renderer 161 | - uses: codecov/codecov-action@d9f34f8cd5cb3b3eb79b3e4b5dae3a16df499a70 # v3.1.1 162 | with: 163 | flags: electron-renderer 164 | 165 | release: 166 | needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-webkit, test-webkit-webworker, test-electron-main, test-electron-renderer] 167 | runs-on: ubuntu-latest 168 | if: github.event_name == 'push' && github.ref == 'refs/heads/master' 169 | steps: 170 | - uses: actions/checkout@v3 171 | with: 172 | fetch-depth: 0 173 | - uses: actions/setup-node@v3 174 | with: 175 | node-version: lts/* 176 | - uses: ipfs/aegir/actions/cache-node-modules@master 177 | - uses: ipfs/aegir/actions/docker-login@master 178 | with: 179 | docker-token: ${{ secrets.DOCKER_TOKEN }} 180 | docker-username: ${{ secrets.DOCKER_USERNAME }} 181 | - run: npm run --if-present release 182 | env: 183 | GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN || github.token }} 184 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 185 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## [9.0.4](https://github.com/ipfs/js-datastore-core/compare/v9.0.3...v9.0.4) (2023-03-23) 2 | 3 | 4 | ### Dependencies 5 | 6 | * update interface-store to 5.x.x ([#149](https://github.com/ipfs/js-datastore-core/issues/149)) ([2820b0d](https://github.com/ipfs/js-datastore-core/commit/2820b0ddce94aefe85cb9ba09b387cc59494e130)) 7 | 8 | ## [9.0.3](https://github.com/ipfs/js-datastore-core/compare/v9.0.2...v9.0.3) (2023-03-14) 9 | 10 | 11 | ### Bug Fixes 12 | 13 | * add error for db read failures ([#146](https://github.com/ipfs/js-datastore-core/issues/146)) ([afab18a](https://github.com/ipfs/js-datastore-core/commit/afab18a9b6000ebe4670abee5bd401e024d0ae0f)) 14 | 15 | ## [9.0.2](https://github.com/ipfs/js-datastore-core/compare/v9.0.1...v9.0.2) (2023-03-13) 16 | 17 | 18 | ### Bug Fixes 19 | 20 | * update shard creation ([59081a7](https://github.com/ipfs/js-datastore-core/commit/59081a7a3b8e596705fd5834b35bfb9ac73c3f39)) 21 | 22 | ## [9.0.1](https://github.com/ipfs/js-datastore-core/compare/v9.0.0...v9.0.1) (2023-03-13) 23 | 24 | 25 | ### Bug Fixes 26 | 27 | * update project config ([#145](https://github.com/ipfs/js-datastore-core/issues/145)) ([5bb5601](https://github.com/ipfs/js-datastore-core/commit/5bb5601735c59f6aed7dd52112194d233e86267b)) 28 | 29 | ## [9.0.0](https://github.com/ipfs/js-datastore-core/compare/v8.0.4...v9.0.0) (2023-03-13) 30 | 31 | 32 | ### ⚠ BREAKING CHANGES 33 | 34 | * update to latest datastore interface 35 | 36 | ### Features 37 | 38 | * update to latest datastore interface ([#144](https://github.com/ipfs/js-datastore-core/issues/144)) ([46059f1](https://github.com/ipfs/js-datastore-core/commit/46059f184bb49414fb494135ca544ca6a2d0ee66)) 39 | 40 | ## [8.0.4](https://github.com/ipfs/js-datastore-core/compare/v8.0.3...v8.0.4) (2023-01-11) 41 | 42 | 43 | ### Bug Fixes 44 | 45 | * throw error with message from error in tiered datastore ([#133](https://github.com/ipfs/js-datastore-core/issues/133)) ([2fd4be4](https://github.com/ipfs/js-datastore-core/commit/2fd4be4b72ff203b4bc3643ca469aab7a17abbc3)) 46 | 47 | ## [8.0.3](https://github.com/ipfs/js-datastore-core/compare/v8.0.2...v8.0.3) (2022-12-23) 48 | 49 | 50 | ### Dependencies 51 | 52 | * bump it-* deps ([#132](https://github.com/ipfs/js-datastore-core/issues/132)) ([4b7fd55](https://github.com/ipfs/js-datastore-core/commit/4b7fd559e21bfb5645037a177b32c973e7f05b4a)) 53 | 54 | ## [8.0.2](https://github.com/ipfs/js-datastore-core/compare/v8.0.1...v8.0.2) (2022-10-12) 55 | 56 | 57 | ### Dependencies 58 | 59 | * update uint8arrays from 3.x.x to 4.x.x ([#121](https://github.com/ipfs/js-datastore-core/issues/121)) ([16d99b7](https://github.com/ipfs/js-datastore-core/commit/16d99b72454df33453f68d644a1daa6656d9dcf2)) 60 | 61 | ## [8.0.1](https://github.com/ipfs/js-datastore-core/compare/v8.0.0...v8.0.1) (2022-08-12) 62 | 63 | 64 | ### Dependencies 65 | 66 | * update interface-datastore and interface-datastore-tests ([#118](https://github.com/ipfs/js-datastore-core/issues/118)) ([e5f47e2](https://github.com/ipfs/js-datastore-core/commit/e5f47e200059e056c34da1fcf4b795a3272f1459)) 67 | 68 | ## [8.0.0](https://github.com/ipfs/js-datastore-core/compare/v7.0.3...v8.0.0) (2022-08-12) 69 | 70 | 71 | ### ⚠ BREAKING CHANGES 72 | 73 | * this module is now ESM-only 74 | 75 | ### Features 76 | 77 | * switch to ESM-only ([#109](https://github.com/ipfs/js-datastore-core/issues/109)) ([cbaef20](https://github.com/ipfs/js-datastore-core/commit/cbaef2000a78bcdaa750e932b3a681d8e41cd727)) 78 | 79 | ### [7.0.3](https://github.com/ipfs/js-datastore-core/compare/v7.0.2...v7.0.3) (2022-07-25) 80 | 81 | 82 | ### Bug Fixes 83 | 84 | * errors export ([#111](https://github.com/ipfs/js-datastore-core/issues/111)) ([be15e74](https://github.com/ipfs/js-datastore-core/commit/be15e74bb2a505aeee81d06da2dfdc07fc919096)), closes [#109](https://github.com/ipfs/js-datastore-core/issues/109) 85 | 86 | ### [7.0.2](https://github.com/ipfs/js-datastore-core/compare/v7.0.1...v7.0.2) (2022-07-21) 87 | 88 | 89 | ### Bug Fixes 90 | 91 | * pass options to tiered put ([#108](https://github.com/ipfs/js-datastore-core/issues/108)) ([7a8f9ff](https://github.com/ipfs/js-datastore-core/commit/7a8f9ffb226c4940145637a6c881b835ce4886cb)) 92 | 93 | 94 | ### Trivial Changes 95 | 96 | * Update .github/workflows/stale.yml [skip ci] ([7e56180](https://github.com/ipfs/js-datastore-core/commit/7e56180b4d1da793a2ab3102135a4665b80bd462)) 97 | * update package.json version ([54b543d](https://github.com/ipfs/js-datastore-core/commit/54b543d08aef9c75e6bd3cb015d2a3d000ef26c6)) 98 | 99 | ### [7.0.1](https://github.com/ipfs/js-datastore-core/compare/v7.0.0...v7.0.1) (2022-01-28) 100 | 101 | 102 | ### Bug Fixes 103 | 104 | * fix generated types to missing import ([#86](https://github.com/ipfs/js-datastore-core/issues/86)) ([8805e71](https://github.com/ipfs/js-datastore-core/commit/8805e71f5840dac2dea4d3220b4f3975b39d5161)), closes [#85](https://github.com/ipfs/js-datastore-core/issues/85) 105 | 106 | ## [7.0.0](https://github.com/ipfs/js-datastore-core/compare/v6.0.7...v7.0.0) (2022-01-19) 107 | 108 | 109 | ### ⚠ BREAKING CHANGES 110 | 111 | * key prefixes are no longer stripped by MountDatastore 112 | 113 | ### Bug Fixes 114 | 115 | * do not strip prefixes for MountDatastores ([#82](https://github.com/ipfs/js-datastore-core/issues/82)) ([73bae74](https://github.com/ipfs/js-datastore-core/commit/73bae74ac794bf7585d432a3337e40fe85cb68f4)) 116 | 117 | 118 | ### Trivial Changes 119 | 120 | * switch to unified ci ([#83](https://github.com/ipfs/js-datastore-core/issues/83)) ([b3f79e6](https://github.com/ipfs/js-datastore-core/commit/b3f79e6d07440e37aabc83d1ebabe2b5c0e2bd30)) 121 | 122 | ## [6.0.7](https://github.com/ipfs/js-datastore-core/compare/v6.0.6...v6.0.7) (2021-09-09) 123 | 124 | 125 | 126 | ## [6.0.6](https://github.com/ipfs/js-datastore-core/compare/v6.0.5...v6.0.6) (2021-09-09) 127 | 128 | 129 | 130 | ## [6.0.5](https://github.com/ipfs/js-datastore-core/compare/v6.0.4...v6.0.5) (2021-09-08) 131 | 132 | 133 | 134 | ## [6.0.4](https://github.com/ipfs/js-datastore-core/compare/v6.0.3...v6.0.4) (2021-09-08) 135 | 136 | 137 | 138 | ## [6.0.3](https://github.com/ipfs/js-datastore-core/compare/v6.0.2...v6.0.3) (2021-09-08) 139 | 140 | 141 | 142 | ## [6.0.2](https://github.com/ipfs/js-datastore-core/compare/v6.0.1...v6.0.2) (2021-09-08) 143 | 144 | 145 | 146 | ## [6.0.1](https://github.com/ipfs/js-datastore-core/compare/v6.0.0...v6.0.1) (2021-09-08) 147 | 148 | 149 | 150 | # [6.0.0](https://github.com/ipfs/js-datastore-core/compare/v5.0.2...v6.0.0) (2021-09-08) 151 | 152 | 153 | ### chore 154 | 155 | * update to esm ([#71](https://github.com/ipfs/js-datastore-core/issues/71)) ([ace9a06](https://github.com/ipfs/js-datastore-core/commit/ace9a06e879ff3fef9493754a7bbc7981595d9a4)) 156 | 157 | 158 | ### BREAKING CHANGES 159 | 160 | * deep requires/imports are no longer possible 161 | 162 | 163 | 164 | ## [5.0.2](https://github.com/ipfs/js-datastore-core/compare/v5.0.1...v5.0.2) (2021-08-23) 165 | 166 | 167 | 168 | ## [5.0.1](https://github.com/ipfs/js-datastore-core/compare/v5.0.0...v5.0.1) (2021-07-23) 169 | 170 | 171 | ### Bug Fixes 172 | 173 | * use streaming methods where possible ([#65](https://github.com/ipfs/js-datastore-core/issues/65)) ([ec90398](https://github.com/ipfs/js-datastore-core/commit/ec9039827829dcd0e6e5c652ecb8707e80c48010)) 174 | 175 | 176 | 177 | # [5.0.0](https://github.com/ipfs/js-datastore-core/compare/v4.0.0...v5.0.0) (2021-07-06) 178 | 179 | 180 | ### chore 181 | 182 | * update deps ([e154404](https://github.com/ipfs/js-datastore-core/commit/e154404317838f147fbb9ab11417a8d671aba9a4)) 183 | 184 | 185 | ### BREAKING CHANGES 186 | 187 | * implements new interface-datastore types, will cause duplicates in the dep tree 188 | 189 | 190 | 191 | # [4.0.0](https://github.com/ipfs/js-datastore-core/compare/v3.0.0...v4.0.0) (2021-04-15) 192 | 193 | 194 | ### Bug Fixes 195 | 196 | * fix async sharding tests ([#49](https://github.com/ipfs/js-datastore-core/issues/49)) ([a546afb](https://github.com/ipfs/js-datastore-core/commit/a546afb0cef751025aac73c5f3bcfa3bedd1dff4)) 197 | 198 | 199 | ### Features 200 | 201 | * split .query into .query and .queryKeys ([#59](https://github.com/ipfs/js-datastore-core/issues/59)) ([6f829db](https://github.com/ipfs/js-datastore-core/commit/6f829db2e8d1aed3b9c36a7bb95625a15c077e02)) 202 | 203 | 204 | 205 | # [3.0.0](https://github.com/ipfs/js-datastore-core/compare/v2.0.1...v3.0.0) (2021-01-22) 206 | 207 | 208 | ### Bug Fixes 209 | 210 | * fix ShardingDatastore creation ([#48](https://github.com/ipfs/js-datastore-core/issues/48)) ([21b48e7](https://github.com/ipfs/js-datastore-core/commit/21b48e7ccc965a1422f8a0f7eadbd83715b5cac6)) 211 | 212 | 213 | ### Features 214 | 215 | * ts types, github ci and clean up ([#39](https://github.com/ipfs/js-datastore-core/issues/39)) ([bee45ae](https://github.com/ipfs/js-datastore-core/commit/bee45ae0b778171d0919e135ea8affcf2d6de635)) 216 | 217 | 218 | 219 | ## [2.0.1](https://github.com/ipfs/js-datastore-core/compare/v2.0.0...v2.0.1) (2020-11-09) 220 | 221 | 222 | 223 | 224 | # [2.0.0](https://github.com/ipfs/js-datastore-core/compare/v1.1.0...v2.0.0) (2020-07-29) 225 | 226 | 227 | ### Bug Fixes 228 | 229 | * remove node buffers ([#27](https://github.com/ipfs/js-datastore-core/issues/27)) ([a9786b9](https://github.com/ipfs/js-datastore-core/commit/a9786b9)) 230 | 231 | 232 | ### BREAKING CHANGES 233 | 234 | * no longer uses node Buffers, only Uint8Arrays 235 | 236 | 237 | 238 | 239 | # [1.1.0](https://github.com/ipfs/js-datastore-core/compare/v1.0.0...v1.1.0) (2020-05-07) 240 | 241 | 242 | ### Bug Fixes 243 | 244 | * **ci:** add empty commit to fix lint checks on master ([a19da65](https://github.com/ipfs/js-datastore-core/commit/a19da65)) 245 | 246 | 247 | ### Features 248 | 249 | * add streaming/cancellable API ([#23](https://github.com/ipfs/js-datastore-core/issues/23)) ([5e7858e](https://github.com/ipfs/js-datastore-core/commit/5e7858e)) 250 | 251 | 252 | 253 | 254 | # [1.0.0](https://github.com/ipfs/js-datastore-core/compare/v0.7.0...v1.0.0) (2020-04-06) 255 | 256 | 257 | ### Bug Fixes 258 | 259 | * add buffer and cleanup ([#22](https://github.com/ipfs/js-datastore-core/issues/22)) ([f0f64a9](https://github.com/ipfs/js-datastore-core/commit/f0f64a9)) 260 | 261 | 262 | 263 | 264 | # [0.7.0](https://github.com/ipfs/js-datastore-core/compare/v0.6.1...v0.7.0) (2019-05-29) 265 | 266 | 267 | 268 | 269 | ## [0.6.1](https://github.com/ipfs/js-datastore-core/compare/v0.6.0...v0.6.1) (2019-05-23) 270 | 271 | 272 | ### Bug Fixes 273 | 274 | * remove leftpad and cleanup ([8558fcd](https://github.com/ipfs/js-datastore-core/commit/8558fcd)) 275 | 276 | 277 | 278 | 279 | # [0.6.0](https://github.com/ipfs/js-datastore-core/compare/v0.5.0...v0.6.0) (2018-10-24) 280 | 281 | 282 | 283 | 284 | # [0.5.0](https://github.com/ipfs/js-datastore-core/compare/v0.4.0...v0.5.0) (2018-09-19) 285 | 286 | 287 | ### Bug Fixes 288 | 289 | * **ci:** build on appveyor ([#9](https://github.com/ipfs/js-datastore-core/issues/9)) ([687314b](https://github.com/ipfs/js-datastore-core/commit/687314b)) 290 | 291 | 292 | ### Features 293 | 294 | * add basic error codes ([bf79768](https://github.com/ipfs/js-datastore-core/commit/bf79768)) 295 | 296 | 297 | 298 | 299 | # [0.4.0](https://github.com/ipfs/js-datastore-core/compare/v0.3.0...v0.4.0) (2017-11-04) 300 | 301 | 302 | ### Bug Fixes 303 | 304 | * sharding and query for windows interop ([#6](https://github.com/ipfs/js-datastore-core/issues/6)) ([845316d](https://github.com/ipfs/js-datastore-core/commit/845316d)) 305 | 306 | 307 | 308 | 309 | # [0.3.0](https://github.com/ipfs/js-datastore-core/compare/v0.2.0...v0.3.0) (2017-07-23) 310 | 311 | 312 | 313 | 314 | # [0.2.0](https://github.com/ipfs/js-datastore-core/compare/v0.1.0...v0.2.0) (2017-03-23) 315 | 316 | 317 | ### Features 318 | 319 | * add open method ([1462fcc](https://github.com/ipfs/js-datastore-core/commit/1462fcc)) 320 | 321 | 322 | 323 | 324 | # 0.1.0 (2017-03-15) 325 | --------------------------------------------------------------------------------